Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

fix(fetcher): remove weird proxy headers in request

Browse source 
looks like this is finally causing weird issues and leading to some
Cloudflare blocks. we might be better of without it, let's try..
This commit is contained in:
WebStreamr 2025-08-11 11:45:44 +00:00
parent 0efdfa4a6f
commit ee09e10e6e
No known key found for this signature in database
2 changed files with 2 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/utils/Fetcher.test.ts
View file

@ -31,12 +31,8 @@ describe('fetch', () => {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'Authorization': 'Basic dXNlcjpwYXNz',
'Forwarded': 'for=0.0.0.0',
'Priority': 'u=0',
'User-Agent': 'node',
'X-Forwarded-For': '0.0.0.0',
'X-Forwarded-Proto': 'https',
'X-Real-IP': '0.0.0.0',
});
});

12
src/utils/Fetcher.ts
View file

@ -46,7 +46,6 @@ export type CustomRequestInit = RequestInit & {
timeoutsCountThrow?: number;
noCache?: boolean;
noFlareSolverr?: boolean;
noProxyHeaders?: boolean;
queueLimit?: number;
queueTimeout?: number;
timeout?: number;
@ -87,9 +86,8 @@ export class Fetcher {
return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders();
};
private getInit(ctx: Context, url: URL, init?: CustomRequestInit): CustomRequestInit {
private getInit(url: URL, init?: CustomRequestInit): CustomRequestInit {
const cookieString = this.cookieJar.getCookieStringSync(url.href);
const noProxyHeaders = init?.noProxyHeaders ?? false;
return {
...init,
@ -100,12 +98,6 @@ export class Fetcher {
'Priority': 'u=0',
'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'node',
...(cookieString && { Cookie: cookieString }),
...(ctx.ip && !noProxyHeaders && {
'Forwarded': `for=${ctx.ip}`,
'X-Forwarded-For': ctx.ip,
'X-Forwarded-Proto': url.protocol.slice(0, -1),
'X-Real-IP': ctx.ip,
}),
...init?.headers,
},
};
@ -227,7 +219,7 @@ export class Fetcher {
};
private async cachedFetch(ctx: Context, url: URL, init?: CustomRequestInit): Promise<HttpCacheItem> {
const newInit = this.getInit(ctx, url, init);
const newInit = this.getInit(url, init);
const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} };