Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

revert: "fix(fetcher): remove weird proxy headers in request"

Browse source 
This reverts commit ee09e10e6e.
This commit is contained in:
WebStreamr 2025-08-12 15:15:03 +00:00
parent 8a81a5c14e
commit e28de7d4d9
No known key found for this signature in database
2 changed files with 14 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/utils/Fetcher.test.ts
View file

@ -31,8 +31,12 @@ describe('fetch', () => {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'Authorization': 'Basic dXNlcjpwYXNz',
'Forwarded': 'for=0.0.0.0',
'Priority': 'u=0',
'User-Agent': 'node',
'X-Forwarded-For': '0.0.0.0',
'X-Forwarded-Proto': 'https',
'X-Real-IP': '0.0.0.0',
});
});

12
src/utils/Fetcher.ts
View file

@ -47,6 +47,7 @@ export type CustomRequestInit = RequestInit & {
timeoutsCountThrow?: number;
noCache?: boolean;
noFlareSolverr?: boolean;
noProxyHeaders?: boolean;
queueLimit?: number;
queueTimeout?: number;
timeout?: number;
@ -87,8 +88,9 @@ export class Fetcher {
return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).headers;
};
private getInit(url: URL, init?: CustomRequestInit): CustomRequestInit {
private getInit(ctx: Context, url: URL, init?: CustomRequestInit): CustomRequestInit {
const cookieString = this.cookieJar.getCookieStringSync(url.href);
const noProxyHeaders = init?.noProxyHeaders ?? false;
return {
...init,
@ -99,6 +101,12 @@ export class Fetcher {
'Priority': 'u=0',
'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'node',
...(cookieString && { Cookie: cookieString }),
...(ctx.ip && !noProxyHeaders && {
'Forwarded': `for=${ctx.ip}`,
'X-Forwarded-For': ctx.ip,
'X-Forwarded-Proto': url.protocol.slice(0, -1),
'X-Real-IP': ctx.ip,
}),
...init?.headers,
},
};
@ -215,7 +223,7 @@ export class Fetcher {
};
private async cachedFetch(ctx: Context, url: URL, init?: CustomRequestInit): Promise<HttpCacheItem> {
const newInit = this.getInit(url, init);
const newInit = this.getInit(ctx, url, init);
const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} };