Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

revert: Revert "chore(fetcher): try to remove the proxy headers (again)"

Browse source 
This reverts commit 00bd11495f.
This commit is contained in:
WebStreamr 2025-09-23 08:32:59 +00:00
parent 9709c31bcd
commit 7e771c2644
No known key found for this signature in database
2 changed files with 14 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/utils/Fetcher.test.ts
View file

@ -31,8 +31,12 @@ describe('fetch', () => {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'Authorization': 'Basic dXNlcjpwYXNz',
'Forwarded': 'for=0.0.0.0',
'Priority': 'u=0',
'User-Agent': 'node',
'X-Forwarded-For': '0.0.0.0',
'X-Forwarded-Proto': 'https',
'X-Real-IP': '0.0.0.0',
});
});

12
src/utils/Fetcher.ts
View file

@ -48,6 +48,7 @@ export type CustomRequestInit = RequestInit & {
minCacheTtl?: number;
noCache?: boolean;
noFlareSolverr?: boolean;
noProxyHeaders?: boolean;
queueLimit?: number;
queueTimeout?: number;
timeout?: number;
@ -100,8 +101,9 @@ export class Fetcher {
return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).headers;
};
private getInit(url: URL, init?: CustomRequestInit): CustomRequestInit {
private getInit(ctx: Context, url: URL, init?: CustomRequestInit): CustomRequestInit {
const cookieString = this.cookieJar.getCookieStringSync(url.href);
const noProxyHeaders = init?.noProxyHeaders ?? false;
return {
...init,
@ -112,6 +114,12 @@ export class Fetcher {
'Priority': 'u=0',
'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'node',
...(cookieString && { Cookie: cookieString }),
...(ctx.ip && !noProxyHeaders && {
'Forwarded': `for=${ctx.ip}`,
'X-Forwarded-For': ctx.ip,
'X-Forwarded-Proto': url.protocol.slice(0, -1),
'X-Real-IP': ctx.ip,
}),
...init?.headers,
},
};
@ -228,7 +236,7 @@ export class Fetcher {
};
private async cachedFetch(ctx: Context, url: URL, init?: CustomRequestInit): Promise<HttpCacheItem> {
const newInit = this.getInit(url, init);
const newInit = this.getInit(ctx, url, init);
const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} };