Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

fix(fetcher): re-use proxy agents

Browse source
This commit is contained in:
WebStreamr 2025-09-14 18:18:09 +00:00
parent aded562304
commit ae32376758
No known key found for this signature in database
2 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/utils/Fetcher.ts
View file

@ -8,7 +8,7 @@ import winston from 'winston';
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
import { BlockedReason, Context } from '../types';
import { noCache } from './config';
import { createProxyAgent, getProxyForUrl } from './dispatcher';
import { getProxyAgent, getProxyForUrl } from './dispatcher';
import { envGet } from './env';
interface HttpCacheItem {
@ -288,7 +288,7 @@ export class Fetcher {
...init,
keepalive: true,
signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT),
...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }),
...(/* istanbul ignore next */ proxyUrl && { dispatcher: getProxyAgent(proxyUrl) }),
};
response = await fetch(finalUrl, finalInit);

15
src/utils/dispatcher.ts
View file

@ -24,10 +24,17 @@ export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => {
return undefined;
};
export const createProxyAgent = (proxyUrl: URL): Dispatcher => {
if (proxyUrl.protocol === 'socks5:') {
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
const proxyAgents = new Map<string, Dispatcher>();
export const getProxyAgent = (proxyUrl: URL): Dispatcher => {
let proxyAgent = proxyAgents.get(proxyUrl.href);
if (!proxyAgent) {
proxyAgent = proxyUrl.protocol === 'socks5:'
? socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) })
: new ProxyAgent({ uri: proxyUrl.href });
proxyAgents.set(proxyUrl.href, proxyAgent);
}
return new ProxyAgent({ uri: proxyUrl.href });
return proxyAgent;
};