Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore(fetcher): remove interceptors and retries

Browse source
This commit is contained in:
WebStreamr 2025-09-12 09:49:45 +00:00
parent 81208ab99b
commit 7ee87ceb9e
No known key found for this signature in database
2 changed files with 8 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/utils/Fetcher.ts
View file

@ -288,13 +288,6 @@ export class Fetcher {
response = await fetch(finalUrl, finalInit);
} catch (error) {
if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) {
if (tryCount < 1) {
this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of timeout`, ctx);
await this.sleep(333);
return await this.fetchWithTimeout(ctx, url, init, ++tryCount);
}
await this.increaseTimeoutsCount(url);
throw new TimeoutError();
}
@ -309,7 +302,7 @@ export class Fetcher {
if (response.status === 429) {
const retryAfter = parseInt(`${response.headers.get('retry-after')}`) * 1000;
if (retryAfter <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) {
this.logger.info('Wait out rate limit', ctx);
this.logger.info(`Wait out rate limit for ${url.host}`, ctx);
await this.sleep(retryAfter);
@ -317,13 +310,6 @@ export class Fetcher {
}
}
if (response.status >= 500 && tryCount < 3) {
this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of error`, ctx);
await this.sleep(333);
return await this.fetchWithTimeout(ctx, url, init, ++tryCount);
}
return response;
};

17
src/utils/dispatcher.ts
View file

@ -1,17 +1,17 @@
import { socksDispatcher } from 'fetch-socks';
import { minimatch } from 'minimatch';
import { Agent, Dispatcher, interceptors, ProxyAgent } from 'undici';
import { Dispatcher, ProxyAgent } from 'undici';
import { Context } from '../types';
const createProxyAgent = (proxyUrl: URL): Dispatcher => {
if (proxyUrl.protocol === 'socks5:') {
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }, { allowH2: true });
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
}
return new ProxyAgent({ uri: proxyUrl.href, allowH2: true });
return new ProxyAgent({ uri: proxyUrl.href });
};
const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher => {
const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => {
const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG'];
if (proxyConfig) {
@ -29,12 +29,9 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher => {
return createProxyAgent(new URL(process.env['ALL_PROXY']));
}
return new Agent({ allowH2: true });
return undefined;
};
export const createDispatcher = (ctx: Context, url: URL): Dispatcher => {
return createBasicDispatcher(ctx, url).compose(
interceptors.dns(),
interceptors.retry({ maxRetries: 3 }),
);
export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => {
return createBasicDispatcher(ctx, url);
};