diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index eb5e9c0..77be3a1 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -8,7 +8,7 @@ import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; import { noCache } from './config'; -import { createDispatcher } from './dispatcher'; +import { createProxyAgent, getProxyForUrl } from './dispatcher'; import { envGet } from './env'; interface HttpCacheItem { @@ -254,7 +254,9 @@ export class Fetcher { }; protected async fetchWithTimeout(ctx: Context, url: URL, init?: CustomRequestInit, tryCount = 0): Promise { - this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx); + const proxyUrl = getProxyForUrl(ctx, url); + + this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url} via proxy ${proxyUrl}`, ctx); const isRateLimitedRaw = await this.rateLimitedCache.getRaw(url.host); /* istanbul ignore if */ @@ -282,13 +284,11 @@ export class Fetcher { finalUrl.username = ''; finalUrl.password = ''; - const dispatcher = createDispatcher(ctx, url); - const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT), - ...(/* istanbul ignore next */ dispatcher && { dispatcher }), + ...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }), }; response = await fetch(finalUrl, finalInit); diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index 1e4ac48..5b78976 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -39,7 +39,7 @@ export class StreamResolver { const streams: Stream[] = []; - let sourceErrorOccurred = false; + let sourceErrorCount = 0; const urlResults: UrlResult[] = []; const sourcePromises = sources.map(async (source) => { if (!source.contentTypes.includes(type)) { @@ -55,7 +55,7 @@ export class StreamResolver { urlResults.push(...sourceUrlResults.flat()); } catch (error) { - sourceErrorOccurred = true; + sourceErrorCount++; if (showErrors(ctx.config)) { streams.push({ @@ -86,7 +86,7 @@ export class StreamResolver { return a.label.localeCompare(b.label); }); - const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, 0); + const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, sourceErrorCount); this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx); streams.push( @@ -107,7 +107,7 @@ export class StreamResolver { })), ); - const ttl = !sourceErrorOccurred ? this.determineTtl(urlResults) : undefined; + const ttl = sourceErrorCount === 0 ? this.determineTtl(urlResults) : undefined; return { streams, diff --git a/src/utils/dispatcher.ts b/src/utils/dispatcher.ts index e15b340..8e05c77 100644 --- a/src/utils/dispatcher.ts +++ b/src/utils/dispatcher.ts @@ -3,15 +3,7 @@ import { minimatch } from 'minimatch'; import { Dispatcher, ProxyAgent } from 'undici'; import { Context } from '../types'; -const createProxyAgent = (proxyUrl: URL): Dispatcher => { - if (proxyUrl.protocol === 'socks5:') { - return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); - } - - return new ProxyAgent({ uri: proxyUrl.href }); -}; - -const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { +export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => { const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG']; if (proxyConfig) { @@ -22,16 +14,20 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined = } if (hostPattern === '*' || minimatch(url.host, hostPattern)) { - return createProxyAgent(new URL(proxy)); + return proxy === 'false' ? undefined : new URL(proxy); } } } else if (process.env['ALL_PROXY']) { - return createProxyAgent(new URL(process.env['ALL_PROXY'])); + return new URL(process.env['ALL_PROXY']); } return undefined; }; -export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { - return createBasicDispatcher(ctx, url); +export const createProxyAgent = (proxyUrl: URL): Dispatcher => { + if (proxyUrl.protocol === 'socks5:') { + return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); + } + + return new ProxyAgent({ uri: proxyUrl.href }); };