From 87893819b08fa48a76a833638ff5ee909aef8bcb Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Thu, 11 Sep 2025 14:34:02 +0000 Subject: [PATCH] =?UTF-8?q?chore(fetcher):=20switch=20back=20to=20native?= =?UTF-8?q?=20fetch=20=F0=9F=A4=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 51 +++++++ package.json | 1 + src/error/index.ts | 2 +- src/extractor/VidSrc.test.ts | 38 ++--- .../__snapshots__/VidSrc.test.ts.snap | 2 +- src/utils/Fetcher.test.ts | 137 ++++++++---------- src/utils/Fetcher.ts | 10 +- src/utils/FetcherMock.ts | 1 - 8 files changed, 140 insertions(+), 102 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3c68c09..0173674 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "babel-jest": "^30.0.0", "eslint": "^9.26.0", "eslint-plugin-import": "^2.32.0", + "fetch-mock": "^12.5.4", "jest": "^30.0.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", @@ -1804,6 +1805,13 @@ "@types/send": "*" } }, + "node_modules/@types/glob-to-regexp": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz", + "integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", @@ -4118,6 +4126,16 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/detect-libc": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", @@ -5038,6 +5056,22 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "license": "MIT" }, + "node_modules/fetch-mock": { + "version": "12.5.4", + "resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.4.tgz", + "integrity": "sha512-zeECXue5losbXbbnHYzLDl0ZKLnuyRmLZ3auSJISoCnKiOGXNJA6nRiRa4qxI9pyQqeTgHL5d0Ryz+3CUliLOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/glob-to-regexp": "^0.4.4", + "dequal": "^2.0.3", + "glob-to-regexp": "^0.4.1", + "regexparam": "^3.0.0" + }, + "engines": { + "node": ">=18.11.0" + } + }, "node_modules/fetch-socks": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz", @@ -5548,6 +5582,13 @@ "node": ">=10.13.0" } }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/glob/node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", @@ -9115,6 +9156,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/regexparam": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz", + "integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/package.json b/package.json index 1b7e762..e76ab9f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "babel-jest": "^30.0.0", "eslint": "^9.26.0", "eslint-plugin-import": "^2.32.0", + "fetch-mock": "^12.5.4", "jest": "^30.0.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", diff --git a/src/error/index.ts b/src/error/index.ts index 17a3b0d..5eb8011 100644 --- a/src/error/index.ts +++ b/src/error/index.ts @@ -44,7 +44,7 @@ export const logErrorAndReturnNiceString = (ctx: Context, logger: winston.Logger if ( error instanceof TimeoutError - || (error instanceof DOMException && error.name === 'TimeoutError') // sometimes this gets through, no idea why.. + || (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) // sometimes this gets through, no idea why.. ) { logger.warn(`${source}: Request timed out.`, ctx); diff --git a/src/extractor/VidSrc.test.ts b/src/extractor/VidSrc.test.ts index 6160f5a..150f10f 100644 --- a/src/extractor/VidSrc.test.ts +++ b/src/extractor/VidSrc.test.ts @@ -1,4 +1,4 @@ -import { MockAgent, setGlobalDispatcher } from 'undici'; +import fetchMock from 'fetch-mock'; import winston from 'winston'; import { createTestContext } from '../test'; import { Fetcher, FetcherMock } from '../utils'; @@ -11,6 +11,18 @@ const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherM const ctx = createTestContext(); describe('VidSrc', () => { + beforeAll(() => { + fetchMock.mockGlobal(); + }); + + afterAll(() => { + fetchMock.unmockGlobal(); + }); + + afterEach(() => { + fetchMock.clearHistory(); + }); + test('Full Metal Jacket', async () => { expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt0093058'))).toMatchSnapshot(); }); @@ -24,14 +36,9 @@ describe('VidSrc', () => { }); test('rate limit issues are retried and fail if no tlds are left', async () => { - const mockAgent = new MockAgent({ enableCallHistory: true }); - mockAgent.disableNetConnect(); - setGlobalDispatcher(mockAgent); - - mockAgent.get('https://vidsrc.xyz') - .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429); - mockAgent.get('https://vidsrc.net') - .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429); + fetchMock + .getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/1', 429) + .getOnce('https://vidsrc.net/embed/movie/tt33043892/1/1', 429); const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); const vidSrc = new VidSrc(fetcher, ['net', 'xyz']); @@ -40,18 +47,13 @@ describe('VidSrc', () => { }); test('blocking issues are retried and fail if no tlds are left', async () => { - const mockAgent = new MockAgent({ enableCallHistory: true }); - mockAgent.disableNetConnect(); - setGlobalDispatcher(mockAgent); - - mockAgent.get('https://vidsrc.xyz') - .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403); - mockAgent.get('https://vidsrc.net') - .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403); + fetchMock + .getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/2', 403) + .getOnce('https://vidsrc.net/embed/movie/tt33043892/1/2', 403); const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); const vidSrc = new VidSrc(fetcher, ['net', 'xyz']); - expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/1'), {})).toMatchSnapshot(); + expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/2'), {})).toMatchSnapshot(); }); }); diff --git a/src/extractor/__snapshots__/VidSrc.test.ts.snap b/src/extractor/__snapshots__/VidSrc.test.ts.snap index 4d62b6c..5486738 100644 --- a/src/extractor/__snapshots__/VidSrc.test.ts.snap +++ b/src/extractor/__snapshots__/VidSrc.test.ts.snap @@ -42,7 +42,7 @@ exports[`VidSrc blocking issues are retried and fail if no tlds are left 1`] = ` "isExternal": true, "label": "VidSrc", "sourceId": "vidsrc", - "url": "https://vidsrc.xyz/embed/movie/tt33043892/1/1", + "url": "https://vidsrc.xyz/embed/movie/tt33043892/1/2", }, ] `; diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index b272ca4..34a402a 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -1,25 +1,29 @@ -import { MockAgent, setGlobalDispatcher } from 'undici'; +import fetchMock from 'fetch-mock'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { createTestContext } from '../test'; import { BlockedReason } from '../types'; import { Fetcher } from './Fetcher'; -let mockAgent: MockAgent; const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); describe('fetch', () => { const ctx = { ...createTestContext(), ip: '0.0.0.0' }; - beforeEach(() => { - mockAgent = new MockAgent({ enableCallHistory: true }); - mockAgent.disableNetConnect(); - setGlobalDispatcher(mockAgent); + beforeAll(() => { + fetchMock.mockGlobal(); + }); + + afterAll(() => { + fetchMock.unmockGlobal(); + }); + + afterEach(() => { + fetchMock.clearHistory(); }); test('text passes successful response through setting headers', async () => { - const mockPool = mockAgent.get('https://some-url.test'); - mockPool.intercept({ path: '/' }).reply(200, 'some text'); + fetchMock.get('https://some-url.test', 'some text'); const responseText1 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/')); const responseText2 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/'), { headers: { 'User-Agent': 'jest' } }); @@ -27,39 +31,37 @@ describe('fetch', () => { expect(responseText1).toBe('some text'); expect(responseText2).toStrictEqual(responseText1); - expect(mockAgent.getCallHistory()?.firstCall()?.headers).toMatchObject({ - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en', - 'Authorization': 'Basic dXNlcjpwYXNz', - 'Priority': 'u=0', - 'User-Agent': 'node', + expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({ + headers: { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + 'Authorization': 'Basic dXNlcjpwYXNz', + 'Priority': 'u=0', + 'User-Agent': 'node', + }, }); }); test('textPost ', async () => { - const mockPool = mockAgent.get('https://some-post-url.test'); - mockPool.intercept({ path: '/', method: 'POST' }).reply(200, 'some text'); + fetchMock.post('https://some-post-url.test', 'some text'); expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text'); }); test('head', async () => { - const mockPool = mockAgent.get('https://some-head-url.test'); - mockPool.intercept({ path: '/', method: 'HEAD' }).reply(200, 'some text', { headers: { 'X-Fake-Response': 'foo' } }); + fetchMock.head('https://some-head-url.test', { status: 200, headers: { 'X-Fake-Response': 'foo' } }); expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' }); }); test('converts 404 to custom NotFoundError', async () => { - const mockPool = mockAgent.get('https://some-404-url.test'); - mockPool.intercept({ path: '/' }).reply(404); + fetchMock.get('https://some-404-url.test', 404); await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError); }); test('converts Cloudflare challenge block to custom BlockedError', async () => { - const mockPool = mockAgent.get('https://some-cloudflare-url.test'); - mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); + fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); try { await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/')); @@ -72,30 +74,21 @@ describe('fetch', () => { test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => { process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x'; - const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-success-url.test'); - mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); - const mockPool2 = mockAgent.get('http://localhost:8191'); - mockPool2.intercept({ path: '/v1/x', method: 'POST' }).reply( - 200, + fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); + + fetchMock.post( + 'http://localhost:8191/v1/x', { - status: 'ok', - solution: { - response: 'some response', - cookies: [ - { - name: 'cf_clearance', - value: 'some_value', - expires: Date.now() / 1000 + 12345, - domain: 'some-cloudflare-flaresolverr-success-url.test', - }, - { - name: 'irrelevant', - value: 'some_other_value', - expires: Date.now() / 1000 + 12345, - domain: 'some-other.domain', - }, - ], - userAgent: 'user agent that works', + body: { + status: 'ok', + solution: { + response: 'some response', + cookies: [ + { name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' }, + { name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' }, + ], + userAgent: 'user agent that works', + }, }, }, ); @@ -104,11 +97,12 @@ describe('fetch', () => { expect(response).toBe('some response'); // The next request has to use the cookie and user agent we got - mockPool.intercept({ path: '/' }).reply(200, 'some response'); await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/')); - expect(mockAgent.getCallHistory()?.calls()[2]?.headers).toMatchObject({ - 'Cookie': 'cf_clearance=some_value', - 'User-Agent': 'user agent that works', + expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({ + headers: { + 'Cookie': 'cf_clearance=some_value', + 'User-Agent': 'user agent that works', + }, }); delete process.env['FLARESOLVERR_ENDPOINT']; @@ -116,10 +110,8 @@ describe('fetch', () => { test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => { process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y'; - const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-fail-url.test'); - mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); - const mockPool2 = mockAgent.get('http://localhost:8191'); - mockPool2.intercept({ path: '/v1/y', method: 'POST' }).reply(200, { status: 'nok' }); + fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); + fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } }); try { await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/')); @@ -133,8 +125,7 @@ describe('fetch', () => { }); test('converts generic forbidden to custom BlockedError', async () => { - const mockPool = mockAgent.get('https://some-forbidden-url.test'); - mockPool.intercept({ path: '/' }).reply(403); + fetchMock.get('https://some-forbidden-url.test/', { status: 403 }); try { await fetcher.text(ctx, new URL('https://some-forbidden-url.test/')); @@ -146,8 +137,7 @@ describe('fetch', () => { }); test('converts cloudflare 451 to custom BlockedError', async () => { - const mockPool = mockAgent.get('https://some-cloudflare-forbidden-url.test'); - mockPool.intercept({ path: '/' }).reply(451); + fetchMock.get('https://some-cloudflare-forbidden-url.test/', { status: 451 }); try { await fetcher.text(ctx, new URL('https://some-cloudflare-forbidden-url.test/')); @@ -159,8 +149,7 @@ describe('fetch', () => { }); test('converts MediaFlow Proxy forbidden to custom BlockedError', async () => { - const mockPool = mockAgent.get('https://media-flow-proxy-forbidden-url.test'); - mockPool.intercept({ path: '/' }).reply(403); + fetchMock.get('https://media-flow-proxy-forbidden-url.test/', { status: 403 }); try { await fetcher.text(createTestContext({ mediaFlowProxyUrl: 'https://media-flow-proxy-forbidden-url.test/' }), new URL('https://media-flow-proxy-forbidden-url.test/')); @@ -172,8 +161,7 @@ describe('fetch', () => { }); test('converts 429 to custom TooManyRequestsError', async () => { - const mockPool = mockAgent.get('https://some-rate-limited-url.test'); - mockPool.intercept({ path: '/' }).reply(429); + fetchMock.get('https://some-rate-limited-url.test/', { status: 429 }); try { await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/')); @@ -185,9 +173,7 @@ describe('fetch', () => { }); test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => { - const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test'); - mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '60' } }); - mockPool.intercept({ path: '/' }).reply(200); + fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '60' } }); try { await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/')); @@ -206,17 +192,16 @@ describe('fetch', () => { }); test('retries rate limits with short ttl', async () => { - const mockPool = mockAgent.get('https://some-rate-limited-retry-after-short-url.test'); - mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '1' } }); - mockPool.intercept({ path: '/' }).reply(200); + fetchMock + .getOnce('https://some-rate-limited-retry-after-short-url.test/', { status: 429, headers: { 'retry-after': '1' } }) + .getOnce('https://some-rate-limited-retry-after-short-url.test/', 'success'); const responseText = await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-short-url.test/')); - expect(responseText).toBe(''); + expect(responseText).toBe('success'); }); test('passes through other error as HttpError after retrying 3 times', async () => { - const mockPool = mockAgent.get('https://some-error-url.test'); - mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } }).times(4); + fetchMock.get('https://some-error-url.test', { status: 500, headers: { 'x-foo': 'bar' } }); try { await fetcher.text(ctx, new URL('https://some-error-url.test/')); @@ -228,22 +213,19 @@ describe('fetch', () => { }); test('passes through exceptions', async () => { - const mockPool = mockAgent.get('https://some-error-url.test'); - mockPool.intercept({ path: '/' }).replyWithError((new TypeError('Failed to fetch'))); + fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') }); await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError); }); test('times out, retries and times out again', async () => { - const mockPool = mockAgent.get('https://some-timeout-url.test'); - mockPool.intercept({ path: '/' }).reply(200).delay(20).persist(); + fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 }); await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError); }); test('queues requests and throws if queue is full', async () => { - const mockPool = mockAgent.get('https://some-full-queue-url.test'); - mockPool.intercept({ path: '/' }).reply(200).delay(20).persist(); + fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 }); const allPromises = Promise.all([ fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }), @@ -257,8 +239,7 @@ describe('fetch', () => { }); test('throws if too many recent requests timed-out', async () => { - const mockPool = mockAgent.get('https://too-many-recent-timeouts-url.test'); - mockPool.intercept({ path: '/' }).reply(200).delay(20).persist(); + fetchMock.get('https://too-many-recent-timeouts-url.test/', 200, { delay: 20 }); await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError); await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 3f4b866..7be9e93 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -3,7 +3,6 @@ import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex'; import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import CachePolicy from 'http-cache-semantics'; import { Cookie, CookieJar } from 'tough-cookie'; -import { fetch, Headers, RequestInit, Response } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; @@ -274,17 +273,20 @@ export class Fetcher { throw new TooManyTimeoutsError(); } + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), init?.timeout ?? this.DEFAULT_TIMEOUT); + let response; try { const finalUrl = new URL(url.href); finalUrl.username = ''; finalUrl.password = ''; - const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT) }; + const finalInit = { ...init, keepalive: true, signal: controller.signal }; response = await fetch(finalUrl, finalInit); } catch (error) { - if (error instanceof DOMException && error.name === 'TimeoutError') { + if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) { if (tryCount < 1) { this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of timeout`, ctx); await this.sleep(333); @@ -297,6 +299,8 @@ export class Fetcher { } throw error; + } finally { + clearTimeout(timer); } await this.decreaseTimeoutsCount(url); diff --git a/src/utils/FetcherMock.ts b/src/utils/FetcherMock.ts index 0c22d8e..837a546 100644 --- a/src/utils/FetcherMock.ts +++ b/src/utils/FetcherMock.ts @@ -3,7 +3,6 @@ import crypto from 'crypto'; import fs from 'node:fs'; import CachePolicy from 'http-cache-semantics'; import slugify from 'slugify'; -import { RequestInit } from 'undici'; import winston from 'winston'; import { Context } from '../types'; import { envGet } from './env';