diff --git a/package-lock.json b/package-lock.json index 8d19f0e..2a1328f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,6 @@ "babel-jest": "^30.0.0", "eslint": "^9.26.0", "eslint-plugin-import": "^2.32.0", - "fetch-mock": "^12.5.2", "jest": "^30.0.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", @@ -1606,13 +1605,6 @@ "@types/send": "*" } }, - "node_modules/@types/glob-to-regexp": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz", - "integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", @@ -3581,16 +3573,6 @@ "node": ">= 0.8" } }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -4473,22 +4455,6 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "license": "MIT" }, - "node_modules/fetch-mock": { - "version": "12.5.3", - "resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.3.tgz", - "integrity": "sha512-SiqPv1IXvDjNjLWCvfFUltba3VeiYucxjyynoVW8Ft07GLFQRitlzjYZI/f5wZpeQFRIVZ84fmMgJfjwb/dAEA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/glob-to-regexp": "^0.4.4", - "dequal": "^2.0.3", - "glob-to-regexp": "^0.4.1", - "regexparam": "^3.0.0" - }, - "engines": { - "node": ">=18.11.0" - } - }, "node_modules/fetch-socks": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz", @@ -4901,13 +4867,6 @@ "node": ">=10.13.0" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/glob/node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", @@ -7704,16 +7663,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/regexparam": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz", - "integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/package.json b/package.json index e93ac4c..4e88c0e 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,6 @@ "babel-jest": "^30.0.0", "eslint": "^9.26.0", "eslint-plugin-import": "^2.32.0", - "fetch-mock": "^12.5.2", "jest": "^30.0.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", diff --git a/src/index.ts b/src/index.ts index 1cb5752..0f7662b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,6 @@ import express, { NextFunction, Request, Response } from 'express'; +import { socksDispatcher } from 'fetch-socks'; +import { ProxyAgent, setGlobalDispatcher } from 'undici'; import { v4 as uuidv4 } from 'uuid'; import winston from 'winston'; import { ConfigureController, ManifestController, StreamController } from './controller'; @@ -17,6 +19,14 @@ const logger = winston.createLogger({ ], }); +if (process.env['ALL_PROXY']) { + const proxyUrl = new URL(process.env['ALL_PROXY']); + if (proxyUrl.protocol === 'socks5:') { + setGlobalDispatcher(socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) })); + } else { + setGlobalDispatcher(new ProxyAgent({ uri: proxyUrl.href })); + } +} const fetcher = new Fetcher(logger); const sources: Source[] = [ diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 31501d2..6b106a7 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -1,23 +1,25 @@ -import fetchMock from 'fetch-mock'; -import { Agent, ProxyAgent } from 'undici'; +import { MockAgent, setGlobalDispatcher } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { createTestContext } from '../test'; import { BlockedReason } from '../types'; import { Fetcher } from './Fetcher'; -fetchMock.mockGlobal(); +let mockAgent: MockAgent; const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); describe('fetch', () => { const ctx = { ...createTestContext(), ip: '0.0.0.0' }; - afterEach(() => { - fetchMock.clearHistory(); + beforeEach(() => { + mockAgent = new MockAgent({ enableCallHistory: true }); + mockAgent.disableNetConnect(); + setGlobalDispatcher(mockAgent); }); test('text passes successful response through setting headers', async () => { - fetchMock.get('https://some-url.test/', 'some text'); + const mockPool = mockAgent.get('https://some-url.test'); + mockPool.intercept({ path: '/' }).reply(200, 'some text'); const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/')); const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } }); @@ -25,64 +27,42 @@ describe('fetch', () => { expect(responseText1).toBe('some text'); expect(responseText2).toStrictEqual(responseText1); - expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({ - headers: { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en', - 'Forwarded': 'for=0.0.0.0', - 'Priority': 'u=0', - 'User-Agent': 'node', - 'X-Forwarded-For': '0.0.0.0', - 'X-Forwarded-Proto': 'https', - 'X-Real-IP': '0.0.0.0', - }, + expect(mockAgent.getCallHistory()?.firstCall()?.headers).toMatchObject({ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + 'Forwarded': 'for=0.0.0.0', + 'Priority': 'u=0', + 'User-Agent': 'node', + 'X-Forwarded-For': '0.0.0.0', + 'X-Forwarded-Proto': 'https', + 'X-Real-IP': '0.0.0.0', }); }); - test('supports socks5 proxy', async () => { - process.env['ALL_PROXY'] = 'socks5://127.0.0.1:1080'; - const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); - delete process.env['MANIFEST_NAME']; - fetchMock.get('https://socks5-proxy-url.test/', 'some text'); - - await socks5Fetcher.text(ctx, new URL('https://socks5-proxy-url.test/')); - - // @ts-expect-error non-official property - expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(Agent); - }); - - test('supports http proxy', async () => { - process.env['ALL_PROXY'] = 'https://127.0.0.1:8080'; - const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); - delete process.env['MANIFEST_NAME']; - fetchMock.get('https://http-proxy-url.test/', 'some text'); - - await socks5Fetcher.text(ctx, new URL('https://http-proxy-url.test/')); - - // @ts-expect-error non-official property - expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(ProxyAgent); - }); - test('textPost ', async () => { - fetchMock.post('https://some-post-url.test/', 'some text'); + const mockPool = mockAgent.get('https://some-post-url.test'); + mockPool.intercept({ path: '/', method: 'POST' }).reply(200, 'some text'); expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text'); }); test('head', async () => { - fetchMock.head('https://some-head-url.test/', { status: 200, headers: { 'X-Fake-Response': 'foo' } }); + const mockPool = mockAgent.get('https://some-head-url.test'); + mockPool.intercept({ path: '/', method: 'HEAD' }).reply(200, 'some text', { headers: { 'X-Fake-Response': 'foo' } }); expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' }); }); test('converts 404 to custom NotFoundError', async () => { - fetchMock.get('https://some-404-url.test/', 404); + const mockPool = mockAgent.get('https://some-404-url.test'); + mockPool.intercept({ path: '/' }).reply(404); await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError); }); test('converts Cloudflare challenge block to custom BlockedError', async () => { - fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); + const mockPool = mockAgent.get('https://some-cloudflare-url.test'); + mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); try { await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/')); @@ -95,20 +75,30 @@ describe('fetch', () => { test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => { process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x'; - fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); - fetchMock.post( - 'http://localhost:8191/v1/x', + const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-success-url.test'); + mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); + const mockPool2 = mockAgent.get('http://localhost:8191'); + mockPool2.intercept({ path: '/v1/x', method: 'POST' }).reply( + 200, { - body: { - status: 'ok', - solution: { - response: 'some response', - cookies: [ - { name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' }, - { name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' }, - ], - userAgent: 'user agent that works', - }, + status: 'ok', + solution: { + response: 'some response', + cookies: [ + { + name: 'cf_clearance', + value: 'some_value', + expires: Date.now() / 1000 + 12345, + domain: 'some-cloudflare-flaresolverr-success-url.test', + }, + { + name: 'irrelevant', + value: 'some_other_value', + expires: Date.now() / 1000 + 12345, + domain: 'some-other.domain', + }, + ], + userAgent: 'user agent that works', }, }, ); @@ -117,12 +107,11 @@ describe('fetch', () => { expect(response).toBe('some response'); // The next request has to use the cookie and user agent we got + mockPool.intercept({ path: '/' }).reply(200, 'some response'); await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/')); - expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({ - headers: { - 'Cookie': 'cf_clearance=some_value', - 'User-Agent': 'user agent that works', - }, + expect(mockAgent.getCallHistory()?.calls()[2]?.headers).toMatchObject({ + 'Cookie': 'cf_clearance=some_value', + 'User-Agent': 'user agent that works', }); delete process.env['FLARESOLVERR_ENDPOINT']; @@ -130,8 +119,10 @@ describe('fetch', () => { test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => { process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y'; - fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); - fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } }); + const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-fail-url.test'); + mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } }); + const mockPool2 = mockAgent.get('http://localhost:8191'); + mockPool2.intercept({ path: '/v1/y', method: 'POST' }).reply(200, { status: 'nok' }); try { await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/')); @@ -145,7 +136,8 @@ describe('fetch', () => { }); test('converts generic forbidden to custom BlockedError', async () => { - fetchMock.get('https://some-forbidden-url.test/', { status: 403 }); + const mockPool = mockAgent.get('https://some-forbidden-url.test'); + mockPool.intercept({ path: '/' }).reply(403); try { await fetcher.text(ctx, new URL('https://some-forbidden-url.test/')); @@ -157,7 +149,8 @@ describe('fetch', () => { }); test('converts MediaFlow Proxy forbidden to custom BlockedError', async () => { - fetchMock.get('https://media-flow-proxy-forbidden-url.test/', { status: 403 }); + const mockPool = mockAgent.get('https://media-flow-proxy-forbidden-url.test'); + mockPool.intercept({ path: '/' }).reply(403); try { await fetcher.text(createTestContext({ mediaFlowProxyUrl: 'https://media-flow-proxy-forbidden-url.test/' }), new URL('https://media-flow-proxy-forbidden-url.test/')); @@ -169,7 +162,8 @@ describe('fetch', () => { }); test('converts 429 to custom TooManyRequestsError', async () => { - fetchMock.get('https://some-rate-limited-url.test/', { status: 429 }); + const mockPool = mockAgent.get('https://some-rate-limited-url.test'); + mockPool.intercept({ path: '/' }).reply(429); try { await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/')); @@ -181,7 +175,8 @@ describe('fetch', () => { }); test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => { - fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '10' } }); + const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test'); + mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '10' } }); try { await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/')); @@ -200,7 +195,8 @@ describe('fetch', () => { }); test('passes through other error as HttpError', async () => { - fetchMock.get('https://some-error-url.test/', { status: 500, headers: { 'x-foo': 'bar' } }); + const mockPool = mockAgent.get('https://some-error-url.test'); + mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } }); try { await fetcher.text(ctx, new URL('https://some-error-url.test/')); @@ -212,19 +208,22 @@ describe('fetch', () => { }); test('passes through exceptions', async () => { - fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') }); + const mockPool = mockAgent.get('https://some-error-url.test'); + mockPool.intercept({ path: '/' }).replyWithError((new TypeError('Failed to fetch'))); await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError); }); test('times out', async () => { - fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 }); + const mockPool = mockAgent.get('https://some-timeout-url.test'); + mockPool.intercept({ path: '/' }).reply(200).delay(20); await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError); }); test('queues requests and throws if queue is full', async () => { - fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 }); + const mockPool = mockAgent.get('https://some-full-queue-url.test'); + mockPool.intercept({ path: '/' }).reply(200).delay(20); const allPromises = Promise.all([ fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }), @@ -238,10 +237,13 @@ describe('fetch', () => { }); test('throws if too many recent requests timed-out', async () => { - fetchMock.get('https://too-many-recent-timeouts-url.test/', 200, { delay: 20 }); + const mockPool = mockAgent.get('https://too-many-recent-timeouts-url.test'); + mockPool.intercept({ path: '/' }).reply(200).delay(20); await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError); + mockPool.intercept({ path: '/' }).reply(200).delay(20); await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError); + mockPool.intercept({ path: '/' }).reply(200).delay(20); await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TooManyTimeoutsError); }); }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 1ca621d..5574fbc 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,12 +1,10 @@ -import { clearTimeout } from 'node:timers'; import { gunzipSync, gzipSync } from 'zlib'; import TTLCache from '@isaacs/ttlcache'; import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex'; -import { socksDispatcher } from 'fetch-socks'; import CachePolicy from 'http-cache-semantics'; import { LRUCache } from 'lru-cache'; import { Cookie, CookieJar } from 'tough-cookie'; -import { Dispatcher, ProxyAgent } from 'undici'; +import { fetch, RequestInit, Response } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; @@ -61,7 +59,6 @@ export class Fetcher { private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h private readonly logger: winston.Logger; - private readonly dispatcher: Dispatcher | undefined; private readonly httpCache = new LRUCache({ max: 1024 }); private readonly rateLimitedCache = new TTLCache({ max: 1024 }); @@ -74,15 +71,6 @@ export class Fetcher { public constructor(logger: winston.Logger) { this.logger = logger; - - if (process.env['ALL_PROXY']) { - const proxyUrl = new URL(process.env['ALL_PROXY']); - if (proxyUrl.protocol === 'socks5:') { - this.dispatcher = socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); - } else { - this.dispatcher = new ProxyAgent(proxyUrl.href); - } - } } public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise { @@ -263,21 +251,16 @@ export class Fetcher { throw new TooManyTimeoutsError(); } - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), init?.timeout ?? this.DEFAULT_TIMEOUT); - let response; try { - response = await fetch(url, { ...init, dispatcher: this.dispatcher, keepalive: true, signal: controller.signal } as RequestInit); + response = await fetch(url, { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT) }); } catch (error) { - if (error instanceof DOMException && error.name === 'AbortError') { + if (error instanceof DOMException && error.name === 'TimeoutError') { await this.increaseTimeoutsCount(url); throw new TimeoutError(); } throw error; - } finally { - clearTimeout(timer); } await this.decreaseTimeoutsCount(url); diff --git a/src/utils/FetcherMock.ts b/src/utils/FetcherMock.ts index 78f9bca..53f48a5 100644 --- a/src/utils/FetcherMock.ts +++ b/src/utils/FetcherMock.ts @@ -3,6 +3,7 @@ import crypto from 'crypto'; import fs from 'node:fs'; import CachePolicy from 'http-cache-semantics'; import slugify from 'slugify'; +import { RequestInit } from 'undici'; import winston from 'winston'; import { Context } from '../types'; import { envGet } from './env';