From f90a53da680f73c08216ebbe01cc5c2aa635d17f Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 21 Jun 2025 12:03:34 +0000 Subject: [PATCH] feat(fetcher): add proxy support --- package-lock.json | 61 +++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ src/utils/Fetcher.test.ts | 25 ++++++++++++++++ src/utils/Fetcher.ts | 14 ++++++++- 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 39615f1..fe71cb1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,12 @@ "bytes": "^3.1.2", "cheerio": "^1.0.0", "express": "^5.1.0", + "fetch-socks": "^1.3.2", "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", "tough-cookie": "^5.1.2", + "undici": "^7.10.0", "unpacker": "^1.0.1", "uuid": "^11.1.0", "winston": "^3.17.0" @@ -4460,6 +4462,16 @@ "node": ">=18.11.0" } }, + "node_modules/fetch-socks": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz", + "integrity": "sha512-vkH5+Zgj2yEbU57Cei0iyLgTZ4OkEKJj56Xu3ViB5dpsl599JgEooQ3x6NVagIFRHWnWJ+7K0MO0aIV1TMgvnw==", + "license": "MIT", + "dependencies": { + "socks": "^2.8.2", + "undici": ">=6" + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -5054,6 +5066,25 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -6369,6 +6400,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" + }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -7797,6 +7834,30 @@ "node": ">=8.0.0" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.5.tgz", + "integrity": "sha512-iF+tNDQla22geJdTyJB1wM/qrX9DMRwWrciEPwWLPRWAUEM8sQiyxgckLxWT1f7+9VabJS0jTGGr4QgBuvi6Ww==", + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index 3fcb69d..fe06c4e 100644 --- a/package.json +++ b/package.json @@ -36,10 +36,12 @@ "bytes": "^3.1.2", "cheerio": "^1.0.0", "express": "^5.1.0", + "fetch-socks": "^1.3.2", "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", "tough-cookie": "^5.1.2", + "undici": "^7.10.0", "unpacker": "^1.0.1", "uuid": "^11.1.0", "winston": "^3.17.0" diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index e02e22b..c8535c2 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -1,3 +1,4 @@ +import { Agent, ProxyAgent } from 'undici'; import winston from 'winston'; import fetchMock from 'fetch-mock'; import { Fetcher } from './Fetcher'; @@ -37,6 +38,30 @@ describe('fetch', () => { }); }); + test('supports socks5 proxy', async () => { + process.env['ALL_PROXY'] = 'socks5://127.0.0.1:1080'; + const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); + delete process.env['MANIFEST_NAME']; + fetchMock.get('https://socks5-proxy-url.test/', 'some text'); + + await socks5Fetcher.text(ctx, new URL('https://socks5-proxy-url.test/')); + + // @ts-expect-error non-official property + expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(Agent); + }); + + test('supports http proxy', async () => { + process.env['ALL_PROXY'] = 'https://127.0.0.1:8080'; + const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); + delete process.env['MANIFEST_NAME']; + fetchMock.get('https://http-proxy-url.test/', 'some text'); + + await socks5Fetcher.text(ctx, new URL('https://http-proxy-url.test/')); + + // @ts-expect-error non-official property + expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(ProxyAgent); + }); + test('textPost ', async () => { fetchMock.post('https://some-post-url.test/', 'some text'); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index a3b5741..eb65646 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,3 +1,5 @@ +import { Dispatcher, ProxyAgent } from 'undici'; +import { socksDispatcher } from 'fetch-socks'; import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; @@ -57,6 +59,7 @@ export class Fetcher { private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h private readonly logger: winston.Logger; + private readonly dispatcher: Dispatcher | undefined; private readonly httpCache = new TTLCache(); private readonly rateLimitedCache = new TTLCache(); @@ -69,6 +72,15 @@ export class Fetcher { public constructor(logger: winston.Logger) { this.logger = logger; + + if (process.env['ALL_PROXY']) { + const proxyUrl = new URL(process.env['ALL_PROXY']); + if (proxyUrl.protocol === 'socks5:') { + this.dispatcher = socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); + } else { + this.dispatcher = new ProxyAgent(proxyUrl.href); + } + } } public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise { @@ -242,7 +254,7 @@ export class Fetcher { let response; try { - response = await fetch(url, { ...init, keepalive: true, signal: controller.signal }); + response = await fetch(url, { ...init, dispatcher: this.dispatcher, keepalive: true, signal: controller.signal } as RequestInit); } catch (error) { if (error instanceof DOMException && error.name === 'AbortError') { await this.increaseTimeoutsCount(url);