feat(fetcher): add proxy support

package-lock.json

@@ -14,10 +14,12 @@
         "bytes": "^3.1.2",
         "cheerio": "^1.0.0",
         "express": "^5.1.0",
+        "fetch-socks": "^1.3.2",
         "http-cache-semantics": "^4.2.0",
         "randomstring": "^1.3.1",
         "slugify": "^1.6.6",
         "tough-cookie": "^5.1.2",
+        "undici": "^7.10.0",
         "unpacker": "^1.0.1",
         "uuid": "^11.1.0",
         "winston": "^3.17.0"
@@ -4460,6 +4462,16 @@
         "node": ">=18.11.0"
       }
     },
+    "node_modules/fetch-socks": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz",
+      "integrity": "sha512-vkH5+Zgj2yEbU57Cei0iyLgTZ4OkEKJj56Xu3ViB5dpsl599JgEooQ3x6NVagIFRHWnWJ+7K0MO0aIV1TMgvnw==",
+      "license": "MIT",
+      "dependencies": {
+        "socks": "^2.8.2",
+        "undici": ">=6"
+      }
+    },
     "node_modules/file-entry-cache": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
@@ -5054,6 +5066,25 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "license": "ISC"
     },
+    "node_modules/ip-address": {
+      "version": "9.0.5",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
+      "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==",
+      "license": "MIT",
+      "dependencies": {
+        "jsbn": "1.1.0",
+        "sprintf-js": "^1.1.3"
+      },
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/ip-address/node_modules/sprintf-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+      "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
+      "license": "BSD-3-Clause"
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6369,6 +6400,12 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsbn": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz",
+      "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==",
+      "license": "MIT"
+    },
     "node_modules/jsesc": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz",
@@ -7797,6 +7834,30 @@
         "node": ">=8.0.0"
       }
     },
+    "node_modules/smart-buffer": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
+      "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks": {
+      "version": "2.8.5",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.5.tgz",
+      "integrity": "sha512-iF+tNDQla22geJdTyJB1wM/qrX9DMRwWrciEPwWLPRWAUEM8sQiyxgckLxWT1f7+9VabJS0jTGGr4QgBuvi6Ww==",
+      "license": "MIT",
+      "dependencies": {
+        "ip-address": "^9.0.5",
+        "smart-buffer": "^4.2.0"
+      },
+      "engines": {
+        "node": ">= 10.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
     "node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

package.json

@@ -36,10 +36,12 @@
     "bytes": "^3.1.2",
     "cheerio": "^1.0.0",
     "express": "^5.1.0",
+    "fetch-socks": "^1.3.2",
     "http-cache-semantics": "^4.2.0",
     "randomstring": "^1.3.1",
     "slugify": "^1.6.6",
     "tough-cookie": "^5.1.2",
+    "undici": "^7.10.0",
     "unpacker": "^1.0.1",
     "uuid": "^11.1.0",
     "winston": "^3.17.0"

src/utils/Fetcher.test.ts

@@ -1,3 +1,4 @@
+import { Agent, ProxyAgent } from 'undici';
 import winston from 'winston';
 import fetchMock from 'fetch-mock';
 import { Fetcher } from './Fetcher';
@@ -37,6 +38,30 @@ describe('fetch', () => {
     });
   });
 
+  test('supports socks5 proxy', async () => {
+    process.env['ALL_PROXY'] = 'socks5://127.0.0.1:1080';
+    const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
+    delete process.env['MANIFEST_NAME'];
+    fetchMock.get('https://socks5-proxy-url.test/', 'some text');
+
+    await socks5Fetcher.text(ctx, new URL('https://socks5-proxy-url.test/'));
+
+    // @ts-expect-error non-official property
+    expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(Agent);
+  });
+
+  test('supports http proxy', async () => {
+    process.env['ALL_PROXY'] = 'https://127.0.0.1:8080';
+    const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
+    delete process.env['MANIFEST_NAME'];
+    fetchMock.get('https://http-proxy-url.test/', 'some text');
+
+    await socks5Fetcher.text(ctx, new URL('https://http-proxy-url.test/'));
+
+    // @ts-expect-error non-official property
+    expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(ProxyAgent);
+  });
+
   test('textPost ', async () => {
     fetchMock.post('https://some-post-url.test/', 'some text');
 

src/utils/Fetcher.ts

@@ -1,3 +1,5 @@
+import { Dispatcher, ProxyAgent } from 'undici';
+import { socksDispatcher } from 'fetch-socks';
 import CachePolicy from 'http-cache-semantics';
 import TTLCache from '@isaacs/ttlcache';
 import winston from 'winston';
@@ -57,6 +59,7 @@ export class Fetcher {
   private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h
 
   private readonly logger: winston.Logger;
+  private readonly dispatcher: Dispatcher | undefined;
 
   private readonly httpCache = new TTLCache<string, HttpCacheItem>();
   private readonly rateLimitedCache = new TTLCache<string, undefined>();
@@ -69,6 +72,15 @@ export class Fetcher {
 
   public constructor(logger: winston.Logger) {
     this.logger = logger;
+
+    if (process.env['ALL_PROXY']) {
+      const proxyUrl = new URL(process.env['ALL_PROXY']);
+      if (proxyUrl.protocol === 'socks5:') {
+        this.dispatcher = socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
+      } else {
+        this.dispatcher = new ProxyAgent(proxyUrl.href);
+      }
+    }
   }
 
   public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise<string> {
@@ -242,7 +254,7 @@ export class Fetcher {
 
     let response;
     try {
-      response = await fetch(url, { ...init, keepalive: true, signal: controller.signal });
+      response = await fetch(url, { ...init, dispatcher: this.dispatcher, keepalive: true, signal: controller.signal } as RequestInit);
     } catch (error) {
       if (error instanceof DOMException && error.name === 'AbortError') {
         await this.increaseTimeoutsCount(url);