refactor(fetcher): switch to undici fetch
This commit is contained in:
WebStreamr
parent
b11bba761a
commit
2c210941aa
6 changed files with 90 additions and 146 deletions
51
package-lock.json
generated
51
package-lock.json
generated
|
|
@ -40,7 +40,6 @@
|
|||
"babel-jest": "^30.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"fetch-mock": "^12.5.2",
|
||||
"jest": "^30.0.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
@ -1606,13 +1605,6 @@
|
|||
"@types/send": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/glob-to-regexp": {
|
||||
"version": "0.4.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz",
|
||||
"integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/http-cache-semantics": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
|
||||
|
|
@ -3581,16 +3573,6 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-newline": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
|
||||
|
|
@ -4473,22 +4455,6 @@
|
|||
"integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fetch-mock": {
|
||||
"version": "12.5.3",
|
||||
"resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.3.tgz",
|
||||
"integrity": "sha512-SiqPv1IXvDjNjLWCvfFUltba3VeiYucxjyynoVW8Ft07GLFQRitlzjYZI/f5wZpeQFRIVZ84fmMgJfjwb/dAEA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/glob-to-regexp": "^0.4.4",
|
||||
"dequal": "^2.0.3",
|
||||
"glob-to-regexp": "^0.4.1",
|
||||
"regexparam": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-socks": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz",
|
||||
|
|
@ -4901,13 +4867,6 @@
|
|||
"node": ">=10.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-to-regexp": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
|
||||
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/glob/node_modules/brace-expansion": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
|
||||
|
|
@ -7704,16 +7663,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/regexparam": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
||||
"integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
|
|
|
|||
|
|
@ -62,7 +62,6 @@
|
|||
"babel-jest": "^30.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"fetch-mock": "^12.5.2",
|
||||
"jest": "^30.0.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
|
|||
10
src/index.ts
10
src/index.ts
|
|
@ -1,4 +1,6 @@
|
|||
import express, { NextFunction, Request, Response } from 'express';
|
||||
import { socksDispatcher } from 'fetch-socks';
|
||||
import { ProxyAgent, setGlobalDispatcher } from 'undici';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import winston from 'winston';
|
||||
import { ConfigureController, ManifestController, StreamController } from './controller';
|
||||
|
|
@ -17,6 +19,14 @@ const logger = winston.createLogger({
|
|||
],
|
||||
});
|
||||
|
||||
if (process.env['ALL_PROXY']) {
|
||||
const proxyUrl = new URL(process.env['ALL_PROXY']);
|
||||
if (proxyUrl.protocol === 'socks5:') {
|
||||
setGlobalDispatcher(socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }));
|
||||
} else {
|
||||
setGlobalDispatcher(new ProxyAgent({ uri: proxyUrl.href }));
|
||||
}
|
||||
}
|
||||
const fetcher = new Fetcher(logger);
|
||||
|
||||
const sources: Source[] = [
|
||||
|
|
|
|||
|
|
@ -1,23 +1,25 @@
|
|||
import fetchMock from 'fetch-mock';
|
||||
import { Agent, ProxyAgent } from 'undici';
|
||||
import { MockAgent, setGlobalDispatcher } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
|
||||
import { createTestContext } from '../test';
|
||||
import { BlockedReason } from '../types';
|
||||
import { Fetcher } from './Fetcher';
|
||||
fetchMock.mockGlobal();
|
||||
|
||||
let mockAgent: MockAgent;
|
||||
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
|
||||
describe('fetch', () => {
|
||||
const ctx = { ...createTestContext(), ip: '0.0.0.0' };
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.clearHistory();
|
||||
beforeEach(() => {
|
||||
mockAgent = new MockAgent({ enableCallHistory: true });
|
||||
mockAgent.disableNetConnect();
|
||||
setGlobalDispatcher(mockAgent);
|
||||
});
|
||||
|
||||
test('text passes successful response through setting headers', async () => {
|
||||
fetchMock.get('https://some-url.test/', 'some text');
|
||||
const mockPool = mockAgent.get('https://some-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200, 'some text');
|
||||
|
||||
const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/'));
|
||||
const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
|
||||
|
|
@ -25,64 +27,42 @@ describe('fetch', () => {
|
|||
expect(responseText1).toBe('some text');
|
||||
expect(responseText2).toStrictEqual(responseText1);
|
||||
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Forwarded': 'for=0.0.0.0',
|
||||
'Priority': 'u=0',
|
||||
'User-Agent': 'node',
|
||||
'X-Forwarded-For': '0.0.0.0',
|
||||
'X-Forwarded-Proto': 'https',
|
||||
'X-Real-IP': '0.0.0.0',
|
||||
},
|
||||
expect(mockAgent.getCallHistory()?.firstCall()?.headers).toMatchObject({
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Forwarded': 'for=0.0.0.0',
|
||||
'Priority': 'u=0',
|
||||
'User-Agent': 'node',
|
||||
'X-Forwarded-For': '0.0.0.0',
|
||||
'X-Forwarded-Proto': 'https',
|
||||
'X-Real-IP': '0.0.0.0',
|
||||
});
|
||||
});
|
||||
|
||||
test('supports socks5 proxy', async () => {
|
||||
process.env['ALL_PROXY'] = 'socks5://127.0.0.1:1080';
|
||||
const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
delete process.env['MANIFEST_NAME'];
|
||||
fetchMock.get('https://socks5-proxy-url.test/', 'some text');
|
||||
|
||||
await socks5Fetcher.text(ctx, new URL('https://socks5-proxy-url.test/'));
|
||||
|
||||
// @ts-expect-error non-official property
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(Agent);
|
||||
});
|
||||
|
||||
test('supports http proxy', async () => {
|
||||
process.env['ALL_PROXY'] = 'https://127.0.0.1:8080';
|
||||
const socks5Fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
delete process.env['MANIFEST_NAME'];
|
||||
fetchMock.get('https://http-proxy-url.test/', 'some text');
|
||||
|
||||
await socks5Fetcher.text(ctx, new URL('https://http-proxy-url.test/'));
|
||||
|
||||
// @ts-expect-error non-official property
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]['dispatcher']).toBeInstanceOf(ProxyAgent);
|
||||
});
|
||||
|
||||
test('textPost ', async () => {
|
||||
fetchMock.post('https://some-post-url.test/', 'some text');
|
||||
const mockPool = mockAgent.get('https://some-post-url.test');
|
||||
mockPool.intercept({ path: '/', method: 'POST' }).reply(200, 'some text');
|
||||
|
||||
expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text');
|
||||
});
|
||||
|
||||
test('head', async () => {
|
||||
fetchMock.head('https://some-head-url.test/', { status: 200, headers: { 'X-Fake-Response': 'foo' } });
|
||||
const mockPool = mockAgent.get('https://some-head-url.test');
|
||||
mockPool.intercept({ path: '/', method: 'HEAD' }).reply(200, 'some text', { headers: { 'X-Fake-Response': 'foo' } });
|
||||
|
||||
expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
|
||||
});
|
||||
|
||||
test('converts 404 to custom NotFoundError', async () => {
|
||||
fetchMock.get('https://some-404-url.test/', 404);
|
||||
const mockPool = mockAgent.get('https://some-404-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(404);
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
|
||||
});
|
||||
|
||||
test('converts Cloudflare challenge block to custom BlockedError', async () => {
|
||||
fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'));
|
||||
|
|
@ -95,20 +75,30 @@ describe('fetch', () => {
|
|||
|
||||
test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => {
|
||||
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x';
|
||||
fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
fetchMock.post(
|
||||
'http://localhost:8191/v1/x',
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-success-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool2 = mockAgent.get('http://localhost:8191');
|
||||
mockPool2.intercept({ path: '/v1/x', method: 'POST' }).reply(
|
||||
200,
|
||||
{
|
||||
body: {
|
||||
status: 'ok',
|
||||
solution: {
|
||||
response: 'some response',
|
||||
cookies: [
|
||||
{ name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' },
|
||||
{ name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' },
|
||||
],
|
||||
userAgent: 'user agent that works',
|
||||
},
|
||||
status: 'ok',
|
||||
solution: {
|
||||
response: 'some response',
|
||||
cookies: [
|
||||
{
|
||||
name: 'cf_clearance',
|
||||
value: 'some_value',
|
||||
expires: Date.now() / 1000 + 12345,
|
||||
domain: 'some-cloudflare-flaresolverr-success-url.test',
|
||||
},
|
||||
{
|
||||
name: 'irrelevant',
|
||||
value: 'some_other_value',
|
||||
expires: Date.now() / 1000 + 12345,
|
||||
domain: 'some-other.domain',
|
||||
},
|
||||
],
|
||||
userAgent: 'user agent that works',
|
||||
},
|
||||
},
|
||||
);
|
||||
|
|
@ -117,12 +107,11 @@ describe('fetch', () => {
|
|||
expect(response).toBe('some response');
|
||||
|
||||
// The next request has to use the cookie and user agent we got
|
||||
mockPool.intercept({ path: '/' }).reply(200, 'some response');
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
|
||||
expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Cookie': 'cf_clearance=some_value',
|
||||
'User-Agent': 'user agent that works',
|
||||
},
|
||||
expect(mockAgent.getCallHistory()?.calls()[2]?.headers).toMatchObject({
|
||||
'Cookie': 'cf_clearance=some_value',
|
||||
'User-Agent': 'user agent that works',
|
||||
});
|
||||
|
||||
delete process.env['FLARESOLVERR_ENDPOINT'];
|
||||
|
|
@ -130,8 +119,10 @@ describe('fetch', () => {
|
|||
|
||||
test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => {
|
||||
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y';
|
||||
fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } });
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-fail-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool2 = mockAgent.get('http://localhost:8191');
|
||||
mockPool2.intercept({ path: '/v1/y', method: 'POST' }).reply(200, { status: 'nok' });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/'));
|
||||
|
|
@ -145,7 +136,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts generic forbidden to custom BlockedError', async () => {
|
||||
fetchMock.get('https://some-forbidden-url.test/', { status: 403 });
|
||||
const mockPool = mockAgent.get('https://some-forbidden-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-forbidden-url.test/'));
|
||||
|
|
@ -157,7 +149,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts MediaFlow Proxy forbidden to custom BlockedError', async () => {
|
||||
fetchMock.get('https://media-flow-proxy-forbidden-url.test/', { status: 403 });
|
||||
const mockPool = mockAgent.get('https://media-flow-proxy-forbidden-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403);
|
||||
|
||||
try {
|
||||
await fetcher.text(createTestContext({ mediaFlowProxyUrl: 'https://media-flow-proxy-forbidden-url.test/' }), new URL('https://media-flow-proxy-forbidden-url.test/'));
|
||||
|
|
@ -169,7 +162,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts 429 to custom TooManyRequestsError', async () => {
|
||||
fetchMock.get('https://some-rate-limited-url.test/', { status: 429 });
|
||||
const mockPool = mockAgent.get('https://some-rate-limited-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(429);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/'));
|
||||
|
|
@ -181,7 +175,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => {
|
||||
fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '10' } });
|
||||
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '10' } });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
|
||||
|
|
@ -200,7 +195,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('passes through other error as HttpError', async () => {
|
||||
fetchMock.get('https://some-error-url.test/', { status: 500, headers: { 'x-foo': 'bar' } });
|
||||
const mockPool = mockAgent.get('https://some-error-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-error-url.test/'));
|
||||
|
|
@ -212,19 +208,22 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('passes through exceptions', async () => {
|
||||
fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') });
|
||||
const mockPool = mockAgent.get('https://some-error-url.test');
|
||||
mockPool.intercept({ path: '/' }).replyWithError((new TypeError('Failed to fetch')));
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError);
|
||||
});
|
||||
|
||||
test('times out', async () => {
|
||||
fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://some-timeout-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20);
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
});
|
||||
|
||||
test('queues requests and throws if queue is full', async () => {
|
||||
fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://some-full-queue-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20);
|
||||
|
||||
const allPromises = Promise.all([
|
||||
fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
|
||||
|
|
@ -238,10 +237,13 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('throws if too many recent requests timed-out', async () => {
|
||||
fetchMock.get('https://too-many-recent-timeouts-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://too-many-recent-timeouts-url.test');
|
||||
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20);
|
||||
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20);
|
||||
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20);
|
||||
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TooManyTimeoutsError);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
import { clearTimeout } from 'node:timers';
|
||||
import { gunzipSync, gzipSync } from 'zlib';
|
||||
import TTLCache from '@isaacs/ttlcache';
|
||||
import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
|
||||
import { socksDispatcher } from 'fetch-socks';
|
||||
import CachePolicy from 'http-cache-semantics';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { Cookie, CookieJar } from 'tough-cookie';
|
||||
import { Dispatcher, ProxyAgent } from 'undici';
|
||||
import { fetch, RequestInit, Response } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
|
||||
import { BlockedReason, Context } from '../types';
|
||||
|
|
@ -61,7 +59,6 @@ export class Fetcher {
|
|||
private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h
|
||||
|
||||
private readonly logger: winston.Logger;
|
||||
private readonly dispatcher: Dispatcher | undefined;
|
||||
|
||||
private readonly httpCache = new LRUCache<string, Buffer>({ max: 1024 });
|
||||
private readonly rateLimitedCache = new TTLCache<string, undefined>({ max: 1024 });
|
||||
|
|
@ -74,15 +71,6 @@ export class Fetcher {
|
|||
|
||||
public constructor(logger: winston.Logger) {
|
||||
this.logger = logger;
|
||||
|
||||
if (process.env['ALL_PROXY']) {
|
||||
const proxyUrl = new URL(process.env['ALL_PROXY']);
|
||||
if (proxyUrl.protocol === 'socks5:') {
|
||||
this.dispatcher = socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
|
||||
} else {
|
||||
this.dispatcher = new ProxyAgent(proxyUrl.href);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise<string> {
|
||||
|
|
@ -263,21 +251,16 @@ export class Fetcher {
|
|||
throw new TooManyTimeoutsError();
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), init?.timeout ?? this.DEFAULT_TIMEOUT);
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, { ...init, dispatcher: this.dispatcher, keepalive: true, signal: controller.signal } as RequestInit);
|
||||
response = await fetch(url, { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT) });
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
await this.increaseTimeoutsCount(url);
|
||||
throw new TimeoutError();
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
await this.decreaseTimeoutsCount(url);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import crypto from 'crypto';
|
|||
import fs from 'node:fs';
|
||||
import CachePolicy from 'http-cache-semantics';
|
||||
import slugify from 'slugify';
|
||||
import { RequestInit } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { Context } from '../types';
|
||||
import { envGet } from './env';
|
||||
|
|
|
|||
Loading…
Reference in a new issue