chore(fetcher): switch back to undici fetch 🤡
This commit is contained in:
WebStreamr
parent
0f00aecde5
commit
2a47d97d5c
7 changed files with 107 additions and 138 deletions
51
package-lock.json
generated
51
package-lock.json
generated
|
|
@ -43,7 +43,6 @@
|
|||
"babel-jest": "^30.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"fetch-mock": "^12.5.4",
|
||||
"jest": "^30.0.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
@ -1853,13 +1852,6 @@
|
|||
"@types/send": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/glob-to-regexp": {
|
||||
"version": "0.4.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz",
|
||||
"integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/http-cache-semantics": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
|
||||
|
|
@ -4187,16 +4179,6 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-libc": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
|
||||
|
|
@ -5143,22 +5125,6 @@
|
|||
"integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fetch-mock": {
|
||||
"version": "12.5.4",
|
||||
"resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.4.tgz",
|
||||
"integrity": "sha512-zeECXue5losbXbbnHYzLDl0ZKLnuyRmLZ3auSJISoCnKiOGXNJA6nRiRa4qxI9pyQqeTgHL5d0Ryz+3CUliLOw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/glob-to-regexp": "^0.4.4",
|
||||
"dequal": "^2.0.3",
|
||||
"glob-to-regexp": "^0.4.1",
|
||||
"regexparam": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-socks": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz",
|
||||
|
|
@ -5669,13 +5635,6 @@
|
|||
"node": ">=10.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-to-regexp": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
|
||||
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/glob/node_modules/brace-expansion": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
|
||||
|
|
@ -9271,16 +9230,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/regexparam": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
||||
"integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
|
|
|
|||
|
|
@ -65,7 +65,6 @@
|
|||
"babel-jest": "^30.0.0",
|
||||
"eslint": "^9.26.0",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"fetch-mock": "^12.5.4",
|
||||
"jest": "^30.0.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import fetchMock from 'fetch-mock';
|
||||
import { MockAgent, setGlobalDispatcher } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { createTestContext } from '../test';
|
||||
import { Fetcher, FetcherMock } from '../utils';
|
||||
|
|
@ -11,18 +11,6 @@ const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherM
|
|||
const ctx = createTestContext();
|
||||
|
||||
describe('VidSrc', () => {
|
||||
beforeAll(() => {
|
||||
fetchMock.mockGlobal();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fetchMock.unmockGlobal();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.clearHistory();
|
||||
});
|
||||
|
||||
test('Full Metal Jacket', async () => {
|
||||
expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt0093058'))).toMatchSnapshot();
|
||||
});
|
||||
|
|
@ -36,9 +24,14 @@ describe('VidSrc', () => {
|
|||
});
|
||||
|
||||
test('rate limit issues are retried and fail if no tlds are left', async () => {
|
||||
fetchMock
|
||||
.getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/1', 429)
|
||||
.getOnce('https://vidsrc.net/embed/movie/tt33043892/1/1', 429);
|
||||
const mockAgent = new MockAgent({ enableCallHistory: true });
|
||||
mockAgent.disableNetConnect();
|
||||
setGlobalDispatcher(mockAgent);
|
||||
|
||||
mockAgent.get('https://vidsrc.xyz')
|
||||
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
|
||||
mockAgent.get('https://vidsrc.net')
|
||||
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
|
||||
|
||||
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
const vidSrc = new VidSrc(fetcher, ['net', 'xyz']);
|
||||
|
|
@ -47,13 +40,18 @@ describe('VidSrc', () => {
|
|||
});
|
||||
|
||||
test('blocking issues are retried and fail if no tlds are left', async () => {
|
||||
fetchMock
|
||||
.getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/2', 403)
|
||||
.getOnce('https://vidsrc.net/embed/movie/tt33043892/1/2', 403);
|
||||
const mockAgent = new MockAgent({ enableCallHistory: true });
|
||||
mockAgent.disableNetConnect();
|
||||
setGlobalDispatcher(mockAgent);
|
||||
|
||||
mockAgent.get('https://vidsrc.xyz')
|
||||
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403);
|
||||
mockAgent.get('https://vidsrc.net')
|
||||
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403);
|
||||
|
||||
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
const vidSrc = new VidSrc(fetcher, ['net', 'xyz']);
|
||||
|
||||
expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/2'), {})).toMatchSnapshot();
|
||||
expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/1'), {})).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ exports[`VidSrc blocking issues are retried and fail if no tlds are left 1`] = `
|
|||
"isExternal": true,
|
||||
"label": "VidSrc",
|
||||
"sourceId": "vidsrc",
|
||||
"url": "https://vidsrc.xyz/embed/movie/tt33043892/1/2",
|
||||
"url": "https://vidsrc.xyz/embed/movie/tt33043892/1/1",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -1,29 +1,25 @@
|
|||
import fetchMock from 'fetch-mock';
|
||||
import { MockAgent, setGlobalDispatcher } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
|
||||
import { createTestContext } from '../test';
|
||||
import { BlockedReason } from '../types';
|
||||
import { Fetcher } from './Fetcher';
|
||||
|
||||
let mockAgent: MockAgent;
|
||||
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
|
||||
describe('fetch', () => {
|
||||
const ctx = { ...createTestContext(), ip: '0.0.0.0' };
|
||||
|
||||
beforeAll(() => {
|
||||
fetchMock.mockGlobal();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fetchMock.unmockGlobal();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.clearHistory();
|
||||
beforeEach(() => {
|
||||
mockAgent = new MockAgent({ enableCallHistory: true });
|
||||
mockAgent.disableNetConnect();
|
||||
setGlobalDispatcher(mockAgent);
|
||||
});
|
||||
|
||||
test('text passes successful response through setting headers', async () => {
|
||||
fetchMock.get('https://some-url.test', 'some text');
|
||||
const mockPool = mockAgent.get('https://some-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200, 'some text');
|
||||
|
||||
const responseText1 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/'));
|
||||
const responseText2 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/'), { headers: { 'User-Agent': 'jest' } });
|
||||
|
|
@ -31,37 +27,39 @@ describe('fetch', () => {
|
|||
expect(responseText1).toBe('some text');
|
||||
expect(responseText2).toStrictEqual(responseText1);
|
||||
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Authorization': 'Basic dXNlcjpwYXNz',
|
||||
'Priority': 'u=0',
|
||||
'User-Agent': 'node',
|
||||
},
|
||||
expect(mockAgent.getCallHistory()?.firstCall()?.headers).toMatchObject({
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Authorization': 'Basic dXNlcjpwYXNz',
|
||||
'Priority': 'u=0',
|
||||
'User-Agent': 'node',
|
||||
});
|
||||
});
|
||||
|
||||
test('textPost ', async () => {
|
||||
fetchMock.post('https://some-post-url.test', 'some text');
|
||||
const mockPool = mockAgent.get('https://some-post-url.test');
|
||||
mockPool.intercept({ path: '/', method: 'POST' }).reply(200, 'some text');
|
||||
|
||||
expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text');
|
||||
});
|
||||
|
||||
test('head', async () => {
|
||||
fetchMock.head('https://some-head-url.test', { status: 200, headers: { 'X-Fake-Response': 'foo' } });
|
||||
const mockPool = mockAgent.get('https://some-head-url.test');
|
||||
mockPool.intercept({ path: '/', method: 'HEAD' }).reply(200, 'some text', { headers: { 'X-Fake-Response': 'foo' } });
|
||||
|
||||
expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
|
||||
});
|
||||
|
||||
test('converts 404 to custom NotFoundError', async () => {
|
||||
fetchMock.get('https://some-404-url.test', 404);
|
||||
const mockPool = mockAgent.get('https://some-404-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(404);
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
|
||||
});
|
||||
|
||||
test('converts Cloudflare challenge block to custom BlockedError', async () => {
|
||||
fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'));
|
||||
|
|
@ -74,21 +72,30 @@ describe('fetch', () => {
|
|||
|
||||
test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => {
|
||||
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x';
|
||||
fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
|
||||
fetchMock.post(
|
||||
'http://localhost:8191/v1/x',
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-success-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool2 = mockAgent.get('http://localhost:8191');
|
||||
mockPool2.intercept({ path: '/v1/x', method: 'POST' }).reply(
|
||||
200,
|
||||
{
|
||||
body: {
|
||||
status: 'ok',
|
||||
solution: {
|
||||
response: 'some response',
|
||||
cookies: [
|
||||
{ name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' },
|
||||
{ name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' },
|
||||
],
|
||||
userAgent: 'user agent that works',
|
||||
},
|
||||
status: 'ok',
|
||||
solution: {
|
||||
response: 'some response',
|
||||
cookies: [
|
||||
{
|
||||
name: 'cf_clearance',
|
||||
value: 'some_value',
|
||||
expires: Date.now() / 1000 + 12345,
|
||||
domain: 'some-cloudflare-flaresolverr-success-url.test',
|
||||
},
|
||||
{
|
||||
name: 'irrelevant',
|
||||
value: 'some_other_value',
|
||||
expires: Date.now() / 1000 + 12345,
|
||||
domain: 'some-other.domain',
|
||||
},
|
||||
],
|
||||
userAgent: 'user agent that works',
|
||||
},
|
||||
},
|
||||
);
|
||||
|
|
@ -97,12 +104,11 @@ describe('fetch', () => {
|
|||
expect(response).toBe('some response');
|
||||
|
||||
// The next request has to use the cookie and user agent we got
|
||||
mockPool.intercept({ path: '/' }).reply(200, 'some response');
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
|
||||
expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Cookie': 'cf_clearance=some_value',
|
||||
'User-Agent': 'user agent that works',
|
||||
},
|
||||
expect(mockAgent.getCallHistory()?.calls()[2]?.headers).toMatchObject({
|
||||
'Cookie': 'cf_clearance=some_value',
|
||||
'User-Agent': 'user agent that works',
|
||||
});
|
||||
|
||||
delete process.env['FLARESOLVERR_ENDPOINT'];
|
||||
|
|
@ -110,8 +116,10 @@ describe('fetch', () => {
|
|||
|
||||
test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => {
|
||||
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y';
|
||||
fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } });
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-fail-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
|
||||
const mockPool2 = mockAgent.get('http://localhost:8191');
|
||||
mockPool2.intercept({ path: '/v1/y', method: 'POST' }).reply(200, { status: 'nok' });
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/'));
|
||||
|
|
@ -125,7 +133,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts generic forbidden to custom BlockedError', async () => {
|
||||
fetchMock.get('https://some-forbidden-url.test/', { status: 403 });
|
||||
const mockPool = mockAgent.get('https://some-forbidden-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-forbidden-url.test/'));
|
||||
|
|
@ -137,7 +146,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts cloudflare 451 to custom BlockedError', async () => {
|
||||
fetchMock.get('https://some-cloudflare-forbidden-url.test/', { status: 451 });
|
||||
const mockPool = mockAgent.get('https://some-cloudflare-forbidden-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(451);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-forbidden-url.test/'));
|
||||
|
|
@ -149,7 +159,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts MediaFlow Proxy forbidden to custom BlockedError', async () => {
|
||||
fetchMock.get('https://media-flow-proxy-forbidden-url.test/', { status: 403 });
|
||||
const mockPool = mockAgent.get('https://media-flow-proxy-forbidden-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(403);
|
||||
|
||||
try {
|
||||
await fetcher.text(createTestContext({ mediaFlowProxyUrl: 'https://media-flow-proxy-forbidden-url.test/' }), new URL('https://media-flow-proxy-forbidden-url.test/'));
|
||||
|
|
@ -161,7 +172,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts 429 to custom TooManyRequestsError', async () => {
|
||||
fetchMock.get('https://some-rate-limited-url.test/', { status: 429 });
|
||||
const mockPool = mockAgent.get('https://some-rate-limited-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(429);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/'));
|
||||
|
|
@ -173,7 +185,9 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => {
|
||||
fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '60' } });
|
||||
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '60' } });
|
||||
mockPool.intercept({ path: '/' }).reply(200);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
|
||||
|
|
@ -192,16 +206,17 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('retries rate limits with short ttl', async () => {
|
||||
fetchMock
|
||||
.getOnce('https://some-rate-limited-retry-after-short-url.test/', { status: 429, headers: { 'retry-after': '1' } })
|
||||
.getOnce('https://some-rate-limited-retry-after-short-url.test/', 'success');
|
||||
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-short-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '1' } });
|
||||
mockPool.intercept({ path: '/' }).reply(200);
|
||||
|
||||
const responseText = await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-short-url.test/'));
|
||||
expect(responseText).toBe('success');
|
||||
expect(responseText).toBe('');
|
||||
});
|
||||
|
||||
test('passes through other error as HttpError after retrying 3 times', async () => {
|
||||
fetchMock.get('https://some-error-url.test', { status: 500, headers: { 'x-foo': 'bar' } });
|
||||
const mockPool = mockAgent.get('https://some-error-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } }).times(4);
|
||||
|
||||
try {
|
||||
await fetcher.text(ctx, new URL('https://some-error-url.test/'));
|
||||
|
|
@ -213,19 +228,22 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('passes through exceptions', async () => {
|
||||
fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') });
|
||||
const mockPool = mockAgent.get('https://some-error-url.test');
|
||||
mockPool.intercept({ path: '/' }).replyWithError((new TypeError('Failed to fetch')));
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError);
|
||||
});
|
||||
|
||||
test('times out, retries and times out again', async () => {
|
||||
fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://some-timeout-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
});
|
||||
|
||||
test('queues requests and throws if queue is full', async () => {
|
||||
fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://some-full-queue-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
|
||||
|
||||
const allPromises = Promise.all([
|
||||
fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
|
||||
|
|
@ -239,7 +257,8 @@ describe('fetch', () => {
|
|||
});
|
||||
|
||||
test('throws if too many recent requests timed-out', async () => {
|
||||
fetchMock.get('https://too-many-recent-timeouts-url.test/', 200, { delay: 20 });
|
||||
const mockPool = mockAgent.get('https://too-many-recent-timeouts-url.test');
|
||||
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
|
|||
import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
|
||||
import CachePolicy from 'http-cache-semantics';
|
||||
import { Cookie, CookieJar } from 'tough-cookie';
|
||||
import { fetch, Headers, RequestInit, Response } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
|
||||
import { BlockedReason, Context } from '../types';
|
||||
|
|
@ -274,16 +275,20 @@ export class Fetcher {
|
|||
throw new TooManyTimeoutsError();
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), init?.timeout ?? this.DEFAULT_TIMEOUT);
|
||||
|
||||
let response;
|
||||
try {
|
||||
const finalUrl = new URL(url.href);
|
||||
finalUrl.username = '';
|
||||
finalUrl.password = '';
|
||||
|
||||
const finalInit = { ...init, keepalive: true, signal: controller.signal, dispatcher: createDispatcher(ctx, url) };
|
||||
const dispatcher = createDispatcher(ctx, url);
|
||||
|
||||
const finalInit = {
|
||||
...init,
|
||||
keepalive: true,
|
||||
signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT),
|
||||
...(/* istanbul ignore next */ dispatcher && { dispatcher }),
|
||||
};
|
||||
|
||||
response = await fetch(finalUrl, finalInit);
|
||||
} catch (error) {
|
||||
|
|
@ -293,8 +298,6 @@ export class Fetcher {
|
|||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
await this.decreaseTimeoutsCount(url);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import crypto from 'crypto';
|
|||
import fs from 'node:fs';
|
||||
import CachePolicy from 'http-cache-semantics';
|
||||
import slugify from 'slugify';
|
||||
import { RequestInit } from 'undici';
|
||||
import winston from 'winston';
|
||||
import { Context } from '../types';
|
||||
import { envGet } from './env';
|
||||
|
|
|
|||
Loading…
Reference in a new issue