Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore(fetcher): switch back to native fetch 🤡

Browse source
This commit is contained in:
WebStreamr 2025-09-11 14:34:02 +00:00
parent c29ec82d6a
commit 87893819b0
No known key found for this signature in database
8 changed files with 140 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
package-lock.json generated
View file

@ -42,6 +42,7 @@
"babel-jest": "^30.0.0",
"eslint": "^9.26.0",
"eslint-plugin-import": "^2.32.0",
"fetch-mock": "^12.5.4",
"jest": "^30.0.0",
"nodemon": "^3.1.10",
"ts-jest": "^29.3.2",
@ -1804,6 +1805,13 @@
"@types/send": "*"
}
},
"node_modules/@types/glob-to-regexp": {
"version": "0.4.4",
"resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz",
"integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/http-cache-semantics": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
@ -4118,6 +4126,16 @@
"node": ">= 0.8"
}
},
"node_modules/dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/detect-libc": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
@ -5038,6 +5056,22 @@
"integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==",
"license": "MIT"
},
"node_modules/fetch-mock": {
"version": "12.5.4",
"resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.4.tgz",
"integrity": "sha512-zeECXue5losbXbbnHYzLDl0ZKLnuyRmLZ3auSJISoCnKiOGXNJA6nRiRa4qxI9pyQqeTgHL5d0Ryz+3CUliLOw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/glob-to-regexp": "^0.4.4",
"dequal": "^2.0.3",
"glob-to-regexp": "^0.4.1",
"regexparam": "^3.0.0"
},
"engines": {
"node": ">=18.11.0"
}
},
"node_modules/fetch-socks": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.2.tgz",
@ -5548,6 +5582,13 @@
"node": ">=10.13.0"
}
},
"node_modules/glob-to-regexp": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
"dev": true,
"license": "BSD-2-Clause"
},
"node_modules/glob/node_modules/brace-expansion": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
@ -9115,6 +9156,16 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/regexparam": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
"integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",

1
package.json
View file

@ -64,6 +64,7 @@
"babel-jest": "^30.0.0",
"eslint": "^9.26.0",
"eslint-plugin-import": "^2.32.0",
"fetch-mock": "^12.5.4",
"jest": "^30.0.0",
"nodemon": "^3.1.10",
"ts-jest": "^29.3.2",

2
src/error/index.ts
View file

@ -44,7 +44,7 @@ export const logErrorAndReturnNiceString = (ctx: Context, logger: winston.Logger
if (
error instanceof TimeoutError
|| (error instanceof DOMException && error.name === 'TimeoutError') // sometimes this gets through, no idea why..
|| (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) // sometimes this gets through, no idea why..
) {
logger.warn(`${source}: Request timed out.`, ctx);

38
src/extractor/VidSrc.test.ts
View file

@ -1,4 +1,4 @@
import { MockAgent, setGlobalDispatcher } from 'undici';
import fetchMock from 'fetch-mock';
import winston from 'winston';
import { createTestContext } from '../test';
import { Fetcher, FetcherMock } from '../utils';
@ -11,6 +11,18 @@ const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherM
const ctx = createTestContext();
describe('VidSrc', () => {
beforeAll(() => {
fetchMock.mockGlobal();
});
afterAll(() => {
fetchMock.unmockGlobal();
});
afterEach(() => {
fetchMock.clearHistory();
});
test('Full Metal Jacket', async () => {
expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt0093058'))).toMatchSnapshot();
});
@ -24,14 +36,9 @@ describe('VidSrc', () => {
});
test('rate limit issues are retried and fail if no tlds are left', async () => {
const mockAgent = new MockAgent({ enableCallHistory: true });
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
mockAgent.get('https://vidsrc.xyz')
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
mockAgent.get('https://vidsrc.net')
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
fetchMock
.getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/1', 429)
.getOnce('https://vidsrc.net/embed/movie/tt33043892/1/1', 429);
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
const vidSrc = new VidSrc(fetcher, ['net', 'xyz']);
@ -40,18 +47,13 @@ describe('VidSrc', () => {
});
test('blocking issues are retried and fail if no tlds are left', async () => {
const mockAgent = new MockAgent({ enableCallHistory: true });
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
mockAgent.get('https://vidsrc.xyz')
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403);
mockAgent.get('https://vidsrc.net')
.intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(403);
fetchMock
.getOnce('https://vidsrc.xyz/embed/movie/tt33043892/1/2', 403)
.getOnce('https://vidsrc.net/embed/movie/tt33043892/1/2', 403);
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
const vidSrc = new VidSrc(fetcher, ['net', 'xyz']);
expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/1'), {})).toMatchSnapshot();
expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/2'), {})).toMatchSnapshot();
});
});

2
src/extractor/__snapshots__/VidSrc.test.ts.snap
View file

@ -42,7 +42,7 @@ exports[`VidSrc blocking issues are retried and fail if no tlds are left 1`] = `
"isExternal": true,
"label": "VidSrc",
"sourceId": "vidsrc",
"url": "https://vidsrc.xyz/embed/movie/tt33043892/1/1",
"url": "https://vidsrc.xyz/embed/movie/tt33043892/1/2",
},
]
`;

137
src/utils/Fetcher.test.ts
View file

@ -1,25 +1,29 @@
import { MockAgent, setGlobalDispatcher } from 'undici';
import fetchMock from 'fetch-mock';
import winston from 'winston';
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
import { createTestContext } from '../test';
import { BlockedReason } from '../types';
import { Fetcher } from './Fetcher';
let mockAgent: MockAgent;
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
describe('fetch', () => {
const ctx = { ...createTestContext(), ip: '0.0.0.0' };
beforeEach(() => {
mockAgent = new MockAgent({ enableCallHistory: true });
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
beforeAll(() => {
fetchMock.mockGlobal();
});
afterAll(() => {
fetchMock.unmockGlobal();
});
afterEach(() => {
fetchMock.clearHistory();
});
test('text passes successful response through setting headers', async () => {
const mockPool = mockAgent.get('https://some-url.test');
mockPool.intercept({ path: '/' }).reply(200, 'some text');
fetchMock.get('https://some-url.test', 'some text');
const responseText1 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/'));
const responseText2 = await fetcher.text(ctx, new URL('https://user:pass@some-url.test/'), { headers: { 'User-Agent': 'jest' } });
@ -27,39 +31,37 @@ describe('fetch', () => {
expect(responseText1).toBe('some text');
expect(responseText2).toStrictEqual(responseText1);
expect(mockAgent.getCallHistory()?.firstCall()?.headers).toMatchObject({
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'Authorization': 'Basic dXNlcjpwYXNz',
'Priority': 'u=0',
'User-Agent': 'node',
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'Authorization': 'Basic dXNlcjpwYXNz',
'Priority': 'u=0',
'User-Agent': 'node',
},
});
});
test('textPost ', async () => {
const mockPool = mockAgent.get('https://some-post-url.test');
mockPool.intercept({ path: '/', method: 'POST' }).reply(200, 'some text');
fetchMock.post('https://some-post-url.test', 'some text');
expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text');
});
test('head', async () => {
const mockPool = mockAgent.get('https://some-head-url.test');
mockPool.intercept({ path: '/', method: 'HEAD' }).reply(200, 'some text', { headers: { 'X-Fake-Response': 'foo' } });
fetchMock.head('https://some-head-url.test', { status: 200, headers: { 'X-Fake-Response': 'foo' } });
expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
});
test('converts 404 to custom NotFoundError', async () => {
const mockPool = mockAgent.get('https://some-404-url.test');
mockPool.intercept({ path: '/' }).reply(404);
fetchMock.get('https://some-404-url.test', 404);
await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
});
test('converts Cloudflare challenge block to custom BlockedError', async () => {
const mockPool = mockAgent.get('https://some-cloudflare-url.test');
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
try {
await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'));
@ -72,30 +74,21 @@ describe('fetch', () => {
test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => {
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x';
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-success-url.test');
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
const mockPool2 = mockAgent.get('http://localhost:8191');
mockPool2.intercept({ path: '/v1/x', method: 'POST' }).reply(
200,
fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
fetchMock.post(
'http://localhost:8191/v1/x',
{
status: 'ok',
solution: {
response: 'some response',
cookies: [
{
name: 'cf_clearance',
value: 'some_value',
expires: Date.now() / 1000 + 12345,
domain: 'some-cloudflare-flaresolverr-success-url.test',
},
{
name: 'irrelevant',
value: 'some_other_value',
expires: Date.now() / 1000 + 12345,
domain: 'some-other.domain',
},
],
userAgent: 'user agent that works',
body: {
status: 'ok',
solution: {
response: 'some response',
cookies: [
{ name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' },
{ name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' },
],
userAgent: 'user agent that works',
},
},
},
);
@ -104,11 +97,12 @@ describe('fetch', () => {
expect(response).toBe('some response');
// The next request has to use the cookie and user agent we got
mockPool.intercept({ path: '/' }).reply(200, 'some response');
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
expect(mockAgent.getCallHistory()?.calls()[2]?.headers).toMatchObject({
'Cookie': 'cf_clearance=some_value',
'User-Agent': 'user agent that works',
expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({
headers: {
'Cookie': 'cf_clearance=some_value',
'User-Agent': 'user agent that works',
},
});
delete process.env['FLARESOLVERR_ENDPOINT'];
@ -116,10 +110,8 @@ describe('fetch', () => {
test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => {
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y';
const mockPool = mockAgent.get('https://some-cloudflare-flaresolverr-fail-url.test');
mockPool.intercept({ path: '/' }).reply(403, undefined, { headers: { 'cf-mitigated': 'challenge' } });
const mockPool2 = mockAgent.get('http://localhost:8191');
mockPool2.intercept({ path: '/v1/y', method: 'POST' }).reply(200, { status: 'nok' });
fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } });
try {
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/'));
@ -133,8 +125,7 @@ describe('fetch', () => {
});
test('converts generic forbidden to custom BlockedError', async () => {
const mockPool = mockAgent.get('https://some-forbidden-url.test');
mockPool.intercept({ path: '/' }).reply(403);
fetchMock.get('https://some-forbidden-url.test/', { status: 403 });
try {
await fetcher.text(ctx, new URL('https://some-forbidden-url.test/'));
@ -146,8 +137,7 @@ describe('fetch', () => {
});
test('converts cloudflare 451 to custom BlockedError', async () => {
const mockPool = mockAgent.get('https://some-cloudflare-forbidden-url.test');
mockPool.intercept({ path: '/' }).reply(451);
fetchMock.get('https://some-cloudflare-forbidden-url.test/', { status: 451 });
try {
await fetcher.text(ctx, new URL('https://some-cloudflare-forbidden-url.test/'));
@ -159,8 +149,7 @@ describe('fetch', () => {
});
test('converts MediaFlow Proxy forbidden to custom BlockedError', async () => {
const mockPool = mockAgent.get('https://media-flow-proxy-forbidden-url.test');
mockPool.intercept({ path: '/' }).reply(403);
fetchMock.get('https://media-flow-proxy-forbidden-url.test/', { status: 403 });
try {
await fetcher.text(createTestContext({ mediaFlowProxyUrl: 'https://media-flow-proxy-forbidden-url.test/' }), new URL('https://media-flow-proxy-forbidden-url.test/'));
@ -172,8 +161,7 @@ describe('fetch', () => {
});
test('converts 429 to custom TooManyRequestsError', async () => {
const mockPool = mockAgent.get('https://some-rate-limited-url.test');
mockPool.intercept({ path: '/' }).reply(429);
fetchMock.get('https://some-rate-limited-url.test/', { status: 429 });
try {
await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/'));
@ -185,9 +173,7 @@ describe('fetch', () => {
});
test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => {
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test');
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '60' } });
mockPool.intercept({ path: '/' }).reply(200);
fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '60' } });
try {
await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
@ -206,17 +192,16 @@ describe('fetch', () => {
});
test('retries rate limits with short ttl', async () => {
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-short-url.test');
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '1' } });
mockPool.intercept({ path: '/' }).reply(200);
fetchMock
.getOnce('https://some-rate-limited-retry-after-short-url.test/', { status: 429, headers: { 'retry-after': '1' } })
.getOnce('https://some-rate-limited-retry-after-short-url.test/', 'success');
const responseText = await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-short-url.test/'));
expect(responseText).toBe('');
expect(responseText).toBe('success');
});
test('passes through other error as HttpError after retrying 3 times', async () => {
const mockPool = mockAgent.get('https://some-error-url.test');
mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } }).times(4);
fetchMock.get('https://some-error-url.test', { status: 500, headers: { 'x-foo': 'bar' } });
try {
await fetcher.text(ctx, new URL('https://some-error-url.test/'));
@ -228,22 +213,19 @@ describe('fetch', () => {
});
test('passes through exceptions', async () => {
const mockPool = mockAgent.get('https://some-error-url.test');
mockPool.intercept({ path: '/' }).replyWithError((new TypeError('Failed to fetch')));
fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') });
await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError);
});
test('times out, retries and times out again', async () => {
const mockPool = mockAgent.get('https://some-timeout-url.test');
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 });
await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError);
});
test('queues requests and throws if queue is full', async () => {
const mockPool = mockAgent.get('https://some-full-queue-url.test');
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 });
const allPromises = Promise.all([
fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
@ -257,8 +239,7 @@ describe('fetch', () => {
});
test('throws if too many recent requests timed-out', async () => {
const mockPool = mockAgent.get('https://too-many-recent-timeouts-url.test');
mockPool.intercept({ path: '/' }).reply(200).delay(20).persist();
fetchMock.get('https://too-many-recent-timeouts-url.test/', 200, { delay: 20 });
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);
await expect(fetcher.text(ctx, new URL('https://too-many-recent-timeouts-url.test/'), { timeout: 10, timeoutsCountThrow: 2 })).rejects.toBeInstanceOf(TimeoutError);

10
src/utils/Fetcher.ts
View file

@ -3,7 +3,6 @@ import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
import CachePolicy from 'http-cache-semantics';
import { Cookie, CookieJar } from 'tough-cookie';
import { fetch, Headers, RequestInit, Response } from 'undici';
import winston from 'winston';
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
import { BlockedReason, Context } from '../types';
@ -274,17 +273,20 @@ export class Fetcher {
throw new TooManyTimeoutsError();
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), init?.timeout ?? this.DEFAULT_TIMEOUT);
let response;
try {
const finalUrl = new URL(url.href);
finalUrl.username = '';
finalUrl.password = '';
const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT) };
const finalInit = { ...init, keepalive: true, signal: controller.signal };
response = await fetch(finalUrl, finalInit);
} catch (error) {
if (error instanceof DOMException && error.name === 'TimeoutError') {
if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) {
if (tryCount < 1) {
this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of timeout`, ctx);
await this.sleep(333);
@ -297,6 +299,8 @@ export class Fetcher {
}
throw error;
} finally {
clearTimeout(timer);
}
await this.decreaseTimeoutsCount(url);

1
src/utils/FetcherMock.ts
View file

@ -3,7 +3,6 @@ import crypto from 'crypto';
import fs from 'node:fs';
import CachePolicy from 'http-cache-semantics';
import slugify from 'slugify';
import { RequestInit } from 'undici';
import winston from 'winston';
import { Context } from '../types';
import { envGet } from './env';