From 6492e6be820e0e50b814aa3e661d10517184ac38 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Thu, 14 Aug 2025 07:35:03 +0000 Subject: [PATCH] chore(extractor): use all VidSrc TLDs to handle rate-limit better --- src/extractor/VidSrc.test.ts | 23 +++++++++++-- src/extractor/VidSrc.ts | 33 ++++++++++++++++--- .../__snapshots__/VidSrc.test.ts.snap | 19 +++++++++++ src/extractor/index.ts | 2 +- src/types.ts | 2 ++ 5 files changed, 70 insertions(+), 9 deletions(-) diff --git a/src/extractor/VidSrc.test.ts b/src/extractor/VidSrc.test.ts index 314bc0a..9dc912f 100644 --- a/src/extractor/VidSrc.test.ts +++ b/src/extractor/VidSrc.test.ts @@ -1,12 +1,13 @@ +import { MockAgent, setGlobalDispatcher } from 'undici'; import winston from 'winston'; import { createTestContext } from '../test'; import { CountryCode } from '../types'; -import { FetcherMock } from '../utils'; +import { Fetcher, FetcherMock } from '../utils'; import { ExtractorRegistry } from './ExtractorRegistry'; import { VidSrc } from './VidSrc'; const logger = winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }); -const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherMock(`${__dirname}/__fixtures__/VidSrc`))]); +const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherMock(`${__dirname}/__fixtures__/VidSrc`), ['xyz'])]); const ctx = createTestContext(); @@ -20,6 +21,22 @@ describe('VidSrc', () => { }); test('not found', async () => { - expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt35628853'), CountryCode.en, 'Black Mirror 4x2')).toMatchSnapshot(); + expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt35628853'), CountryCode.en, 'Titan: The OceanGate Disaster')).toMatchSnapshot(); + }); + + test('rate limit issues are retried and fail if no tlds are left', async () => { + const mockAgent = new MockAgent({ enableCallHistory: true }); + mockAgent.disableNetConnect(); + setGlobalDispatcher(mockAgent); + + mockAgent.get('https://vidsrc.xyz') + .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429); + mockAgent.get('https://vidsrc.net') + .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429); + + const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); + const vidSrc = new VidSrc(fetcher, ['net', 'xyz']); + + expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/1'), CountryCode.en, 'Dexter: Resurrection 1x1')).toMatchSnapshot(); }); }); diff --git a/src/extractor/VidSrc.ts b/src/extractor/VidSrc.ts index d310c14..9616d5e 100644 --- a/src/extractor/VidSrc.ts +++ b/src/extractor/VidSrc.ts @@ -1,7 +1,7 @@ import * as cheerio from 'cheerio'; import slugify from 'slugify'; -import { NotFoundError } from '../error'; -import { Context, CountryCode, Format, UrlResult } from '../types'; +import { NotFoundError, TooManyRequestsError } from '../error'; +import { Context, CountryCode, Format, NonEmptyArray, UrlResult } from '../types'; import { Fetcher, guessHeightFromPlaylist } from '../utils'; import { Extractor } from './Extractor'; @@ -11,11 +11,13 @@ export class VidSrc extends Extractor { public readonly label = 'VidSrc'; private readonly fetcher: Fetcher; + private readonly tlds: NonEmptyArray; - public constructor(fetcher: Fetcher) { + public constructor(fetcher: Fetcher, tlds: NonEmptyArray) { super(); this.fetcher = fetcher; + this.tlds = tlds; } public supports(_ctx: Context, url: URL): boolean { @@ -23,7 +25,28 @@ export class VidSrc extends Extractor { } protected async extractInternal(ctx: Context, url: URL, countryCode: CountryCode): Promise { - const html = await this.fetcher.text(ctx, url); + return this.extractUsingRandomTld(ctx, url, countryCode, [...this.tlds]); + }; + + private async extractUsingRandomTld(ctx: Context, url: URL, countryCode: CountryCode, tlds: string[]): Promise { + const tldIndex = Math.floor(Math.random() * tlds.length); + const [tld] = tlds.splice(tldIndex, 1) as [string]; + + const newUrl = new URL(url); + const hostnameParts = newUrl.hostname.split('.'); + hostnameParts[hostnameParts.length - 1] = tld; + newUrl.hostname = hostnameParts.join('.'); + + let html: string; + try { + html = await this.fetcher.text(ctx, newUrl); + } catch (error) { + if (error instanceof TooManyRequestsError && tlds.length) { + return this.extractUsingRandomTld(ctx, url, countryCode, tlds); + } + + throw error; + } const $ = cheerio.load(html); @@ -61,5 +84,5 @@ export class VidSrc extends Extractor { }; }), ); - }; + } } diff --git a/src/extractor/__snapshots__/VidSrc.test.ts.snap b/src/extractor/__snapshots__/VidSrc.test.ts.snap index e838c6d..c5ce4c4 100644 --- a/src/extractor/__snapshots__/VidSrc.test.ts.snap +++ b/src/extractor/__snapshots__/VidSrc.test.ts.snap @@ -39,3 +39,22 @@ exports[`VidSrc Full Metal Jacket 1`] = ` `; exports[`VidSrc not found 1`] = `[]`; + +exports[`VidSrc rate limit issues are retried and fail if no tlds are left 1`] = ` +[ + { + "error": [Error], + "format": "unknown", + "isExternal": true, + "label": "VidSrc", + "meta": { + "countryCodes": [ + "en", + ], + "title": "Dexter: Resurrection 1x1", + }, + "sourceId": "vidsrc", + "url": "https://vidsrc.xyz/embed/movie/tt33043892/1/1", + }, +] +`; diff --git a/src/extractor/index.ts b/src/extractor/index.ts index 1eb94ac..a10a406 100644 --- a/src/extractor/index.ts +++ b/src/extractor/index.ts @@ -30,7 +30,7 @@ export const createExtractors = (fetcher: Fetcher): Extractor[] => [ new Streamtape(fetcher), new SuperVideo(fetcher), new Uqload(fetcher), - new VidSrc(fetcher), + new VidSrc(fetcher, ['in', 'pm', 'net', 'xyz', 'io', 'vc']), // https://vidsrc.domains/ new VixSrc(fetcher), new ExternalUrl(fetcher), // fallback extractor which must come last ]; diff --git a/src/types.ts b/src/types.ts index 63ed775..968b95a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,7 @@ import { Manifest, ManifestConfig } from 'stremio-addon-sdk'; +export type NonEmptyArray = [T, ...T[]]; + export interface Context { hostUrl: URL; id: string;