chore(extractor): use all VidSrc TLDs to handle rate-limit better

src/extractor/VidSrc.test.ts

@@ -1,12 +1,13 @@
+import { MockAgent, setGlobalDispatcher } from 'undici';
 import winston from 'winston';
 import { createTestContext } from '../test';
 import { CountryCode } from '../types';
-import { FetcherMock } from '../utils';
+import { Fetcher, FetcherMock } from '../utils';
 import { ExtractorRegistry } from './ExtractorRegistry';
 import { VidSrc } from './VidSrc';
 
 const logger = winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] });
-const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherMock(`${__dirname}/__fixtures__/VidSrc`))]);
+const extractorRegistry = new ExtractorRegistry(logger, [new VidSrc(new FetcherMock(`${__dirname}/__fixtures__/VidSrc`), ['xyz'])]);
 
 const ctx = createTestContext();
 
@@ -20,6 +21,22 @@ describe('VidSrc', () => {
   });
 
   test('not found', async () => {
-    expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt35628853'), CountryCode.en, 'Black Mirror 4x2')).toMatchSnapshot();
+    expect(await extractorRegistry.handle(ctx, new URL('https://vidsrc.xyz/embed/movie/tt35628853'), CountryCode.en, 'Titan: The OceanGate Disaster')).toMatchSnapshot();
+  });
+
+  test('rate limit issues are retried and fail if no tlds are left', async () => {
+    const mockAgent = new MockAgent({ enableCallHistory: true });
+    mockAgent.disableNetConnect();
+    setGlobalDispatcher(mockAgent);
+
+    mockAgent.get('https://vidsrc.xyz')
+      .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
+    mockAgent.get('https://vidsrc.net')
+      .intercept({ path: '/embed/movie/tt33043892/1/1' }).reply(429);
+
+    const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
+    const vidSrc = new VidSrc(fetcher, ['net', 'xyz']);
+
+    expect(await vidSrc.extract(ctx, new URL('https://vidsrc.xyz/embed/movie/tt33043892/1/1'), CountryCode.en, 'Dexter: Resurrection 1x1')).toMatchSnapshot();
   });
 });

src/extractor/VidSrc.ts

@@ -1,7 +1,7 @@
 import * as cheerio from 'cheerio';
 import slugify from 'slugify';
-import { NotFoundError } from '../error';
-import { Context, CountryCode, Format, UrlResult } from '../types';
+import { NotFoundError, TooManyRequestsError } from '../error';
+import { Context, CountryCode, Format, NonEmptyArray, UrlResult } from '../types';
 import { Fetcher, guessHeightFromPlaylist } from '../utils';
 import { Extractor } from './Extractor';
 
@@ -11,11 +11,13 @@ export class VidSrc extends Extractor {
   public readonly label = 'VidSrc';
 
   private readonly fetcher: Fetcher;
+  private readonly tlds: NonEmptyArray<string>;
 
-  public constructor(fetcher: Fetcher) {
+  public constructor(fetcher: Fetcher, tlds: NonEmptyArray<string>) {
     super();
 
     this.fetcher = fetcher;
+    this.tlds = tlds;
   }
 
   public supports(_ctx: Context, url: URL): boolean {
@@ -23,7 +25,28 @@ export class VidSrc extends Extractor {
   }
 
   protected async extractInternal(ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> {
-    const html = await this.fetcher.text(ctx, url);
+    return this.extractUsingRandomTld(ctx, url, countryCode, [...this.tlds]);
+  };
+
+  private async extractUsingRandomTld(ctx: Context, url: URL, countryCode: CountryCode, tlds: string[]): Promise<UrlResult[]> {
+    const tldIndex = Math.floor(Math.random() * tlds.length);
+    const [tld] = tlds.splice(tldIndex, 1) as [string];
+
+    const newUrl = new URL(url);
+    const hostnameParts = newUrl.hostname.split('.');
+    hostnameParts[hostnameParts.length - 1] = tld;
+    newUrl.hostname = hostnameParts.join('.');
+
+    let html: string;
+    try {
+      html = await this.fetcher.text(ctx, newUrl);
+    } catch (error) {
+      if (error instanceof TooManyRequestsError && tlds.length) {
+        return this.extractUsingRandomTld(ctx, url, countryCode, tlds);
+      }
+
+      throw error;
+    }
 
     const $ = cheerio.load(html);
 
@@ -61,5 +84,5 @@ export class VidSrc extends Extractor {
           };
         }),
     );
-  };
+  }
 }

src/extractor/__snapshots__/VidSrc.test.ts.snap

@@ -39,3 +39,22 @@ exports[`VidSrc Full Metal Jacket 1`] = `
 `;
 
 exports[`VidSrc not found 1`] = `[]`;
+
+exports[`VidSrc rate limit issues are retried and fail if no tlds are left 1`] = `
+[
+  {
+    "error": [Error],
+    "format": "unknown",
+    "isExternal": true,
+    "label": "VidSrc",
+    "meta": {
+      "countryCodes": [
+        "en",
+      ],
+      "title": "Dexter: Resurrection 1x1",
+    },
+    "sourceId": "vidsrc",
+    "url": "https://vidsrc.xyz/embed/movie/tt33043892/1/1",
+  },
+]
+`;

src/extractor/index.ts

@@ -30,7 +30,7 @@ export const createExtractors = (fetcher: Fetcher): Extractor[] => [
   new Streamtape(fetcher),
   new SuperVideo(fetcher),
   new Uqload(fetcher),
-  new VidSrc(fetcher),
+  new VidSrc(fetcher, ['in', 'pm', 'net', 'xyz', 'io', 'vc']), // https://vidsrc.domains/
   new VixSrc(fetcher),
   new ExternalUrl(fetcher), // fallback extractor which must come last
 ];

src/types.ts

@@ -1,5 +1,7 @@
 import { Manifest, ManifestConfig } from 'stremio-addon-sdk';
 
+export type NonEmptyArray<T> = [T, ...T[]];
+
 export interface Context {
   hostUrl: URL;
   id: string;