refactor(extractor): introduce parent class for simplifications

src/extractor/DoodStream.ts

@@ -1,32 +1,32 @@
 import randomstring from 'randomstring';
 import * as cheerio from 'cheerio';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher, guessFromTitle } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 import { NotFoundError } from '../error';
 
-export class DoodStream implements Extractor {
+export class DoodStream extends Extractor {
   public readonly id = 'doodstream';
 
   public readonly label = 'DoodStream';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go|dooodster|dooood/);
 
-  public readonly normalize = (url: URL): URL => {
+  public override readonly normalize = (url: URL): URL => {
     const videoId = url.pathname.split('/').slice(-1)[0] as string;
 
     return new URL(`http://dood.to/e/${videoId}`);
   };
 
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const html = await this.fetcher.text(ctx, new URL(url));
 
     const passMd5Match = html.match(/\/pass_md5\/[\w-]+\/([\w-]+)/);

src/extractor/Dropload.ts

@@ -1,28 +1,28 @@
 import bytes from 'bytes';
 import * as cheerio from 'cheerio';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { extractUrlFromPacked, Fetcher } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 import { NotFoundError } from '../error';
 
-export class Dropload implements Extractor {
+export class Dropload extends Extractor {
   public readonly id = 'dropload';
 
   public readonly label = 'Dropload';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/);
 
-  public readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
+  public override readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
 
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const html = await this.fetcher.text(ctx, url);
 
     if (html.includes('File Not Found')) {

src/extractor/ExternalUrl.ts

@@ -1,25 +1,25 @@
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 
-export class ExternalUrl implements Extractor {
+export class ExternalUrl extends Extractor {
   public readonly id = 'external';
 
   public readonly label = 'External';
 
-  public readonly ttl = 3600000; // 1h
+  public override readonly ttl = 3600000; // 1h
 
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/);
 
-  public readonly normalize = (url: URL): URL => url;
-
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise<UrlResult[]> => {
     try {
       // Make sure the URL is accessible, but avoid causing noise and delays doing this
       await this.fetcher.head(ctx, url, { noFlareSolverr: true, timeout: 1000 });

src/extractor/Extractor.ts

@@ -0,0 +1,40 @@
+import { Context, CountryCode, UrlResult } from '../types';
+import { NotFoundError } from '../error';
+
+export abstract class Extractor {
+  public abstract readonly id: string;
+
+  public abstract readonly label: string;
+
+  public readonly ttl: number = 900000; // 15m
+
+  public abstract readonly supports: (ctx: Context, url: URL) => boolean;
+
+  public readonly normalize = (url: URL): URL => url;
+
+  protected abstract readonly extractInternal: (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined) => Promise<UrlResult[]>;
+
+  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined): Promise<UrlResult[]> => {
+    try {
+      return await this.extractInternal(ctx, url, countryCode, title);
+    } catch (error) {
+      if (error instanceof NotFoundError) {
+        return [];
+      }
+
+      return [
+        {
+          url,
+          isExternal: true,
+          error,
+          label: url.host,
+          sourceId: `${this.id}`,
+          meta: {
+            countryCode,
+            ...(title && { title }),
+          },
+        },
+      ];
+    }
+  };
+}

src/extractor/ExtractorRegistry.test.ts

@@ -48,10 +48,12 @@ describe('ExtractorRegistry', () => {
   });
 
   test('returns external url for error', async () => {
-    const urlResults1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de);
-    expect(urlResults1).toMatchSnapshot();
+    const urlResults = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de);
+    expect(urlResults).toMatchSnapshot();
+  });
 
-    const urlResults2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de, 'a title!');
-    expect(urlResults2).toMatchSnapshot();
+  test('returns external url for error with title', async () => {
+    const urlResults = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked-2.html'), CountryCode.de, 'a title!');
+    expect(urlResults).toMatchSnapshot();
   });
 });

src/extractor/ExtractorRegistry.ts

@@ -1,8 +1,7 @@
 import TTLCache from '@isaacs/ttlcache';
 import winston from 'winston';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Context, CountryCode, UrlResult } from '../types';
-import { NotFoundError } from '../error';
 
 export class ExtractorRegistry {
   private readonly logger: winston.Logger;
@@ -30,30 +29,7 @@ export class ExtractorRegistry {
 
     this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx);
 
-    try {
-      urlResults = await extractor.extract(ctx, normalizedUrl, countryCode, title);
-    } catch (error) {
-      if (error instanceof NotFoundError) {
-        this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl });
-
-        return [];
-      }
-
-      return [
-        {
-          url,
-          isExternal: true,
-          error,
-          label: url.host,
-          sourceId: `${extractor.id}`,
-          meta: {
-            countryCode,
-            ...(title && { title }),
-          },
-        },
-      ];
-    }
-
+    urlResults = await extractor.extract(ctx, normalizedUrl, countryCode, title);
     this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl });
 
     return urlResults;

src/extractor/Fsst.ts

@@ -1,26 +1,24 @@
 import * as cheerio from 'cheerio';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 
-export class Fsst implements Extractor {
+export class Fsst extends Extractor {
   public readonly id = 'fsst';
 
   public readonly label = 'Fsst';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/fsst/);
 
-  public readonly normalize = (url: URL): URL => url;
-
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const html = await this.fetcher.text(ctx, url);
 
     const $ = cheerio.load(html);

src/extractor/KinoGer.ts

@@ -1,27 +1,27 @@
 import crypto from 'crypto';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher, guessFromTitle } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 
 /** @see https://github.com/Gujal00/ResolveURL/blob/master/script.module.resolveurl/lib/resolveurl/plugins/kinoger.py */
-export class KinoGer implements Extractor {
+export class KinoGer extends Extractor {
   public readonly id = 'kinoger';
 
   public readonly label = 'KinoGer';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/kinoger\.re|shiid4u\.upn\.one|moflix\.upns\.xyz|player\.upn\.one|wasuytm\.store|ultrastream\.online/);
 
-  public readonly normalize = (url: URL): URL => new URL(`${url.origin}/api/v1/video?id=${url.hash.slice(1)}`);
+  public override readonly normalize = (url: URL): URL => new URL(`${url.origin}/api/v1/video?id=${url.hash.slice(1)}`);
 
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const hexData = await this.fetcher.text(ctx, url, { headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36' } });
 
     const encrypted = Buffer.from(hexData, 'hex');

src/extractor/Soaper.ts

@@ -1,4 +1,4 @@
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher, guessFromPlaylist } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 
@@ -7,24 +7,22 @@ interface SoaperInfoResponsePartial {
   val_bak: string;
 }
 
-export class Soaper implements Extractor {
+export class Soaper extends Extractor {
   public readonly id = 'soaper';
 
   public readonly label = 'Soaper';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/soaper/) && null !== url.pathname.match(/^\/(episode|movie)_/);
 
-  public readonly normalize = (url: URL): URL => url;
-
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise<UrlResult[]> => {
     const movieOrEpisodeId = (url.pathname.match(/\/\w+_(\w+)/) as string[])[1] as string;
 
     const form = new URLSearchParams();

src/extractor/SuperVideo.ts

@@ -1,32 +1,32 @@
 import bytes from 'bytes';
 import * as cheerio from 'cheerio';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { extractUrlFromPacked, Fetcher } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 import { NotFoundError } from '../error';
 
-export class SuperVideo implements Extractor {
+export class SuperVideo extends Extractor {
   public readonly id = 'supervideo';
 
   public readonly label = 'SuperVideo';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/);
 
-  public readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
+  public override readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
 
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const html = await this.fetcher.text(ctx, url);
 
     if (html.includes('This video can be watched as embed only')) {
-      return await this.extract(ctx, new URL(`/e${url.pathname}`, url.origin), countryCode);
+      return await this.extractInternal(ctx, new URL(`/e${url.pathname}`, url.origin), countryCode);
     }
 
     if (/'The file was deleted|The file expired/.test(html)) {

src/extractor/VidSrc.ts

@@ -1,28 +1,26 @@
 import * as cheerio from 'cheerio';
 import slugify from 'slugify';
-import { Extractor } from './types';
+import { Extractor } from './Extractor';
 import { Fetcher, guessFromPlaylist } from '../utils';
 import { Context, CountryCode, UrlResult } from '../types';
 import { NotFoundError } from '../error';
 
-export class VidSrc implements Extractor {
+export class VidSrc extends Extractor {
   public readonly id = 'vidsrc';
 
   public readonly label = 'VidSrc';
 
-  public readonly ttl = 900000; // 15m
-
   private readonly fetcher: Fetcher;
 
   public constructor(fetcher: Fetcher) {
+    super();
+
     this.fetcher = fetcher;
   }
 
   public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/vidsrc/);
 
-  public readonly normalize = (url: URL): URL => url;
-
-  public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
+  protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise<UrlResult[]> => {
     const html = await this.fetcher.text(ctx, url);
 
     const $ = cheerio.load(html);

src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap

@@ -33,7 +33,7 @@ exports[`ExtractorRegistry returns external url for error 1`] = `
 ]
 `;
 
-exports[`ExtractorRegistry returns external url for error 2`] = `
+exports[`ExtractorRegistry returns external url for error with title 1`] = `
 [
   {
     "error": [Error: Fetcher error: 403: Forbidden
@@ -45,7 +45,7 @@ exports[`ExtractorRegistry returns external url for error 2`] = `
       "title": "a title!",
     },
     "sourceId": "dropload",
-    "url": "https://dropload.io/mocked-blocked.html",
+    "url": "https://dropload.io/mocked-blocked-2.html",
   },
 ]
 `;

src/extractor/index.ts

@@ -1,16 +1,16 @@
-import { Extractor } from './types';
 import { DoodStream } from './DoodStream';
 import { Dropload } from './Dropload';
+import { ExternalUrl } from './ExternalUrl';
+import { Extractor } from './Extractor';
 import { Fsst } from './Fsst';
-import { SuperVideo } from './SuperVideo';
 import { KinoGer } from './KinoGer';
 import { Soaper } from './Soaper';
+import { SuperVideo } from './SuperVideo';
 import { VidSrc } from './VidSrc';
-import { ExternalUrl } from './ExternalUrl';
 import { Fetcher } from '../utils';
 
+export * from './Extractor';
 export * from './ExtractorRegistry';
-export * from './types';
 
 export const createExtractors = (fetcher: Fetcher): Extractor[] => [
   new DoodStream(fetcher),

src/extractor/types.ts

@@ -1,15 +0,0 @@
-import { Context, CountryCode, UrlResult } from '../types';
-
-export interface Extractor {
-  readonly id: string;
-
-  readonly label: string;
-
-  readonly ttl: number;
-
-  readonly supports: (ctx: Context, url: URL) => boolean;
-
-  readonly normalize: (url: URL) => URL;
-
-  readonly extract: (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined) => Promise<UrlResult[]>;
-}

src/utils/StreamResolver.test.ts

@@ -77,18 +77,16 @@ describe('resolve', () => {
       };
     }
 
-    class MockExtractor implements Extractor {
+    class MockExtractor extends Extractor {
       public readonly id = 'mockextractor';
 
       public readonly label = 'MockExtractor';
 
-      public readonly ttl = 1;
+      public override readonly ttl = 1;
 
       public readonly supports = (): boolean => true;
 
-      public readonly normalize = (url: URL): URL => url;
-
-      public readonly extract = async (): Promise<UrlResult[]> =>
+      protected readonly extractInternal = async (): Promise<UrlResult[]> =>
         [
           {
             url: new URL('https://example.com'),

src/utils/__fixtures__/Fetcher/https:dropload.iomocked-blocked-2.html.error

@@ -0,0 +1 @@
+Fetcher error: 403: Forbidden