feat: disable extractors via config

src/controller/ConfigureController.ts

@@ -1,4 +1,5 @@
 import { Request, Response, Router } from 'express';
+import { Extractor } from '../extractor';
 import { landingTemplate } from '../landingTemplate';
 import { Source } from '../source';
 import { Config } from '../types';
@@ -8,11 +9,13 @@ export class ConfigureController {
   public readonly router: Router;
 
   private readonly sources: Source[];
+  private readonly extractors: Extractor[];
 
-  public constructor(sources: Source[]) {
+  public constructor(sources: Source[], extractors: Extractor[]) {
     this.router = Router();
 
     this.sources = sources;
+    this.extractors = extractors;
 
     this.router.get('/configure', this.getConfigure.bind(this));
     this.router.get('/:config/configure', this.getConfigure.bind(this));
@@ -26,7 +29,7 @@ export class ConfigureController {
       config.mediaFlowProxyUrl = `${req.protocol}://${req.host.replace('webstreamr', 'mediaflow-proxy')}`;
     }
 
-    const manifest = buildManifest(this.sources, config);
+    const manifest = buildManifest(this.sources, this.extractors, config);
 
     res.setHeader('content-type', 'text/html');
     res.send(landingTemplate(manifest));

src/controller/ManifestController.ts

@@ -1,4 +1,5 @@
 import { Request, Response, Router } from 'express';
+import { Extractor } from '../extractor';
 import { Source } from '../source';
 import { Config } from '../types';
 import { buildManifest, getDefaultConfig } from '../utils';
@@ -7,11 +8,13 @@ export class ManifestController {
   public readonly router: Router;
 
   private readonly sources: Source[];
+  private readonly extractors: Extractor[];
 
-  public constructor(sources: Source[]) {
+  public constructor(sources: Source[], extractors: Extractor[]) {
     this.router = Router();
 
     this.sources = sources;
+    this.extractors = extractors;
 
     this.router.get('/manifest.json', this.getManifest.bind(this));
     this.router.get('/:config/manifest.json', this.getManifest.bind(this));
@@ -20,7 +23,7 @@ export class ManifestController {
   private getManifest(req: Request, res: Response) {
     const config: Config = JSON.parse(req.params['config'] || JSON.stringify(getDefaultConfig()));
 
-    const manifest = buildManifest(this.sources, config);
+    const manifest = buildManifest(this.sources, this.extractors, config);
 
     res.setHeader('Content-Type', 'application/json');
     res.send(manifest);

src/extractor/ExtractorRegistry.ts

@@ -1,6 +1,7 @@
 import TTLCache from '@isaacs/ttlcache';
 import winston from 'winston';
 import { Context, CountryCode, UrlResult } from '../types';
+import { isExtractorDisabled } from '../utils';
 import { Extractor } from './Extractor';
 
 export class ExtractorRegistry {
@@ -15,7 +16,7 @@ export class ExtractorRegistry {
   }
 
   public async handle(ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined): Promise<UrlResult[]> {
-    const extractor = this.extractors.find(extractor => extractor.supports(ctx, url));
+    const extractor = this.extractors.find(extractor => !isExtractorDisabled(ctx.config, extractor) && extractor.supports(ctx, url));
     if (!extractor) {
       return [];
     }

src/index.ts

@@ -50,8 +50,8 @@ addon.use((_req: Request, res: Response, next: NextFunction) => {
   next();
 });
 
-addon.use('/', (new ConfigureController(sources)).router);
-addon.use('/', (new ManifestController(sources)).router);
+addon.use('/', (new ConfigureController(sources, extractors)).router);
+addon.use('/', (new ManifestController(sources, extractors)).router);
 
 const extractorRegistry = new ExtractorRegistry(logger, extractors);
 const streamResolver = new StreamResolver(logger, extractorRegistry);

src/types.ts

@@ -15,7 +15,7 @@ export type CustomManifest = Manifest & {
   };
 };
 
-export type Config = Partial<Record<CountryCode | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword', string>>;
+export type Config = Partial<Record<CountryCode | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword', string> & Record<`disableExtractor_${string}`, string>>;
 
 export enum CountryCode {
   multi = 'multi',

src/utils/__snapshots__/manifest.test.ts.snap

@@ -1,5 +1,38 @@
 // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
 
+exports[`buildManifest disable extractors 1`] = `
+[
+  {
+    "key": "includeExternalUrls",
+    "title": "Include external URLs in results",
+    "type": "checkbox",
+  },
+  {
+    "default": "",
+    "key": "mediaFlowProxyUrl",
+    "title": "MediaFlow Proxy URL",
+    "type": "text",
+  },
+  {
+    "default": "",
+    "key": "mediaFlowProxyPassword",
+    "title": "MediaFlow Proxy Password",
+    "type": "password",
+  },
+  {
+    "default": "checked",
+    "key": "disableExtractor_doodstream",
+    "title": "Disable extractor DoodStream",
+    "type": "checkbox",
+  },
+  {
+    "key": "disableExtractor_supervideo",
+    "title": "Disable extractor SuperVideo",
+    "type": "checkbox",
+  },
+]
+`;
+
 exports[`buildManifest has checked includeExternalUrls 1`] = `
 [
   {

src/utils/config.ts

@@ -1,3 +1,4 @@
+import { Extractor } from '../extractor';
 import { Config } from '../types';
 
 export const getDefaultConfig = (): Config => {
@@ -7,3 +8,7 @@ export const getDefaultConfig = (): Config => {
 export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config;
 
 export const hasMultiEnabled = (config: Config): boolean => 'multi' in config;
+
+export const disableExtractorConfigKey = (extractor: Extractor): string => `disableExtractor_${extractor.id}`;
+
+export const isExtractorDisabled = (config: Config, extractor: Extractor): boolean => disableExtractorConfigKey(extractor) in config;

src/utils/manifest.test.ts

@@ -1,3 +1,6 @@
+import { DoodStream } from '../extractor/DoodStream';
+import { ExternalUrl } from '../extractor/ExternalUrl';
+import { SuperVideo } from '../extractor/SuperVideo';
 import { MeineCloud } from '../source/MeineCloud';
 import { StreamKiste } from '../source/StreamKiste';
 import { VerHdLink } from '../source/VerHdLink';
@@ -16,7 +19,7 @@ describe('buildManifest', () => {
       new MeineCloud(fetcher),
     ];
 
-    const manifest = buildManifest(sources, {});
+    const manifest = buildManifest(sources, [], {});
 
     expect(manifest.config).toMatchSnapshot();
   });
@@ -27,19 +30,30 @@ describe('buildManifest', () => {
       new StreamKiste(fetcher),
       new MeineCloud(fetcher),
     ];
-    const manifest = buildManifest(sources, { de: 'on', includeExternalUrls: 'on' });
+    const manifest = buildManifest(sources, [], { de: 'on', includeExternalUrls: 'on' });
 
     expect(manifest.config).toMatchSnapshot();
   });
 
   test('includeExternalUrls is unchecked by default', () => {
-    const manifest = buildManifest([], {});
+    const manifest = buildManifest([], [], {});
 
     expect(manifest.config).toMatchSnapshot();
   });
 
   test('has checked includeExternalUrls', () => {
-    const manifest = buildManifest([], { includeExternalUrls: 'on' });
+    const manifest = buildManifest([], [], { includeExternalUrls: 'on' });
+
+    expect(manifest.config).toMatchSnapshot();
+  });
+
+  test('disable extractors', () => {
+    const extractors = [
+      new DoodStream(fetcher),
+      new SuperVideo(fetcher),
+      new ExternalUrl(fetcher),
+    ];
+    const manifest = buildManifest([], extractors, { disableExtractor_doodstream: 'on' });
 
     expect(manifest.config).toMatchSnapshot();
   });

src/utils/manifest.ts

@@ -1,11 +1,13 @@
+import { Extractor } from '../extractor';
 import { Source } from '../source';
 import { Config, CountryCode, CustomManifest } from '../types';
+import { disableExtractorConfigKey, isExtractorDisabled } from './config';
 import { envGetAppId, envGetAppName } from './env';
 import { flagFromCountryCode, languageFromCountryCode } from './language';
 
 const typedEntries = <T extends object>(obj: T): [keyof T, T[keyof T]][] => (Object.entries(obj) as [keyof T, T[keyof T]][]);
 
-export const buildManifest = (sources: Source[], config: Config): CustomManifest => {
+export const buildManifest = (sources: Source[], extractors: Extractor[], config: Config): CustomManifest => {
   const manifest: CustomManifest = {
     id: envGetAppId(),
     version: '0.39.2', // x-release-please-version
@@ -76,5 +78,18 @@ export const buildManifest = (sources: Source[], config: Config): CustomManifest
     default: config['mediaFlowProxyPassword'] ?? '',
   });
 
+  extractors.forEach((extractor) => {
+    if (extractor.id === 'external') {
+      return;
+    }
+
+    manifest.config.push({
+      key: disableExtractorConfigKey(extractor),
+      type: 'checkbox',
+      title: `Disable extractor ${extractor.label}`,
+      ...(isExtractorDisabled(config, extractor) && { default: 'checked' }),
+    });
+  });
+
   return manifest;
 };