feat: make external URL exclusion configurable

src/extractor/DoodStream.ts

@@ -18,7 +18,7 @@ export class DoodStream implements Extractor {
     this.fetcher = fetcher;
   }
 
-  readonly supports = (url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/);
+  readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/);
 
   readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
     const videoId = url.pathname.split('/').slice(-1)[0] as string;

src/extractor/Dropload.ts

@@ -18,7 +18,7 @@ export class Dropload implements Extractor {
     this.fetcher = fetcher;
   }
 
-  readonly supports = (url: URL): boolean => null !== url.host.match(/dropload/);
+  readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/);
 
   readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
     url.pathname = url.pathname.replace('/e/', '').replace('/embed-', '/');

src/extractor/ExternalUrl.ts

@@ -15,7 +15,7 @@ export class ExternalUrl implements Extractor {
     this.fetcher = fetcher;
   }
 
-  readonly supports = (url: URL): boolean => null !== url.host.match(/.*/);
+  readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/);
 
   readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
     // We only want to make sure that the URL is accessible

src/extractor/ExtractorRegistry.test.ts

@@ -18,6 +18,18 @@ describe('ExtractorRegistry', () => {
     expect(urlResult).toBeUndefined();
   });
 
+  test('return external URLs by default', async () => {
+    const urlResult = await extractorRegistry.handle(ctx, new URL('https://mixdrop.ag/e/3nzwveprim63or6'), { countryCode: 'de' });
+
+    expect(urlResult).toMatchSnapshot();
+  });
+
+  test('does not return external URLs if disabled by config', async () => {
+    const urlResult = await extractorRegistry.handle({ ...ctx, config: { ...ctx.config, excludeExternalUrls: 'on' } }, new URL('https://mixdrop.ag/e/l7v73zqrfdj19z'), { countryCode: 'de' });
+
+    expect(urlResult).toBeUndefined();
+  });
+
   test('returns from memory cache if possible', async () => {
     const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), { countryCode: 'de' });
     const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), { countryCode: 'de' });

src/extractor/ExtractorRegistry.ts

@@ -31,7 +31,11 @@ export class ExtractorRegistry {
       return urlResult;
     }
 
-    const extractor = this.extractors.find(extractor => extractor.supports(url)) as Extractor;
+    const extractor = this.extractors.find(extractor => extractor.supports(ctx, url));
+    if (!extractor) {
+      return undefined;
+    }
+
     this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx);
 
     try {
@@ -44,7 +48,7 @@ export class ExtractorRegistry {
       }
 
       /* istanbul ignore next */
-      if (error instanceof CloudflareChallengeError) {
+      if (error instanceof CloudflareChallengeError && !('excludeExternalUrls' in ctx.config)) {
         this.logger.warn(`${extractor.id}: Request was blocked by Cloudflare challenge.`, ctx);
 
         return {

src/extractor/SuperVideo.ts

@@ -17,7 +17,7 @@ export class SuperVideo implements Extractor {
     this.fetcher = fetcher;
   }
 
-  readonly supports = (url: URL): boolean => null !== url.host.match(/supervideo/);
+  readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/);
 
   readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
     url.pathname = url.pathname.replace('/e/', '').replace('/embed-', '/');

src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap

@@ -0,0 +1,13 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`ExtractorRegistry return external URLs by default 1`] = `
+{
+  "isExternal": true,
+  "label": "mixdrop.ag",
+  "meta": {
+    "countryCode": "de",
+  },
+  "sourceId": "external_de",
+  "url": "https://mixdrop.ag/e/3nzwveprim63or6",
+}
+`;

src/extractor/types.ts

@@ -7,7 +7,7 @@ export interface Extractor {
 
   readonly ttl: number;
 
-  readonly supports: (url: URL) => boolean;
+  readonly supports: (ctx: Context, url: URL) => boolean;
 
   readonly extract: (ctx: Context, url: URL, meta: Meta) => Promise<UrlResult>;
 }

src/types.ts

@@ -9,7 +9,7 @@ export interface Context {
 
 export type ManifestWithConfig = Manifest & { config: ManifestConfig[] };
 
-export type Config = Record<string, string>;
+export type Config = Partial<Record<CountryCode | 'excludeExternalUrls', string>>;
 
 export type CountryCode = 'de' | 'en' | 'es' | 'fr' | 'it' | 'mx';
 

src/utils/manifest.test.ts

@@ -10,12 +10,6 @@ const logger = winston.createLogger({ transports: [new winston.transports.Consol
 const fetcher = new Fetcher();
 
 describe('buildManifest', () => {
-  test('has an empty config without handlers', () => {
-    const manifest = buildManifest([], {});
-
-    expect(manifest.config).toStrictEqual([]);
-  });
-
   test('has unchecked handler without a config', () => {
     const extractorRegistry = new ExtractorRegistry(logger, fetcher);
     const handlers = [
@@ -30,6 +24,7 @@ describe('buildManifest', () => {
       { key: 'de', type: 'checkbox', title: '🇩🇪 (KinoKiste, MeineCloud)' },
       { key: 'es', type: 'checkbox', title: '🇪🇸 (VerHdLink)' },
       { key: 'mx', type: 'checkbox', title: '🇲🇽 (VerHdLink)' },
+      { key: 'excludeExternalUrls', type: 'checkbox', title: 'Exclude external URLs from results' },
     ]);
   });
 
@@ -46,6 +41,7 @@ describe('buildManifest', () => {
       { key: 'de', type: 'checkbox', title: '🇩🇪 (KinoKiste, MeineCloud)', default: 'checked' },
       { key: 'es', type: 'checkbox', title: '🇪🇸 (VerHdLink)' },
       { key: 'mx', type: 'checkbox', title: '🇲🇽 (VerHdLink)' },
+      { key: 'excludeExternalUrls', type: 'checkbox', title: 'Exclude external URLs from results' },
     ]);
   });
 });

src/utils/manifest.ts

@@ -47,5 +47,11 @@ export const buildManifest = (handlers: Handler[], config: Config): ManifestWith
     });
   }
 
+  manifest.config.push({
+    key: 'excludeExternalUrls',
+    type: 'checkbox',
+    title: 'Exclude external URLs from results',
+  });
+
   return manifest;
 };