From 799ca975ac9873bbeff6c6d73b58eb04176b8901 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Thu, 29 May 2025 19:17:21 +0000 Subject: [PATCH] feat: make external URL exclusion configurable --- src/extractor/DoodStream.ts | 2 +- src/extractor/Dropload.ts | 2 +- src/extractor/ExternalUrl.ts | 2 +- src/extractor/ExtractorRegistry.test.ts | 12 ++++++++++++ src/extractor/ExtractorRegistry.ts | 8 ++++++-- src/extractor/SuperVideo.ts | 2 +- .../__snapshots__/ExtractorRegistry.test.ts.snap | 13 +++++++++++++ src/extractor/types.ts | 2 +- src/types.ts | 2 +- src/utils/manifest.test.ts | 8 ++------ src/utils/manifest.ts | 6 ++++++ 11 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap diff --git a/src/extractor/DoodStream.ts b/src/extractor/DoodStream.ts index 193f8d7..16b9f85 100644 --- a/src/extractor/DoodStream.ts +++ b/src/extractor/DoodStream.ts @@ -18,7 +18,7 @@ export class DoodStream implements Extractor { this.fetcher = fetcher; } - readonly supports = (url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/); + readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/); readonly extract = async (ctx: Context, url: URL, meta: Meta) => { const videoId = url.pathname.split('/').slice(-1)[0] as string; diff --git a/src/extractor/Dropload.ts b/src/extractor/Dropload.ts index eca1387..7e8c741 100644 --- a/src/extractor/Dropload.ts +++ b/src/extractor/Dropload.ts @@ -18,7 +18,7 @@ export class Dropload implements Extractor { this.fetcher = fetcher; } - readonly supports = (url: URL): boolean => null !== url.host.match(/dropload/); + readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/); readonly extract = async (ctx: Context, url: URL, meta: Meta) => { url.pathname = url.pathname.replace('/e/', '').replace('/embed-', '/'); diff --git a/src/extractor/ExternalUrl.ts b/src/extractor/ExternalUrl.ts index 76125c6..a4f0b05 100644 --- a/src/extractor/ExternalUrl.ts +++ b/src/extractor/ExternalUrl.ts @@ -15,7 +15,7 @@ export class ExternalUrl implements Extractor { this.fetcher = fetcher; } - readonly supports = (url: URL): boolean => null !== url.host.match(/.*/); + readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/); readonly extract = async (ctx: Context, url: URL, meta: Meta) => { // We only want to make sure that the URL is accessible diff --git a/src/extractor/ExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts index 5e39223..e1c3818 100644 --- a/src/extractor/ExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -18,6 +18,18 @@ describe('ExtractorRegistry', () => { expect(urlResult).toBeUndefined(); }); + test('return external URLs by default', async () => { + const urlResult = await extractorRegistry.handle(ctx, new URL('https://mixdrop.ag/e/3nzwveprim63or6'), { countryCode: 'de' }); + + expect(urlResult).toMatchSnapshot(); + }); + + test('does not return external URLs if disabled by config', async () => { + const urlResult = await extractorRegistry.handle({ ...ctx, config: { ...ctx.config, excludeExternalUrls: 'on' } }, new URL('https://mixdrop.ag/e/l7v73zqrfdj19z'), { countryCode: 'de' }); + + expect(urlResult).toBeUndefined(); + }); + test('returns from memory cache if possible', async () => { const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), { countryCode: 'de' }); const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), { countryCode: 'de' }); diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 91853d3..ab3bc13 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -31,7 +31,11 @@ export class ExtractorRegistry { return urlResult; } - const extractor = this.extractors.find(extractor => extractor.supports(url)) as Extractor; + const extractor = this.extractors.find(extractor => extractor.supports(ctx, url)); + if (!extractor) { + return undefined; + } + this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx); try { @@ -44,7 +48,7 @@ export class ExtractorRegistry { } /* istanbul ignore next */ - if (error instanceof CloudflareChallengeError) { + if (error instanceof CloudflareChallengeError && !('excludeExternalUrls' in ctx.config)) { this.logger.warn(`${extractor.id}: Request was blocked by Cloudflare challenge.`, ctx); return { diff --git a/src/extractor/SuperVideo.ts b/src/extractor/SuperVideo.ts index 35052e1..d398ce0 100644 --- a/src/extractor/SuperVideo.ts +++ b/src/extractor/SuperVideo.ts @@ -17,7 +17,7 @@ export class SuperVideo implements Extractor { this.fetcher = fetcher; } - readonly supports = (url: URL): boolean => null !== url.host.match(/supervideo/); + readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/); readonly extract = async (ctx: Context, url: URL, meta: Meta) => { url.pathname = url.pathname.replace('/e/', '').replace('/embed-', '/'); diff --git a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap new file mode 100644 index 0000000..3e656f6 --- /dev/null +++ b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap @@ -0,0 +1,13 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ExtractorRegistry return external URLs by default 1`] = ` +{ + "isExternal": true, + "label": "mixdrop.ag", + "meta": { + "countryCode": "de", + }, + "sourceId": "external_de", + "url": "https://mixdrop.ag/e/3nzwveprim63or6", +} +`; diff --git a/src/extractor/types.ts b/src/extractor/types.ts index fdb7013..ba64921 100644 --- a/src/extractor/types.ts +++ b/src/extractor/types.ts @@ -7,7 +7,7 @@ export interface Extractor { readonly ttl: number; - readonly supports: (url: URL) => boolean; + readonly supports: (ctx: Context, url: URL) => boolean; readonly extract: (ctx: Context, url: URL, meta: Meta) => Promise; } diff --git a/src/types.ts b/src/types.ts index c75386a..1dd3683 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,7 +9,7 @@ export interface Context { export type ManifestWithConfig = Manifest & { config: ManifestConfig[] }; -export type Config = Record; +export type Config = Partial>; export type CountryCode = 'de' | 'en' | 'es' | 'fr' | 'it' | 'mx'; diff --git a/src/utils/manifest.test.ts b/src/utils/manifest.test.ts index 3cf316d..afe2b96 100644 --- a/src/utils/manifest.test.ts +++ b/src/utils/manifest.test.ts @@ -10,12 +10,6 @@ const logger = winston.createLogger({ transports: [new winston.transports.Consol const fetcher = new Fetcher(); describe('buildManifest', () => { - test('has an empty config without handlers', () => { - const manifest = buildManifest([], {}); - - expect(manifest.config).toStrictEqual([]); - }); - test('has unchecked handler without a config', () => { const extractorRegistry = new ExtractorRegistry(logger, fetcher); const handlers = [ @@ -30,6 +24,7 @@ describe('buildManifest', () => { { key: 'de', type: 'checkbox', title: '🇩🇪 (KinoKiste, MeineCloud)' }, { key: 'es', type: 'checkbox', title: '🇪🇸 (VerHdLink)' }, { key: 'mx', type: 'checkbox', title: '🇲🇽 (VerHdLink)' }, + { key: 'excludeExternalUrls', type: 'checkbox', title: 'Exclude external URLs from results' }, ]); }); @@ -46,6 +41,7 @@ describe('buildManifest', () => { { key: 'de', type: 'checkbox', title: '🇩🇪 (KinoKiste, MeineCloud)', default: 'checked' }, { key: 'es', type: 'checkbox', title: '🇪🇸 (VerHdLink)' }, { key: 'mx', type: 'checkbox', title: '🇲🇽 (VerHdLink)' }, + { key: 'excludeExternalUrls', type: 'checkbox', title: 'Exclude external URLs from results' }, ]); }); }); diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index 7702d2d..24d4198 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -47,5 +47,11 @@ export const buildManifest = (handlers: Handler[], config: Config): ManifestWith }); } + manifest.config.push({ + key: 'excludeExternalUrls', + type: 'checkbox', + title: 'Exclude external URLs from results', + }); + return manifest; };