From 63a79f87588bdfc46f72e7c2b01f8a63da9d1c1f Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Fri, 16 May 2025 22:28:17 +0200 Subject: [PATCH] fix(extractor): do not throw for broken/outdated link --- ...stry.test.ts => ExtractorRegistry.test.ts} | 8 +- src/extractor/ExtractorRegistry.ts | 20 +- src/extractor/types.ts | 2 +- src/utils/StreamResolver.ts | 2 +- .../Fetcher/https:mixdrop.mye123456789 | 52 +++++ .../Fetcher/https:mixdrop.myf123456789 | 192 ++++++++++++++++++ 6 files changed, 264 insertions(+), 12 deletions(-) rename src/extractor/{EmbedExtractorRegistry.test.ts => ExtractorRegistry.test.ts} (76%) create mode 100644 src/utils/__fixtures__/Fetcher/https:mixdrop.mye123456789 create mode 100644 src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789 diff --git a/src/extractor/EmbedExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts similarity index 76% rename from src/extractor/EmbedExtractorRegistry.test.ts rename to src/extractor/ExtractorRegistry.test.ts index 6fe6fca..1e43123 100644 --- a/src/extractor/EmbedExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -9,15 +9,19 @@ const logger = winston.createLogger({ transports: [new winston.transports.Consol const fetcher = new Fetcher(); const extractorRegistry = new ExtractorRegistry(logger, fetcher); -describe('EmbedExtractorRegistry', () => { +describe('ExtractorRegistry', () => { const ctx: Context = { ip: '127.0.0.1' }; - test('returns undefined when no embed extractor can be found', async () => { + test('returns undefined when no extractor can be found', async () => { const urlResult = await extractorRegistry.handle(ctx, new URL('https://some-url.test'), 'en'); expect(urlResult).toBeUndefined(); }); + test('returns undefined when extractor fails', async () => { + expect(await extractorRegistry.handle(ctx, new URL('https://mixdrop.my/e/123456789'), 'en')).toBeUndefined(); + }); + test('returns from memory cache if possible', async () => { const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 3480ea7..7d8c9cb 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -10,12 +10,12 @@ import { Mixdrop } from './Mixdrop'; export class ExtractorRegistry { private readonly logger: winston.Logger; - private readonly embedExtractors: Extractor[]; + private readonly extractors: Extractor[]; private readonly urlResultCache: TTLCache; constructor(logger: winston.Logger, fetcher: Fetcher) { this.logger = logger; - this.embedExtractors = [ + this.extractors = [ new DoodStream(fetcher), new Dropload(fetcher), new Mixdrop(fetcher), @@ -30,18 +30,22 @@ export class ExtractorRegistry { return urlResult; } - const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url)); - if (undefined === embedExtractor) { + const extractor = this.extractors.find(embedExtractor => embedExtractor.supports(url)); + if (undefined === extractor) { return undefined; } - this.logger.info(`Extract stream URL using ${embedExtractor.id} extractor from ${url}`); + this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`); - urlResult = await embedExtractor.extract(ctx, url, countryCode); - if (urlResult) { - this.urlResultCache.set(url.href, urlResult, { ttl: embedExtractor.ttl }); + try { + urlResult = await extractor.extract(ctx, url, countryCode); + } catch (error) { + this.logger.warn(`${extractor.id} error: ` + error); + return undefined; } + this.urlResultCache.set(url.href, urlResult, { ttl: extractor.ttl }); + return urlResult; }; } diff --git a/src/extractor/types.ts b/src/extractor/types.ts index e3306b9..124ee64 100644 --- a/src/extractor/types.ts +++ b/src/extractor/types.ts @@ -9,5 +9,5 @@ export interface Extractor { readonly supports: (url: URL) => boolean; - readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise; + readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise; } diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index 0c74b0b..72ea97e 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -41,7 +41,7 @@ export class StreamResolver { ytId: 'E4WlUXrJgy4', }); - this.logger.error(`${handler.id} error: ` + err); + this.logger.error(`${handler.id} error: ${err}`); } }); await Promise.all(handlerPromises); diff --git a/src/utils/__fixtures__/Fetcher/https:mixdrop.mye123456789 b/src/utils/__fixtures__/Fetcher/https:mixdrop.mye123456789 new file mode 100644 index 0000000..0710bf3 --- /dev/null +++ b/src/utils/__fixtures__/Fetcher/https:mixdrop.mye123456789 @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +

WE ARE SORRY

+

We can't find the video you are looking for.

+ +
+
+ + + + + + diff --git a/src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789 b/src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789 new file mode 100644 index 0000000..01ac7e4 --- /dev/null +++ b/src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789 @@ -0,0 +1,192 @@ + + + + + + +MixDrop - File not found + + + + + + + + + + + + + + + + + + +
+
+
+ + + +
+ SIGNUP + LOGIN +
menu
+
+
+
+
+ +

WE ARE SORRY

+

We can't find the file you are looking for.

+
+ +
+
+
+ + +
+
+
Uploads ( 0 / 0 )
+
+
+
+
+
+
+ + + + + + + + + + +
+ LOGIN + SIGNUP + HOME + USER PANEL + SHARE + EMBED + AFFILIATE + FAQ + NEWS + CONTACT +
+ + + + + + + + + + + + + + +