From db860c25c687b87bbec34bc72995ed47aa9115aa Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Mon, 26 Jan 2026 13:37:26 +0000 Subject: [PATCH] fix(extractor): ensure every result has a ttl --- src/extractor/Extractor.ts | 1 + src/extractor/ExtractorRegistry.ts | 22 +++++++++---------- src/extractor/Fsst.ts | 1 + .../ExtractorRegistry.test.ts.snap | 1 + src/extractor/__snapshots__/Fsst.test.ts.snap | 1 + src/types.ts | 2 +- src/utils/StreamResolver.test.ts | 14 ++++++++++++ src/utils/StreamResolver.ts | 4 ---- .../__snapshots__/StreamResolver.test.ts.snap | 2 ++ 9 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/extractor/Extractor.ts b/src/extractor/Extractor.ts index 1195959..0ac5b0a 100644 --- a/src/extractor/Extractor.ts +++ b/src/extractor/Extractor.ts @@ -42,6 +42,7 @@ export abstract class Extractor { isExternal: true, error, label: this.formatLabel(this.label), + ttl: 0, meta, }, ]; diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 0033c78..4927535 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -89,18 +89,18 @@ export class ExtractorRegistry { }, ); - if (!urlResults.length) { - await this.urlResultCache.set(cacheKey, urlResults, 43200000); // 12h - } else if (!urlResults.some(urlResult => urlResult.error)) { - /* istanbul ignore else */ - if (extractor.ttl) { - await this.urlResultCache.set(cacheKey, urlResults, extractor.ttl); - } - if (extractor.id !== 'external') { - await this.lazyUrlResultCache.set(normalizedUrl.href, urlResults, 2629800000); // 1 month - } - } else { + if (urlResults.some(urlResult => urlResult.error)) { await this.lazyUrlResultCache.delete(normalizedUrl.href); + + return urlResults; + } + + const ttl = urlResults.length ? extractor.ttl : 43200000; // 12h + + await this.urlResultCache.set(cacheKey, urlResults, ttl); + + if (extractor.id !== 'external') { + await this.lazyUrlResultCache.set(normalizedUrl.href, urlResults, 2629800000); // 1 month } return urlResults; diff --git a/src/extractor/Fsst.ts b/src/extractor/Fsst.ts index 3e6fcc9..b800ae3 100644 --- a/src/extractor/Fsst.ts +++ b/src/extractor/Fsst.ts @@ -30,6 +30,7 @@ export class Fsst extends Extractor { url: await this.fetcher.getFinalRedirectUrl(ctx, new URL(fileHref), { headers, noProxyHeaders: true }, 1), format: Format.mp4, label: this.label, + ttl: this.ttl, meta: { ...meta, height: parseInt(heightAndUrlMatch[1] as string), diff --git a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap index 28e13fe..330294e 100644 --- a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap +++ b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap @@ -28,6 +28,7 @@ exports[`ExtractorRegistry returns external url for error 1`] = ` "meta": { "extractorId": "dropload", }, + "ttl": 0, "url": "https://dropload.io/mocked-blocked.html", }, ] diff --git a/src/extractor/__snapshots__/Fsst.test.ts.snap b/src/extractor/__snapshots__/Fsst.test.ts.snap index 080cb7f..1d68065 100644 --- a/src/extractor/__snapshots__/Fsst.test.ts.snap +++ b/src/extractor/__snapshots__/Fsst.test.ts.snap @@ -10,6 +10,7 @@ exports[`Fsst Wake up Dead Man 1`] = ` "height": 1080, "title": "Wake.Up.Dead.Man.A.Knives.Out.Mystery.2025.mkv", }, + "ttl": 900000, "url": "https://a41.filevideo1.com/remote_control.php?time=1767712695&cv=a5df8582e6558fcf6bc25d7f02e5e646&lr=0&cv2=81173262e5b3e34fa9ebf67326ca98a5&file=%2Fvideos%2F948000%2F948429%2F948429.mp4&cv3=2d8ee1a66c0ca6ce75d84c67a50a70e9&cv4=7cf7fccae0c48291e3452c8ca2277f70", }, ] diff --git a/src/types.ts b/src/types.ts index 828b354..d4eb231 100644 --- a/src/types.ts +++ b/src/types.ts @@ -100,7 +100,7 @@ export interface UrlResult { ytId?: string; error?: unknown; label: string; - ttl?: number; + ttl: number; meta?: Meta; notWebReady?: boolean; requestHeaders?: Record; diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 857fd51..f037184 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -132,6 +132,7 @@ describe('resolve', () => { format: Format.unknown, isExternal: true, label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -142,6 +143,7 @@ describe('resolve', () => { isExternal: true, error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_challenge, {}), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -152,6 +154,7 @@ describe('resolve', () => { isExternal: true, error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_censor, {}), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -162,6 +165,7 @@ describe('resolve', () => { isExternal: true, error: new BlockedError(new URL('https://example.com'), BlockedReason.media_flow_proxy_auth, {}), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -172,6 +176,7 @@ describe('resolve', () => { isExternal: true, error: new BlockedError(new URL('https://example.com'), BlockedReason.unknown, {}), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -180,6 +185,7 @@ describe('resolve', () => { url: new URL('https://working2.com'), format: Format.unknown, label: 'working1', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -190,6 +196,7 @@ describe('resolve', () => { isExternal: true, error: new TooManyRequestsError(new URL('https://example.com'), 10), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -200,6 +207,7 @@ describe('resolve', () => { isExternal: true, error: new TooManyTimeoutsError(new URL('https://example.com')), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -208,6 +216,7 @@ describe('resolve', () => { url: new URL('https://working1.com'), format: Format.unknown, label: 'working2', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -218,6 +227,7 @@ describe('resolve', () => { isExternal: true, error: new TypeError(), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -228,6 +238,7 @@ describe('resolve', () => { isExternal: true, error: new TimeoutError(new URL('https://example2.com')), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -238,6 +249,7 @@ describe('resolve', () => { isExternal: true, error: new QueueIsFullError(new URL('https://example3.com')), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -248,6 +260,7 @@ describe('resolve', () => { isExternal: true, error: new HttpError(new URL('https://example4.com'), 500, 'Internal Server Error', { 'x-foo': 'bar' }), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, @@ -258,6 +271,7 @@ describe('resolve', () => { isExternal: true, error: new HttpError(new URL('https://example5.com'), 418, 'I\'m a tea pot', { 'x-foo': 'bar' }), label: 'hoster.com', + ttl: this.ttl, meta: { countryCodes: [CountryCode.de], }, diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index ffa7c83..d5317c0 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -176,10 +176,6 @@ export class StreamResolver { return 900000; // 15m } - if (urlResults.some(urlResult => urlResult.ttl === undefined)) { - return undefined; - } - return Math.min(...urlResults.map(urlResult => urlResult.ttl as number)); }; diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index df269cd..8fca882 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -31,6 +31,7 @@ exports[`resolve adds error info 1`] = ` "title": "🔗 hoster.com", }, ], + "ttl": 1, } `; @@ -175,6 +176,7 @@ exports[`resolve adds error info 2`] = ` "title": "🔗 hoster.com", }, ], + "ttl": 1, } `;