fix(extractor): ensure every result has a ttl

This commit is contained in:
WebStreamr 2026-01-26 13:37:26 +00:00
parent 074311107a
commit db860c25c6
No known key found for this signature in database
9 changed files with 32 additions and 16 deletions

View file

@ -42,6 +42,7 @@ export abstract class Extractor {
isExternal: true,
error,
label: this.formatLabel(this.label),
ttl: 0,
meta,
},
];

View file

@ -89,18 +89,18 @@ export class ExtractorRegistry {
},
);
if (!urlResults.length) {
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, 43200000); // 12h
} else if (!urlResults.some(urlResult => urlResult.error)) {
/* istanbul ignore else */
if (extractor.ttl) {
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, extractor.ttl);
}
if (extractor.id !== 'external') {
await this.lazyUrlResultCache.set<UrlResult[]>(normalizedUrl.href, urlResults, 2629800000); // 1 month
}
} else {
if (urlResults.some(urlResult => urlResult.error)) {
await this.lazyUrlResultCache.delete(normalizedUrl.href);
return urlResults;
}
const ttl = urlResults.length ? extractor.ttl : 43200000; // 12h
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, ttl);
if (extractor.id !== 'external') {
await this.lazyUrlResultCache.set<UrlResult[]>(normalizedUrl.href, urlResults, 2629800000); // 1 month
}
return urlResults;

View file

@ -30,6 +30,7 @@ export class Fsst extends Extractor {
url: await this.fetcher.getFinalRedirectUrl(ctx, new URL(fileHref), { headers, noProxyHeaders: true }, 1),
format: Format.mp4,
label: this.label,
ttl: this.ttl,
meta: {
...meta,
height: parseInt(heightAndUrlMatch[1] as string),

View file

@ -28,6 +28,7 @@ exports[`ExtractorRegistry returns external url for error 1`] = `
"meta": {
"extractorId": "dropload",
},
"ttl": 0,
"url": "https://dropload.io/mocked-blocked.html",
},
]

View file

@ -10,6 +10,7 @@ exports[`Fsst Wake up Dead Man 1`] = `
"height": 1080,
"title": "Wake.Up.Dead.Man.A.Knives.Out.Mystery.2025.mkv",
},
"ttl": 900000,
"url": "https://a41.filevideo1.com/remote_control.php?time=1767712695&cv=a5df8582e6558fcf6bc25d7f02e5e646&lr=0&cv2=81173262e5b3e34fa9ebf67326ca98a5&file=%2Fvideos%2F948000%2F948429%2F948429.mp4&cv3=2d8ee1a66c0ca6ce75d84c67a50a70e9&cv4=7cf7fccae0c48291e3452c8ca2277f70",
},
]

View file

@ -100,7 +100,7 @@ export interface UrlResult {
ytId?: string;
error?: unknown;
label: string;
ttl?: number;
ttl: number;
meta?: Meta;
notWebReady?: boolean;
requestHeaders?: Record<string, string>;

View file

@ -132,6 +132,7 @@ describe('resolve', () => {
format: Format.unknown,
isExternal: true,
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -142,6 +143,7 @@ describe('resolve', () => {
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_challenge, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -152,6 +154,7 @@ describe('resolve', () => {
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_censor, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -162,6 +165,7 @@ describe('resolve', () => {
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.media_flow_proxy_auth, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -172,6 +176,7 @@ describe('resolve', () => {
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.unknown, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -180,6 +185,7 @@ describe('resolve', () => {
url: new URL('https://working2.com'),
format: Format.unknown,
label: 'working1',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -190,6 +196,7 @@ describe('resolve', () => {
isExternal: true,
error: new TooManyRequestsError(new URL('https://example.com'), 10),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -200,6 +207,7 @@ describe('resolve', () => {
isExternal: true,
error: new TooManyTimeoutsError(new URL('https://example.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -208,6 +216,7 @@ describe('resolve', () => {
url: new URL('https://working1.com'),
format: Format.unknown,
label: 'working2',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -218,6 +227,7 @@ describe('resolve', () => {
isExternal: true,
error: new TypeError(),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -228,6 +238,7 @@ describe('resolve', () => {
isExternal: true,
error: new TimeoutError(new URL('https://example2.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -238,6 +249,7 @@ describe('resolve', () => {
isExternal: true,
error: new QueueIsFullError(new URL('https://example3.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -248,6 +260,7 @@ describe('resolve', () => {
isExternal: true,
error: new HttpError(new URL('https://example4.com'), 500, 'Internal Server Error', { 'x-foo': 'bar' }),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
@ -258,6 +271,7 @@ describe('resolve', () => {
isExternal: true,
error: new HttpError(new URL('https://example5.com'), 418, 'I\'m a tea pot', { 'x-foo': 'bar' }),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},

View file

@ -176,10 +176,6 @@ export class StreamResolver {
return 900000; // 15m
}
if (urlResults.some(urlResult => urlResult.ttl === undefined)) {
return undefined;
}
return Math.min(...urlResults.map(urlResult => urlResult.ttl as number));
};

View file

@ -31,6 +31,7 @@ exports[`resolve adds error info 1`] = `
"title": "🔗 hoster.com",
},
],
"ttl": 1,
}
`;
@ -175,6 +176,7 @@ exports[`resolve adds error info 2`] = `
"title": "🔗 hoster.com",
},
],
"ttl": 1,
}
`;