fix(extractor): ensure every result has a ttl
This commit is contained in:
parent
074311107a
commit
db860c25c6
9 changed files with 32 additions and 16 deletions
|
|
@ -42,6 +42,7 @@ export abstract class Extractor {
|
|||
isExternal: true,
|
||||
error,
|
||||
label: this.formatLabel(this.label),
|
||||
ttl: 0,
|
||||
meta,
|
||||
},
|
||||
];
|
||||
|
|
|
|||
|
|
@ -89,18 +89,18 @@ export class ExtractorRegistry {
|
|||
},
|
||||
);
|
||||
|
||||
if (!urlResults.length) {
|
||||
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, 43200000); // 12h
|
||||
} else if (!urlResults.some(urlResult => urlResult.error)) {
|
||||
/* istanbul ignore else */
|
||||
if (extractor.ttl) {
|
||||
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, extractor.ttl);
|
||||
}
|
||||
if (extractor.id !== 'external') {
|
||||
await this.lazyUrlResultCache.set<UrlResult[]>(normalizedUrl.href, urlResults, 2629800000); // 1 month
|
||||
}
|
||||
} else {
|
||||
if (urlResults.some(urlResult => urlResult.error)) {
|
||||
await this.lazyUrlResultCache.delete(normalizedUrl.href);
|
||||
|
||||
return urlResults;
|
||||
}
|
||||
|
||||
const ttl = urlResults.length ? extractor.ttl : 43200000; // 12h
|
||||
|
||||
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, ttl);
|
||||
|
||||
if (extractor.id !== 'external') {
|
||||
await this.lazyUrlResultCache.set<UrlResult[]>(normalizedUrl.href, urlResults, 2629800000); // 1 month
|
||||
}
|
||||
|
||||
return urlResults;
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ export class Fsst extends Extractor {
|
|||
url: await this.fetcher.getFinalRedirectUrl(ctx, new URL(fileHref), { headers, noProxyHeaders: true }, 1),
|
||||
format: Format.mp4,
|
||||
label: this.label,
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
...meta,
|
||||
height: parseInt(heightAndUrlMatch[1] as string),
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ exports[`ExtractorRegistry returns external url for error 1`] = `
|
|||
"meta": {
|
||||
"extractorId": "dropload",
|
||||
},
|
||||
"ttl": 0,
|
||||
"url": "https://dropload.io/mocked-blocked.html",
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ exports[`Fsst Wake up Dead Man 1`] = `
|
|||
"height": 1080,
|
||||
"title": "Wake.Up.Dead.Man.A.Knives.Out.Mystery.2025.mkv",
|
||||
},
|
||||
"ttl": 900000,
|
||||
"url": "https://a41.filevideo1.com/remote_control.php?time=1767712695&cv=a5df8582e6558fcf6bc25d7f02e5e646&lr=0&cv2=81173262e5b3e34fa9ebf67326ca98a5&file=%2Fvideos%2F948000%2F948429%2F948429.mp4&cv3=2d8ee1a66c0ca6ce75d84c67a50a70e9&cv4=7cf7fccae0c48291e3452c8ca2277f70",
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ export interface UrlResult {
|
|||
ytId?: string;
|
||||
error?: unknown;
|
||||
label: string;
|
||||
ttl?: number;
|
||||
ttl: number;
|
||||
meta?: Meta;
|
||||
notWebReady?: boolean;
|
||||
requestHeaders?: Record<string, string>;
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ describe('resolve', () => {
|
|||
format: Format.unknown,
|
||||
isExternal: true,
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -142,6 +143,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_challenge, {}),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -152,6 +154,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_censor, {}),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -162,6 +165,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new BlockedError(new URL('https://example.com'), BlockedReason.media_flow_proxy_auth, {}),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -172,6 +176,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new BlockedError(new URL('https://example.com'), BlockedReason.unknown, {}),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -180,6 +185,7 @@ describe('resolve', () => {
|
|||
url: new URL('https://working2.com'),
|
||||
format: Format.unknown,
|
||||
label: 'working1',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -190,6 +196,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new TooManyRequestsError(new URL('https://example.com'), 10),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -200,6 +207,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new TooManyTimeoutsError(new URL('https://example.com')),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -208,6 +216,7 @@ describe('resolve', () => {
|
|||
url: new URL('https://working1.com'),
|
||||
format: Format.unknown,
|
||||
label: 'working2',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -218,6 +227,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new TypeError(),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -228,6 +238,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new TimeoutError(new URL('https://example2.com')),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -238,6 +249,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new QueueIsFullError(new URL('https://example3.com')),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -248,6 +260,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new HttpError(new URL('https://example4.com'), 500, 'Internal Server Error', { 'x-foo': 'bar' }),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
@ -258,6 +271,7 @@ describe('resolve', () => {
|
|||
isExternal: true,
|
||||
error: new HttpError(new URL('https://example5.com'), 418, 'I\'m a tea pot', { 'x-foo': 'bar' }),
|
||||
label: 'hoster.com',
|
||||
ttl: this.ttl,
|
||||
meta: {
|
||||
countryCodes: [CountryCode.de],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -176,10 +176,6 @@ export class StreamResolver {
|
|||
return 900000; // 15m
|
||||
}
|
||||
|
||||
if (urlResults.some(urlResult => urlResult.ttl === undefined)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Math.min(...urlResults.map(urlResult => urlResult.ttl as number));
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ exports[`resolve adds error info 1`] = `
|
|||
"title": "🔗 hoster.com",
|
||||
},
|
||||
],
|
||||
"ttl": 1,
|
||||
}
|
||||
`;
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ exports[`resolve adds error info 2`] = `
|
|||
"title": "🔗 hoster.com",
|
||||
},
|
||||
],
|
||||
"ttl": 1,
|
||||
}
|
||||
`;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue