feat: introduce generic meta cache

This commit is contained in:
WebStreamr 2026-01-12 10:50:55 +00:00
parent 816123ab83
commit f047287fe9
No known key found for this signature in database
34 changed files with 28 additions and 86 deletions

View file

@ -32,7 +32,7 @@ export class Dropload extends Extractor {
const heightMatch = html.match(/\d{3,}x(\d{3,}),/);
const height = heightMatch
? parseInt(heightMatch[1] as string)
: await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url);
: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl);
const sizeMatch = html.match(/([\d.]+ ?[GM]B)/);
const size = sizeMatch ? bytes.parse(sizeMatch[1] as string) as number : undefined;

View file

@ -9,12 +9,19 @@ import { Extractor } from './Extractor';
export class ExtractorRegistry {
private readonly logger: winston.Logger;
private readonly extractors: Extractor[];
private readonly metaCache: Cacheable;
private readonly urlResultCache: Cacheable;
public constructor(logger: winston.Logger, extractors: Extractor[]) {
this.logger = logger;
this.extractors = extractors;
this.metaCache = new Cacheable({
nonBlocking: true,
primary: new Keyv({ store: new CacheableMemory({ lruSize: 16384 }) }),
secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-meta-cache.sqlite`)),
});
this.urlResultCache = new Cacheable({
nonBlocking: true,
primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }),
@ -55,12 +62,14 @@ export class ExtractorRegistry {
this.logger.info(`Extract ${url} using ${extractor.id} extractor`, ctx);
const cachedMeta = await this.metaCache.get<Meta>(normalizedUrl.href);
const urlResults = await extractor.extract(
ctx,
normalizedUrl,
{
...meta,
countryCodes: meta?.countryCodes ?? [],
...cachedMeta,
extractorId: meta?.extractorId ?? extractor.id,
},
);
@ -71,6 +80,10 @@ export class ExtractorRegistry {
await this.urlResultCache.set<UrlResult[]>(cacheKey, urlResults, extractor.ttl);
}
if (urlResults.length) {
await this.metaCache.set<Meta>(normalizedUrl.href, urlResults[0]?.meta as Meta, 2628000); // 1 month
}
return urlResults;
};

View file

@ -70,7 +70,7 @@ export class KinoGer extends Extractor {
ttl: this.ttl,
meta: {
...meta,
height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers }),
height: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers }),
title,
},
requestHeaders: headers,

View file

@ -38,7 +38,7 @@ export class RgShows extends Extractor {
ttl: this.ttl,
meta: {
...meta,
...(isHls && { height: await guessHeightFromPlaylist(ctx, this.fetcher, streamUrl, url, { headers }) }),
...(isHls && { height: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, streamUrl, { headers }) }),
},
requestHeaders: headers,
},

View file

@ -40,7 +40,7 @@ export class StreamUp extends Extractor {
ttl: this.ttl,
meta: {
...meta,
height: await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url),
height: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl),
title,
},
},

View file

@ -39,7 +39,7 @@ export class SuperVideo extends Extractor {
const size = heightAndSizeMatch ? bytes.parse(heightAndSizeMatch[2] as string) as number : undefined;
const height = heightAndSizeMatch
? parseInt(heightAndSizeMatch[1] as string)
: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers: { Referer: url.href } });
: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers: { Referer: url.href } });
const $ = cheerio.load(html);
const title = $('.download__title').text().trim();

View file

@ -35,7 +35,7 @@ export class Vidora extends Extractor {
ttl: this.ttl,
meta: {
...meta,
height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers }),
height: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers }),
title,
},
requestHeaders: headers,

View file

@ -32,7 +32,7 @@ export class VixSrc extends Extractor {
playlistUrl.searchParams.append('expires', expiresMatch[1] as string);
playlistUrl.searchParams.append('h', '1');
const countryCodes = await this.determineCountryCodesFromPlaylist(ctx, playlistUrl, { headers });
const countryCodes = meta.countryCodes ?? await this.determineCountryCodesFromPlaylist(ctx, playlistUrl, { headers });
if (!hasMultiEnabled(ctx.config) && !countryCodes.some(countryCode => countryCode in ctx.config)) {
return [];
@ -47,7 +47,7 @@ export class VixSrc extends Extractor {
meta: {
...meta,
countryCodes,
height: await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url, { headers }),
height: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, { headers }),
},
},
];

View file

@ -157,7 +157,9 @@ export class Voe extends Extractor {
const playlistUrl = await buildMediaFlowProxyExtractorStreamUrl(ctx, this.fetcher, 'Voe', url, headers);
const heightMatch = html.match(/<b>(\d{3,})p<\/b>/);
const height = heightMatch ? parseInt(heightMatch[1] as string) : await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url);
const height = heightMatch
? parseInt(heightMatch[1] as string)
: meta.height ?? await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl);
return [
{

View file

@ -7,7 +7,6 @@ exports[`DoodStream can guess height from title 1`] = `
"label": "DoodStream",
"meta": {
"bytes": 549139251,
"countryCodes": [],
"extractorId": "doodstream",
"title": "Black Mirror S04E02 FRENCH 720p WEB x264-RiPiT",
},
@ -26,7 +25,6 @@ exports[`DoodStream cloudflare storage 1`] = `
"format": "mp4",
"label": "DoodStream",
"meta": {
"countryCodes": [],
"extractorId": "doodstream",
"title": "241653--5bf448d3-184a-4b3f-a766-fa98d93c1300--bllz--2100527-doodstream",
},
@ -46,7 +44,6 @@ exports[`DoodStream dood.to 1`] = `
"label": "DoodStream",
"meta": {
"bytes": 1395864371,
"countryCodes": [],
"extractorId": "doodstream",
"title": "des-teufels-bad-2024",
},

View file

@ -7,7 +7,6 @@ exports[`Dropload download URL 1`] = `
"label": "Dropload",
"meta": {
"bytes": 333027737,
"countryCodes": [],
"extractorId": "dropload",
"height": 720,
"title": "[~Squid~Game~EnG~]~S03E01~720p~WebRip ~❤GLD❤~",
@ -24,7 +23,6 @@ exports[`Dropload dropload unknown height and size 1`] = `
"format": "hls",
"label": "Dropload",
"meta": {
"countryCodes": [],
"extractorId": "dropload",
"height": 800,
"title": "The Lost Bus 2025 1080p WEBRip x265",
@ -42,7 +40,6 @@ exports[`Dropload dropload.io 1`] = `
"label": "Dropload",
"meta": {
"bytes": 1395864371,
"countryCodes": [],
"extractorId": "dropload",
"height": 1080,
"title": "des-teufels-bad-2024",

View file

@ -7,7 +7,6 @@ exports[`ExternalUrl 404 - not found 1`] = `
"isExternal": true,
"label": "streamtape.com",
"meta": {
"countryCodes": [],
"extractorId": "external",
},
"ttl": 21600000,
@ -23,7 +22,6 @@ exports[`ExternalUrl netu.fanstream.us 1`] = `
"isExternal": true,
"label": "netu.fanstream.us",
"meta": {
"countryCodes": [],
"extractorId": "external",
},
"ttl": 21600000,

View file

@ -9,7 +9,6 @@ exports[`ExtractorRegistry returns external URLs if enabled by config 1`] = `
"isExternal": true,
"label": "mixdrop.ag",
"meta": {
"countryCodes": [],
"extractorId": "external",
},
"ttl": 21600000,
@ -27,7 +26,6 @@ exports[`ExtractorRegistry returns external url for error 1`] = `
"isExternal": true,
"label": "Dropload",
"meta": {
"countryCodes": [],
"extractorId": "dropload",
},
"url": "https://dropload.io/mocked-blocked.html",

View file

@ -9,7 +9,6 @@ exports[`Fastream fastream.to embed 1`] = `
"label": "Fastream (MFP)",
"meta": {
"bytes": 1073741824,
"countryCodes": [],
"extractorId": "fastream",
"height": 1080,
"title": "black.mirror.s01e01.lat.720p.mp4",

View file

@ -11,7 +11,6 @@ exports[`FileLions filelions f 1`] = `
"label": "FileLions (MFP)",
"meta": {
"bytes": 481191526,
"countryCodes": [],
"extractorId": "filelions",
"height": 720,
"title": "dexter resurrection s01e01 1080p web h264-successfulcrab",
@ -29,7 +28,6 @@ exports[`FileLions mivalyo v referer lock 1`] = `
"label": "FileLions (MFP)",
"meta": {
"bytes": 2097152,
"countryCodes": [],
"extractorId": "filelions",
"height": 1080,
"referer": "https://kinoger.com",

View file

@ -6,7 +6,6 @@ exports[`FileMoon missing height 1`] = `
"format": "hls",
"label": "FileMoon (MFP)",
"meta": {
"countryCodes": [],
"extractorId": "filemoon",
"title": "tt0307453",
},
@ -24,7 +23,6 @@ exports[`FileMoon z1ekv717 d 1`] = `
"format": "hls",
"label": "FileMoon (MFP)",
"meta": {
"countryCodes": [],
"extractorId": "filemoon",
"height": 720,
"title": "alien earth s01e01 1080p web h264-successfulcrab",

View file

@ -6,7 +6,6 @@ exports[`Fsst Wake up Dead Man 1`] = `
"format": "mp4",
"label": "Fsst",
"meta": {
"countryCodes": [],
"extractorId": "fsst",
"height": 1080,
"title": "Wake.Up.Dead.Man.A.Knives.Out.Mystery.2025.mkv",

View file

@ -7,7 +7,6 @@ exports[`HubCloud handle crayon shin-chan 1993 1`] = `
"label": "HubCloud (FSL)",
"meta": {
"bytes": 28550795100,
"countryCodes": [],
"extractorId": "hubcloud_fsl",
"title": "Crayon Shinchan Action Kamen vs Demon (1993) 1080p BluRay REMUX AVC [Hindi DD 2.0 + Japanese FLAC 2.0] (PitiPati-Ionicboy) (4kHDHub.com).mkv",
},
@ -24,7 +23,6 @@ exports[`HubCloud handle dark 2017 s03e08 1`] = `
"label": "HubCloud (PixelServer)",
"meta": {
"bytes": 3382286745,
"countryCodes": [],
"extractorId": "hubcloud_pixelserver",
"title": "Dark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv",
},
@ -41,7 +39,6 @@ exports[`HubCloud handle dexter original sin 2024 s01e01 1`] = `
"label": "HubCloud (FSL)",
"meta": {
"bytes": 5733781340,
"countryCodes": [],
"extractorId": "hubcloud_fsl",
"title": "Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv",
},
@ -53,7 +50,6 @@ exports[`HubCloud handle dexter original sin 2024 s01e01 1`] = `
"label": "HubCloud (PixelServer)",
"meta": {
"bytes": 5733781340,
"countryCodes": [],
"extractorId": "hubcloud_pixelserver",
"title": "Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv",
},

View file

@ -6,7 +6,6 @@ exports[`KinoGer Blood & Sinners 1`] = `
"format": "hls",
"label": "KinoGer",
"meta": {
"countryCodes": [],
"extractorId": "kinoger",
"height": 1080,
"title": "Blood.and.Sinners.2025.mkv",
@ -28,7 +27,6 @@ exports[`KinoGer Dead City 1`] = `
"format": "hls",
"label": "KinoGer",
"meta": {
"countryCodes": [],
"extractorId": "kinoger",
"height": 1080,
"title": "the.walking.dead.dead.city.s02e05.german.dl.1080p.web.x264-wvf.mkv",
@ -50,7 +48,6 @@ exports[`KinoGer filma365.strp2p.site 1`] = `
"format": "hls",
"label": "KinoGer",
"meta": {
"countryCodes": [],
"extractorId": "kinoger",
"height": 640,
"title": "Our.Fault.2025.mp4",
@ -72,7 +69,6 @@ exports[`KinoGer kinoger.p2pplay.pro 1`] = `
"format": "hls",
"label": "KinoGer",
"meta": {
"countryCodes": [],
"extractorId": "kinoger",
"height": 960,
"title": "Stranger.Things.S01E01.German.AC3.DL.1080p.WebRip.x265-FuN.mkv",

View file

@ -9,7 +9,6 @@ exports[`LuluStream streamhihi 1`] = `
"label": "LuluStream (MFP)",
"meta": {
"bytes": 1288490188,
"countryCodes": [],
"extractorId": "lulustream",
"height": 720,
"title": "The Thursday Murder Club 2025 1080p NF WEB-DL DDP5 1 Atmos H 264-playWEB",
@ -27,7 +26,6 @@ exports[`LuluStream streamhihi d 1`] = `
"label": "LuluStream (MFP)",
"meta": {
"bytes": 1288490188,
"countryCodes": [],
"extractorId": "lulustream",
"height": 720,
"title": "The Thursday Murder Club 2025 1080p NF WEB-DL DDP5 1 Atmos H 264-playWEB",

View file

@ -9,7 +9,6 @@ exports[`Mixdrop mixdrop.my /e/ 1`] = `
"label": "Mixdrop (MFP)",
"meta": {
"bytes": 1059806248,
"countryCodes": [],
"extractorId": "mixdrop",
"title": "28_giorni_dopo_2002_HD_-_Altadefinizione01.mp4",
},

View file

@ -6,7 +6,6 @@ exports[`RgShows handle one battle after another 1`] = `
"format": "hls",
"label": "RgShows",
"meta": {
"countryCodes": [],
"extractorId": "rgshows",
"height": 1080,
},
@ -27,7 +26,6 @@ exports[`RgShows handle parasite 2019 (mp4) 1`] = `
"format": "mp4",
"label": "RgShows",
"meta": {
"countryCodes": [],
"extractorId": "rgshows",
},
"requestHeaders": {
@ -47,7 +45,6 @@ exports[`RgShows handle stranger things s05e01 1`] = `
"format": "hls",
"label": "RgShows",
"meta": {
"countryCodes": [],
"extractorId": "rgshows",
"height": 1080,
},

View file

@ -6,7 +6,6 @@ exports[`SafeFiles savefiles /d/ 1`] = `
"format": "hls",
"label": "SaveFiles",
"meta": {
"countryCodes": [],
"extractorId": "savefiles",
"height": 532,
"title": "dexter resurrection s01e01 1080p web h264-successfulcrab",
@ -23,7 +22,6 @@ exports[`SafeFiles savefiles /e/ 1`] = `
"format": "hls",
"label": "SaveFiles",
"meta": {
"countryCodes": [],
"extractorId": "savefiles",
"height": 720,
"title": "dexter resurrection s01e01 1080p web h264-successfulcrab",
@ -40,7 +38,6 @@ exports[`SafeFiles savefiles 1`] = `
"format": "hls",
"label": "SaveFiles",
"meta": {
"countryCodes": [],
"extractorId": "savefiles",
"height": 720,
"title": "King Of The Hill S14E01 1080p HEVC x265-MeGusta",
@ -61,7 +58,6 @@ exports[`SafeFiles streamhls /e/ 1`] = `
"format": "hls",
"label": "SaveFiles",
"meta": {
"countryCodes": [],
"extractorId": "savefiles",
"height": 720,
"title": "King Of The Hill S14E01 1080p HEVC x265-MeGusta",

View file

@ -8,7 +8,6 @@ exports[`StreamEmbed watch.gxplayer.xyz 1`] = `
"format": "hls",
"label": "StreamEmbed",
"meta": {
"countryCodes": [],
"extractorId": "streamembed",
"height": 720,
"title": "Baymax - Riesiges Robowabohu.mp4",

View file

@ -6,7 +6,6 @@ exports[`StreamUp handle one battle after another 1`] = `
"format": "hls",
"label": "StreamUP",
"meta": {
"countryCodes": [],
"extractorId": "streamup",
"height": 1080,
"title": "Wake.Up.Dead.Man.A.Knives.Out.Mystery.2025",

View file

@ -7,7 +7,6 @@ exports[`Streamtape streamtape.com /e 1`] = `
"label": "Streamtape (MFP)",
"meta": {
"bytes": 1288490188,
"countryCodes": [],
"extractorId": "streamtape",
"title": "WALL-E.2008.1080p.BrRip.x264.YIFY.mp4",
},

View file

@ -8,7 +8,6 @@ exports[`SuperVideo embed only 1`] = `
"format": "hls",
"label": "SuperVideo",
"meta": {
"countryCodes": [],
"extractorId": "supervideo",
"height": 1080,
"title": "",
@ -28,7 +27,6 @@ exports[`SuperVideo supervideo.cc /e/ 1`] = `
"label": "SuperVideo",
"meta": {
"bytes": 1073741824,
"countryCodes": [],
"extractorId": "supervideo",
"height": 720,
"title": "des-teufels-bad-2024",
@ -46,7 +44,6 @@ exports[`SuperVideo supervideo.tv /embed-/ 1`] = `
"label": "SuperVideo",
"meta": {
"bytes": 219571814,
"countryCodes": [],
"extractorId": "supervideo",
"height": 344,
"title": "Babylon 5 2x3",

View file

@ -8,7 +8,6 @@ exports[`Uqload uqload.net /embed- 1`] = `
"format": "mp4",
"label": "Uqload (MFP)",
"meta": {
"countryCodes": [],
"extractorId": "uqload",
"height": 360,
"title": "Black Mirror S07E06 MULTI VFF 1080p WEBrip EAC3 5 1 x265-TyHD",

View file

@ -6,7 +6,6 @@ exports[`Vidora vidora.stream /embed/ 1`] = `
"format": "hls",
"label": "Vidora",
"meta": {
"countryCodes": [],
"extractorId": "vidora",
"height": 800,
"title": "Highest Lowest 2025 1080p ATVP WEB DDP5 Atmos 264 BYNDR",
@ -26,7 +25,6 @@ exports[`Vidora vidora.stream 1`] = `
"format": "hls",
"label": "Vidora",
"meta": {
"countryCodes": [],
"extractorId": "vidora",
"height": 800,
"title": "Highest Lowest 2025 1080p ATVP WEB DDP5 Atmos 264 BYNDR",

View file

@ -6,7 +6,6 @@ exports[`Voe embed only urls which otherwise lead to 404 1`] = `
"format": "hls",
"label": "VOE (MFP)",
"meta": {
"countryCodes": [],
"extractorId": "voe",
"height": 800,
"title": "Weapons.2025.German.DL.1080p.WEB.H264-ZeroTwo_encoded.mp4",
@ -26,7 +25,6 @@ exports[`Voe jilliandescribecompany 1`] = `
"label": "VOE (MFP)",
"meta": {
"bytes": 1202590842,
"countryCodes": [],
"extractorId": "voe",
"height": 720,
"title": "Superman.2025.2160p.WEB-DL.DV.HDR10Plus.X265.DDP5.1.Atmos-YIN.mp4",
@ -44,7 +42,6 @@ exports[`Voe premium only without resolution 1`] = `
"label": "VOE (MFP)",
"meta": {
"bytes": 363216240,
"countryCodes": [],
"extractorId": "voe",
"height": 800,
"title": "268836--21ba9fb2-97f9-4566-b5eb-3d9cef7674cf--zrxn--2352322-voesx.mp4",

View file

@ -6,7 +6,6 @@ exports[`YouTube Solaris 1`] = `
"format": "unknown",
"label": "YouTube",
"meta": {
"countryCodes": [],
"extractorId": "youtube",
"title": "Solaris | SCIENCE FICTION | FULL MOVIE | directed by Tarkovsky",
},

View file

@ -37,6 +37,6 @@ export class VixSrc extends Source {
? new URL(`/tv/${tmdbId.id}/${tmdbId.season}/${tmdbId.episode}`, this.baseUrl)
: new URL(`/movie/${tmdbId.id}`, this.baseUrl);
return [{ url, meta: { countryCodes: [CountryCode.multi], title } }];
return [{ url, meta: { title } }];
};
}

View file

@ -4,9 +4,6 @@ exports[`VixSrc handle imdb black mirror s4e2 1`] = `
[
{
"meta": {
"countryCodes": [
"multi",
],
"title": "Black Mirror S04E02",
},
"url": "https://vixsrc.to/tv/42009/4/2",
@ -18,9 +15,6 @@ exports[`VixSrc handle imdb full metal jacket 1`] = `
[
{
"meta": {
"countryCodes": [
"multi",
],
"title": "Full Metal Jacket (1987)",
},
"url": "https://vixsrc.to/movie/600",

View file

@ -1,23 +1,7 @@
// eslint-disable-next-line import/no-named-as-default
import KeyvSqlite from '@keyv/sqlite';
import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
import { Context } from '../types';
import { getCacheDir } from './env';
import { CustomRequestConfig, Fetcher } from './Fetcher';
const playlistHeightCache = new Cacheable({
nonBlocking: true,
primary: new Keyv({ store: new CacheableMemory({ lruSize: 16384 }) }),
secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-playlist-height-cache.sqlite`)),
});
export const guessHeightFromPlaylist = async (ctx: Context, fetcher: Fetcher, playlistUrl: URL, embedUrl: URL, init?: CustomRequestConfig): Promise<number | undefined> => {
let height = await playlistHeightCache.get<number>(embedUrl.href);
/* istanbul ignore if */
if (height) {
return height;
}
export const guessHeightFromPlaylist = async (ctx: Context, fetcher: Fetcher, playlistUrl: URL, init?: CustomRequestConfig): Promise<number | undefined> => {
let m3u8Data: string;
try {
m3u8Data = await fetcher.text(ctx, playlistUrl, init);
@ -31,9 +15,5 @@ export const guessHeightFromPlaylist = async (ctx: Context, fetcher: Fetcher, pl
.filter(height => height !== undefined)
.map(height => parseInt(height));
height = heights.length ? Math.max(...heights) : /* istanbul ignore next */ undefined;
await playlistHeightCache.set(embedUrl.href, height, 2628000); // 1 month
return height;
return heights.length ? Math.max(...heights) : /* istanbul ignore next */ undefined;
};