chore: cache playlist heights for a month

src/extractor/Dropload.ts

@@ -32,7 +32,7 @@ export class Dropload extends Extractor {
     const heightMatch = html.match(/\d{3,}x(\d{3,}),/);
     const height = heightMatch
       ? parseInt(heightMatch[1] as string)
-      : await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl);
+      : await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url);
 
     const sizeMatch = html.match(/([\d.]+ ?[GM]B)/);
     const size = sizeMatch ? bytes.parse(sizeMatch[1] as string) as number : undefined;

src/extractor/KinoGer.ts

@@ -60,7 +60,7 @@ export class KinoGer extends Extractor {
         ttl: this.ttl,
         meta: {
           ...meta,
-          height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers: { Referer: url.href } }),
+          height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers: { Referer: url.href } }),
           title,
         },
         requestHeaders: {

src/extractor/SuperVideo.ts

@@ -39,7 +39,7 @@ export class SuperVideo extends Extractor {
     const size = heightAndSizeMatch ? bytes.parse(heightAndSizeMatch[2] as string) as number : undefined;
     const height = heightAndSizeMatch
       ? parseInt(heightAndSizeMatch[1] as string)
-      : await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers: { Referer: url.href } });
+      : await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers: { Referer: url.href } });
 
     const $ = cheerio.load(html);
     const title = $('.download__title').text().trim();

src/extractor/VidSrc.ts

@@ -77,7 +77,7 @@ export class VidSrc extends Extractor {
             ttl: this.ttl,
             meta: {
               ...meta,
-              height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, { headers: { Referer: iframeUrl.href } }),
+              height: await guessHeightFromPlaylist(ctx, this.fetcher, m3u8Url, url, { headers: { Referer: iframeUrl.href } }),
               title,
             },
           };

src/extractor/VixSrc.ts

@@ -47,7 +47,7 @@ export class VixSrc extends Extractor {
         ttl: this.ttl,
         meta: {
           countryCodes,
-          height: await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, { headers }),
+          height: await guessHeightFromPlaylist(ctx, this.fetcher, playlistUrl, url, { headers }),
         },
       },
     ];

src/utils/height.ts

@@ -1,13 +1,32 @@
+// eslint-disable-next-line import/no-named-as-default
+import KeyvSqlite from '@keyv/sqlite';
+import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
 import { Context } from '../types';
+import { getCacheDir } from './env';
 import { CustomRequestInit, Fetcher } from './Fetcher';
 
-export const guessHeightFromPlaylist = async (ctx: Context, fetcher: Fetcher, url: URL, init?: CustomRequestInit): Promise<number | undefined> => {
-  const m3u8Data = await fetcher.text(ctx, url, init);
+const playlistHeightCache = new Cacheable({
+  primary: new Keyv({ store: new CacheableMemory({ lruSize: 16384 }) }),
+  secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-playlist-height-cache.sqlite`)),
+});
+
+export const guessHeightFromPlaylist = async (ctx: Context, fetcher: Fetcher, playlistUrl: URL, embedUrl: URL, init?: CustomRequestInit): Promise<number | undefined> => {
+  let height = await playlistHeightCache.get<number>(embedUrl.href);
+  /* istanbul ignore if */
+  if (height) {
+    return height;
+  }
+
+  const m3u8Data = await fetcher.text(ctx, playlistUrl, init);
 
   const heights = Array.from(m3u8Data.matchAll(/\d+x(\d+)|(\d+)p/g))
     .map(heightMatch => heightMatch[1] ?? heightMatch[2])
     .filter(height => height !== undefined)
     .map(height => parseInt(height));
 
-  return heights.length ? Math.max(...heights) : /* istanbul ignore next */ undefined;
+  height = heights.length ? Math.max(...heights) : /* istanbul ignore next */ undefined;
+
+  await playlistHeightCache.set(embedUrl.href, height, 2628000); // 1 month
+
+  return height;
 };