chore: regularly clean-up expired items from sqlite cache

package-lock.json

@@ -28,6 +28,7 @@
         "rot13-cipher": "^1.0.0",
         "slugify": "^1.6.6",
         "socks-proxy-agent": "^8.0.5",
+        "sqlite3": "^5.1.7",
         "tough-cookie": "^6.0.0",
         "unpacker": "^1.0.1",
         "winston": "^3.17.0"
@@ -89,7 +90,6 @@
       "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.28.6",
         "@babel/generator": "^7.28.6",
@@ -1811,7 +1811,6 @@
       "integrity": "sha512-+054pVMzVTmRQV8BhpGv3UyfZ2Llgl8rdpDTon+cUH9+na0ncBVXj3wTUKh14+Kiz18ziM3b4ikpP5/Pc0rQEQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "undici-types": "~7.16.0"
       }
@@ -1940,7 +1939,6 @@
       "integrity": "sha512-npiaib8XzbjtzS2N4HlqPvlpxpmZ14FjSJrteZpPxGUaYPlvhzlzUZ4mZyABo0EFrOWnvyd0Xxroq//hKhtAWg==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "8.53.0",
         "@typescript-eslint/types": "8.53.0",
@@ -2480,7 +2478,6 @@
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -2849,7 +2846,6 @@
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz",
       "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.4",
@@ -3162,7 +3158,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "baseline-browser-mapping": "^2.9.0",
         "caniuse-lite": "^1.0.30001759",
@@ -4569,7 +4564,6 @@
       "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.8.0",
         "@eslint-community/regexpp": "^4.12.1",
@@ -6810,7 +6804,6 @@
       "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@jest/core": "30.2.0",
         "@jest/types": "30.2.0",
@@ -7478,7 +7471,6 @@
       "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.5.tgz",
       "integrity": "sha512-FA5LmZVF1VziNc0bIdCSA1IoSVnDCqE8HJIZZv2/W8YmoAM50+tnUgJR/gQZwEeIMleuIOnRnHA/UaZRNeV4iQ==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@keyv/serialize": "^1.1.1"
       }
@@ -10648,7 +10640,6 @@
       "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@cspotcode/source-map-support": "^0.8.0",
         "@tsconfig/node10": "^1.0.7",
@@ -10881,7 +10872,6 @@
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"

package.json

@@ -60,6 +60,7 @@
     "rot13-cipher": "^1.0.0",
     "slugify": "^1.6.6",
     "socks-proxy-agent": "^8.0.5",
+    "sqlite3": "^5.1.7",
     "tough-cookie": "^6.0.0",
     "unpacker": "^1.0.1",
     "winston": "^3.17.0"

src/extractor/ExtractorRegistry.ts

@@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite';
 import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
 import winston from 'winston';
 import { Context, Format, Meta, UrlResult } from '../types';
-import { envGet, getCacheDir, isExtractorDisabled } from '../utils';
+import { envGet, getCacheDir, isExtractorDisabled, scheduleKeyvSqliteCleanup } from '../utils';
 import { Extractor } from './Extractor';
 
 export class ExtractorRegistry {
@@ -17,18 +17,23 @@ export class ExtractorRegistry {
     this.logger = logger;
     this.extractors = extractors;
 
+    const urlResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-cache.sqlite`);
     this.urlResultCache = new Cacheable({
       nonBlocking: true,
       primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }),
-      secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-cache.sqlite`)),
+      secondary: new Keyv(urlResultKeyvSqlite),
       stats: true,
     });
+    scheduleKeyvSqliteCleanup(urlResultKeyvSqlite);
+
+    const lazyUrlResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-lazy-cache.sqlite`);
     this.lazyUrlResultCache = new Cacheable({
       nonBlocking: true,
       primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }),
-      secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-lazy-cache.sqlite`)),
+      secondary: new Keyv(lazyUrlResultKeyvSqlite),
       stats: true,
     });
+    scheduleKeyvSqliteCleanup(lazyUrlResultKeyvSqlite);
   }
 
   public stats() {

src/source/Source.ts

@@ -4,13 +4,22 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
 import { ContentType } from 'stremio-addon-sdk';
 import { NotFoundError } from '../error';
 import { Context, CountryCode, Meta } from '../types';
-import { getCacheDir, Id } from '../utils';
+import { getCacheDir, Id, scheduleKeyvSqliteCleanup } from '../utils';
 
 export interface SourceResult {
   url: URL;
   meta: Meta;
 }
 
+const sourceResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-source-cache-v2.sqlite`);
+const sourceResultCache = new Cacheable({
+  nonBlocking: true,
+  primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }),
+  secondary: new Keyv(sourceResultKeyvSqlite),
+  stats: true,
+});
+scheduleKeyvSqliteCleanup(sourceResultKeyvSqlite);
+
 export abstract class Source {
   public abstract readonly id: string;
 
@@ -24,25 +33,18 @@ export abstract class Source {
 
   public abstract readonly baseUrl: string;
 
-  private static readonly sourceResultCache = new Cacheable({
-    nonBlocking: true,
-    primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }),
-    secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-source-cache-v2.sqlite`)),
-    stats: true,
-  });
-
   protected abstract handleInternal(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])>;
 
   public static stats() {
     return {
-      sourceResultCache: Source.sourceResultCache.stats,
+      sourceResultCache: sourceResultCache.stats,
     };
   };
 
   public async handle(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])> {
     const cacheKey = `${this.id}_${id.toString()}`;
 
-    let sourceResults = (await Source.sourceResultCache.get<SourceResult[]>(cacheKey))
+    let sourceResults = (await sourceResultCache.get<SourceResult[]>(cacheKey))
       ?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) }));
 
     if (!sourceResults) {
@@ -56,7 +58,7 @@ export abstract class Source {
         }
       }
 
-      await Source.sourceResultCache.set<SourceResult[]>(cacheKey, sourceResults, this.ttl);
+      await sourceResultCache.set<SourceResult[]>(cacheKey, sourceResults, this.ttl);
     }
 
     if (this.countryCodes.includes(CountryCode.multi)) {

src/utils/cache.ts

@@ -0,0 +1,22 @@
+/* istanbul ignore file */
+import fs from 'node:fs';
+// eslint-disable-next-line import/no-named-as-default
+import KeyvSqlite from '@keyv/sqlite';
+import * as sqlite3 from 'sqlite3';
+import { envIsTest } from './env';
+
+export const scheduleKeyvSqliteCleanup = (keyvSqlite: KeyvSqlite): void => {
+  const filename = keyvSqlite.opts.db;
+  if (envIsTest() || !filename || !fs.existsSync(filename)) {
+    return;
+  }
+
+  setInterval(() => {
+    const db = new sqlite3.Database(filename);
+
+    db.serialize(() => {
+      db.run('DELETE FROM keyv WHERE json_extract(value, \'$.expires\') <= (strftime(\'%s\', \'now\') * 1000)');
+    });
+    db.close();
+  }, 60 * 60 * 1000); // every hour
+};

src/utils/env.ts

@@ -18,6 +18,8 @@ export const envGetAppName = (): string => process.env['MANIFEST_NAME'] || 'WebS
 
 export const envIsProd = (): boolean => process.env['NODE_ENV'] === 'production';
 
+export const envIsTest = (): boolean => process.env['NODE_ENV'] === 'test';
+
 export const isElfHostedInstance = (req: Request): boolean => req.host.endsWith('elfhosted.com');
 
 export const getCacheDir = (): string => envGet('CACHE_DIR') ?? os.tmpdir();

src/utils/index.ts

@@ -1,6 +1,7 @@
 export * from './Fetcher';
 export * from './FetcherMock';
 export * from './StreamResolver';
+export * from './cache';
 export * from './config';
 export * from './context';
 export * from './embed';