From 51aa127a33f3a3b5b5cfc738e00cd9f682394ea8 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Wed, 14 Jan 2026 12:58:59 +0000 Subject: [PATCH] chore: regularly clean-up expired items from sqlite cache --- package-lock.json | 12 +----------- package.json | 1 + src/extractor/ExtractorRegistry.ts | 11 ++++++++--- src/source/Source.ts | 24 +++++++++++++----------- src/utils/cache.ts | 22 ++++++++++++++++++++++ src/utils/env.ts | 2 ++ src/utils/index.ts | 1 + 7 files changed, 48 insertions(+), 25 deletions(-) create mode 100644 src/utils/cache.ts diff --git a/package-lock.json b/package-lock.json index 322a816..f12ab8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "rot13-cipher": "^1.0.0", "slugify": "^1.6.6", "socks-proxy-agent": "^8.0.5", + "sqlite3": "^5.1.7", "tough-cookie": "^6.0.0", "unpacker": "^1.0.1", "winston": "^3.17.0" @@ -89,7 +90,6 @@ "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/generator": "^7.28.6", @@ -1811,7 +1811,6 @@ "integrity": "sha512-+054pVMzVTmRQV8BhpGv3UyfZ2Llgl8rdpDTon+cUH9+na0ncBVXj3wTUKh14+Kiz18ziM3b4ikpP5/Pc0rQEQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1940,7 +1939,6 @@ "integrity": "sha512-npiaib8XzbjtzS2N4HlqPvlpxpmZ14FjSJrteZpPxGUaYPlvhzlzUZ4mZyABo0EFrOWnvyd0Xxroq//hKhtAWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.53.0", "@typescript-eslint/types": "8.53.0", @@ -2480,7 +2478,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2849,7 +2846,6 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", "license": "MIT", - "peer": true, "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.4", @@ -3162,7 +3158,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -4569,7 +4564,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6810,7 +6804,6 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -7478,7 +7471,6 @@ "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.5.tgz", "integrity": "sha512-FA5LmZVF1VziNc0bIdCSA1IoSVnDCqE8HJIZZv2/W8YmoAM50+tnUgJR/gQZwEeIMleuIOnRnHA/UaZRNeV4iQ==", "license": "MIT", - "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -10648,7 +10640,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -10881,7 +10872,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 2c7967d..2fd37ab 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "rot13-cipher": "^1.0.0", "slugify": "^1.6.6", "socks-proxy-agent": "^8.0.5", + "sqlite3": "^5.1.7", "tough-cookie": "^6.0.0", "unpacker": "^1.0.1", "winston": "^3.17.0" diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index bd0772e..856aa6c 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite'; import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import winston from 'winston'; import { Context, Format, Meta, UrlResult } from '../types'; -import { envGet, getCacheDir, isExtractorDisabled } from '../utils'; +import { envGet, getCacheDir, isExtractorDisabled, scheduleKeyvSqliteCleanup } from '../utils'; import { Extractor } from './Extractor'; export class ExtractorRegistry { @@ -17,18 +17,23 @@ export class ExtractorRegistry { this.logger = logger; this.extractors = extractors; + const urlResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-cache.sqlite`); this.urlResultCache = new Cacheable({ nonBlocking: true, primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }), - secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-cache.sqlite`)), + secondary: new Keyv(urlResultKeyvSqlite), stats: true, }); + scheduleKeyvSqliteCleanup(urlResultKeyvSqlite); + + const lazyUrlResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-lazy-cache.sqlite`); this.lazyUrlResultCache = new Cacheable({ nonBlocking: true, primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }), - secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-lazy-cache.sqlite`)), + secondary: new Keyv(lazyUrlResultKeyvSqlite), stats: true, }); + scheduleKeyvSqliteCleanup(lazyUrlResultKeyvSqlite); } public stats() { diff --git a/src/source/Source.ts b/src/source/Source.ts index a421293..86364aa 100644 --- a/src/source/Source.ts +++ b/src/source/Source.ts @@ -4,13 +4,22 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import { ContentType } from 'stremio-addon-sdk'; import { NotFoundError } from '../error'; import { Context, CountryCode, Meta } from '../types'; -import { getCacheDir, Id } from '../utils'; +import { getCacheDir, Id, scheduleKeyvSqliteCleanup } from '../utils'; export interface SourceResult { url: URL; meta: Meta; } +const sourceResultKeyvSqlite = new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-source-cache-v2.sqlite`); +const sourceResultCache = new Cacheable({ + nonBlocking: true, + primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }), + secondary: new Keyv(sourceResultKeyvSqlite), + stats: true, +}); +scheduleKeyvSqliteCleanup(sourceResultKeyvSqlite); + export abstract class Source { public abstract readonly id: string; @@ -24,25 +33,18 @@ export abstract class Source { public abstract readonly baseUrl: string; - private static readonly sourceResultCache = new Cacheable({ - nonBlocking: true, - primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }), - secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-source-cache-v2.sqlite`)), - stats: true, - }); - protected abstract handleInternal(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])>; public static stats() { return { - sourceResultCache: Source.sourceResultCache.stats, + sourceResultCache: sourceResultCache.stats, }; }; public async handle(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])> { const cacheKey = `${this.id}_${id.toString()}`; - let sourceResults = (await Source.sourceResultCache.get(cacheKey)) + let sourceResults = (await sourceResultCache.get(cacheKey)) ?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) })); if (!sourceResults) { @@ -56,7 +58,7 @@ export abstract class Source { } } - await Source.sourceResultCache.set(cacheKey, sourceResults, this.ttl); + await sourceResultCache.set(cacheKey, sourceResults, this.ttl); } if (this.countryCodes.includes(CountryCode.multi)) { diff --git a/src/utils/cache.ts b/src/utils/cache.ts new file mode 100644 index 0000000..05a91ad --- /dev/null +++ b/src/utils/cache.ts @@ -0,0 +1,22 @@ +/* istanbul ignore file */ +import fs from 'node:fs'; +// eslint-disable-next-line import/no-named-as-default +import KeyvSqlite from '@keyv/sqlite'; +import * as sqlite3 from 'sqlite3'; +import { envIsTest } from './env'; + +export const scheduleKeyvSqliteCleanup = (keyvSqlite: KeyvSqlite): void => { + const filename = keyvSqlite.opts.db; + if (envIsTest() || !filename || !fs.existsSync(filename)) { + return; + } + + setInterval(() => { + const db = new sqlite3.Database(filename); + + db.serialize(() => { + db.run('DELETE FROM keyv WHERE json_extract(value, \'$.expires\') <= (strftime(\'%s\', \'now\') * 1000)'); + }); + db.close(); + }, 60 * 60 * 1000); // every hour +}; diff --git a/src/utils/env.ts b/src/utils/env.ts index aa145cb..9259d7f 100644 --- a/src/utils/env.ts +++ b/src/utils/env.ts @@ -18,6 +18,8 @@ export const envGetAppName = (): string => process.env['MANIFEST_NAME'] || 'WebS export const envIsProd = (): boolean => process.env['NODE_ENV'] === 'production'; +export const envIsTest = (): boolean => process.env['NODE_ENV'] === 'test'; + export const isElfHostedInstance = (req: Request): boolean => req.host.endsWith('elfhosted.com'); export const getCacheDir = (): string => envGet('CACHE_DIR') ?? os.tmpdir(); diff --git a/src/utils/index.ts b/src/utils/index.ts index 23be2d8..35dec8b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,7 @@ export * from './Fetcher'; export * from './FetcherMock'; export * from './StreamResolver'; +export * from './cache'; export * from './config'; export * from './context'; export * from './embed';