feat: configure sqlite as secondary source and extractor cache

jest.setup.ts

@@ -1,6 +1,8 @@
 import { socksDispatcher } from 'fetch-socks';
 import { Agent, Dispatcher, interceptors, ProxyAgent, setGlobalDispatcher } from 'undici';
 
+process.env['CACHE_DIR'] = '/dev/null';
+
 jest.mock('randomstring', () => ({
   generate: jest.fn(() => 'mocked-random-string'),
 }));

package.json

@@ -31,6 +31,7 @@
     "node": "^22.0.0"
   },
   "dependencies": {
+    "@keyv/sqlite": "^4.0.5",
     "@types/jsdom": "^21.1.7",
     "async-mutex": "^0.5.0",
     "bytes": "^3.1.2",

src/extractor/ExtractorRegistry.ts

@@ -1,7 +1,9 @@
+// eslint-disable-next-line import/no-named-as-default
+import KeyvSqlite from '@keyv/sqlite';
 import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
 import winston from 'winston';
 import { Context, CountryCode, UrlResult } from '../types';
-import { isExtractorDisabled } from '../utils';
+import { getCacheDir, isExtractorDisabled } from '../utils';
 import { Extractor } from './Extractor';
 
 export class ExtractorRegistry {
@@ -13,7 +15,10 @@ export class ExtractorRegistry {
     this.logger = logger;
     this.extractors = extractors;
 
-    this.urlResultCache = new Cacheable({ primary: new Keyv({ store: new CacheableMemory({ lruSize: 4096 }) }) });
+    this.urlResultCache = new Cacheable({
+      primary: new Keyv({ store: new CacheableMemory({ lruSize: 4096 }) }),
+      secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-extractor-cache.sqlite`)),
+    });
   }
 
   public async handle(ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined): Promise<UrlResult[]> {

src/source/Source.ts

@@ -1,8 +1,10 @@
+// eslint-disable-next-line import/no-named-as-default
+import KeyvSqlite from '@keyv/sqlite';
 import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
 import { ContentType } from 'stremio-addon-sdk';
 import { NotFoundError } from '../error';
 import { Context, CountryCode } from '../types';
-import { Id } from '../utils';
+import { getCacheDir, Id } from '../utils';
 
 export interface SourceResult {
   countryCode: CountryCode;
@@ -26,13 +28,16 @@ export abstract class Source {
   private readonly sourceResultCache: Cacheable;
 
   public constructor() {
-    this.sourceResultCache = new Cacheable({ primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024, ttl: this.ttl }) }) });
+    this.sourceResultCache = new Cacheable({
+      primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024, ttl: this.ttl }) }),
+      secondary: new Keyv(new KeyvSqlite(`sqlite://${getCacheDir()}/webstreamr-source-cache.sqlite`)),
+    });
   }
 
   protected abstract handleInternal(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])>;
 
   public async handle(ctx: Context, type: ContentType, id: Id): Promise<(SourceResult[])> {
-    const cacheKey = id.toString();
+    const cacheKey = `${this.id}_${id.toString()}`;
 
     let sourceResults = await this.sourceResultCache.get<SourceResult[]>(cacheKey);
     if (sourceResults) {

src/utils/env.test.ts

@@ -1,5 +1,6 @@
+import * as os from 'node:os';
 import { Request } from 'express';
-import { envGet, envGetAppId, envGetAppName, envIsProd, isElfHostedInstance } from './env';
+import { envGet, envGetAppId, envGetAppName, envIsProd, getCacheDir, isElfHostedInstance } from './env';
 
 describe('env', () => {
   test('envGet', () => {
@@ -35,4 +36,13 @@ describe('env', () => {
     expect(isElfHostedInstance({ host: 'someuser.elfhosted.com' } as Request)).toBeTruthy();
     expect(isElfHostedInstance({ host: 'webstreamr.hayd.uk' } as Request)).toBeFalsy();
   });
+
+  test('getCacheDir', () => {
+    const previousCacheDir = process.env['CACHE_DIR'];
+    delete process.env['CACHE_DIR'];
+    expect(getCacheDir()).toBe(os.tmpdir());
+    process.env['CACHE_DIR'] = previousCacheDir;
+
+    expect(getCacheDir()).toBe('/dev/null');
+  });
 });

src/utils/env.ts

@@ -1,3 +1,4 @@
+import * as os from 'node:os';
 import { Request } from 'express';
 
 export const envGet = (name: string): string | undefined => process.env[name];
@@ -9,3 +10,5 @@ export const envGetAppName = (): string => process.env['MANIFEST_NAME'] || 'WebS
 export const envIsProd = (): boolean => process.env['NODE_ENV'] === 'production';
 
 export const isElfHostedInstance = (req: Request): boolean => req.host.endsWith('elfhosted.com');
+
+export const getCacheDir = (): string => envGet('CACHE_DIR') ?? os.tmpdir();