feat: cache extractor results for 15 minutes

package-lock.json

@@ -9,10 +9,10 @@
       "version": "0.5.0",
       "license": "MIT",
       "dependencies": {
+        "@isaacs/ttlcache": "^1.4.1",
         "bytes": "^3.1.2",
         "cheerio": "^1.0.0",
         "express": "^5.1.0",
-        "lru-cache": "^11.1.0",
         "make-fetch-happen": "^14.0.3",
         "slugify": "^1.6.6",
         "unpacker": "^1.0.1",
@@ -945,6 +945,15 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@isaacs/ttlcache": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-1.4.1.tgz",
+      "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@istanbuljs/load-nyc-config": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@@ -5412,15 +5421,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/lru-cache": {
-      "version": "11.1.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz",
-      "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==",
-      "license": "ISC",
-      "engines": {
-        "node": "20 || >=22"
-      }
-    },
     "node_modules/make-dir": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz",

package.json

@@ -30,10 +30,10 @@
     "node": "^22.0.0"
   },
   "dependencies": {
+    "@isaacs/ttlcache": "^1.4.1",
     "bytes": "^3.1.2",
     "cheerio": "^1.0.0",
     "express": "^5.1.0",
-    "lru-cache": "^11.1.0",
     "make-fetch-happen": "^14.0.3",
     "slugify": "^1.6.6",
     "unpacker": "^1.0.1",

src/embed-extractor/EmbedExtractors.test.ts

@@ -1,5 +1,8 @@
 import { EmbedExtractors } from './EmbedExtractors';
 import { Context } from '../types';
+import { Fetcher } from '../utils';
+import { Dropload } from './Dropload';
+jest.mock('../utils/Fetcher');
 
 describe('EmbedExtractors', () => {
   const ctx: Context = { ip: '127.0.0.1' };
@@ -10,4 +13,15 @@ describe('EmbedExtractors', () => {
     expect(embedExtractors.handle(ctx, new URL('https://some-url.test'), 'en'))
       .rejects.toThrow('No embed extractor found that supports url https://some-url.test');
   });
+
+  test('returns from memory cache if possible', async () => {
+    // @ts-expect-error No constructor args needed
+    const fetcher = new Fetcher();
+    const embedExtractors = new EmbedExtractors([new Dropload(fetcher)]);
+
+    const urlResult1 = await embedExtractors.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');
+    const urlResult2 = await embedExtractors.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');
+
+    expect(urlResult2).toBe(urlResult1);
+  });
 });

src/embed-extractor/EmbedExtractors.ts

@@ -1,20 +1,31 @@
+import TTLCache from '@isaacs/ttlcache';
 import { EmbedExtractor } from './types';
 import { Context, UrlResult } from '../types';
 
 export class EmbedExtractors {
   private readonly embedExtractors: EmbedExtractor[];
 
+  private readonly cache: TTLCache<string, UrlResult>;
+
   constructor(embedExtractors: EmbedExtractor[]) {
     this.embedExtractors = embedExtractors;
+    this.cache = new TTLCache({ max: 1024, ttl: 900000 }); // 15m
   }
 
   readonly handle = async (ctx: Context, url: URL, language: string): Promise<UrlResult> => {
-    const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url));
+    let urlResult = this.cache.get(url.href);
+    if (urlResult) {
+      return urlResult;
+    }
 
+    const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url));
     if (undefined === embedExtractor) {
       throw new Error(`No embed extractor found that supports url ${url}`);
     }
 
-    return embedExtractor.extract(ctx, url, language);
+    urlResult = await embedExtractor.extract(ctx, url, language);
+    this.cache.set(url.href, urlResult);
+
+    return urlResult;
   };
 }

src/utils/Fetcher.ts

@@ -1,17 +1,17 @@
 import { FetchInterface, FetchOptions } from 'make-fetch-happen';
+import TTLCache from '@isaacs/ttlcache';
 import UserAgent from 'user-agents';
-import { LRUCache } from 'lru-cache';
 import { logInfo } from './log';
 import { Context } from '../types';
 
 export class Fetcher {
   private readonly fetch: FetchInterface;
 
-  private readonly cache: LRUCache<string, string>;
+  private readonly cache: TTLCache<string, string>;
 
   constructor(fetch: FetchInterface) {
     this.fetch = fetch;
-    this.cache = new LRUCache({ max: 128 });
+    this.cache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h
   }
 
   readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise<string> => {