From 8226d0755a559573cee1e49f27b26f29642d9f00 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sun, 11 May 2025 21:31:43 +0000 Subject: [PATCH] feat: cache extractor results for 15 minutes --- package-lock.json | 20 ++++++++++---------- package.json | 2 +- src/embed-extractor/EmbedExtractors.test.ts | 14 ++++++++++++++ src/embed-extractor/EmbedExtractors.ts | 15 +++++++++++++-- src/utils/Fetcher.ts | 6 +++--- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index ed57450..c1fd045 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,10 @@ "version": "0.5.0", "license": "MIT", "dependencies": { + "@isaacs/ttlcache": "^1.4.1", "bytes": "^3.1.2", "cheerio": "^1.0.0", "express": "^5.1.0", - "lru-cache": "^11.1.0", "make-fetch-happen": "^14.0.3", "slugify": "^1.6.6", "unpacker": "^1.0.1", @@ -945,6 +945,15 @@ "node": ">=18.0.0" } }, + "node_modules/@isaacs/ttlcache": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-1.4.1.tgz", + "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -5412,15 +5421,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lru-cache": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", - "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", diff --git a/package.json b/package.json index 86d6764..f30305a 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,10 @@ "node": "^22.0.0" }, "dependencies": { + "@isaacs/ttlcache": "^1.4.1", "bytes": "^3.1.2", "cheerio": "^1.0.0", "express": "^5.1.0", - "lru-cache": "^11.1.0", "make-fetch-happen": "^14.0.3", "slugify": "^1.6.6", "unpacker": "^1.0.1", diff --git a/src/embed-extractor/EmbedExtractors.test.ts b/src/embed-extractor/EmbedExtractors.test.ts index c78e567..502c169 100644 --- a/src/embed-extractor/EmbedExtractors.test.ts +++ b/src/embed-extractor/EmbedExtractors.test.ts @@ -1,5 +1,8 @@ import { EmbedExtractors } from './EmbedExtractors'; import { Context } from '../types'; +import { Fetcher } from '../utils'; +import { Dropload } from './Dropload'; +jest.mock('../utils/Fetcher'); describe('EmbedExtractors', () => { const ctx: Context = { ip: '127.0.0.1' }; @@ -10,4 +13,15 @@ describe('EmbedExtractors', () => { expect(embedExtractors.handle(ctx, new URL('https://some-url.test'), 'en')) .rejects.toThrow('No embed extractor found that supports url https://some-url.test'); }); + + test('returns from memory cache if possible', async () => { + // @ts-expect-error No constructor args needed + const fetcher = new Fetcher(); + const embedExtractors = new EmbedExtractors([new Dropload(fetcher)]); + + const urlResult1 = await embedExtractors.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); + const urlResult2 = await embedExtractors.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); + + expect(urlResult2).toBe(urlResult1); + }); }); diff --git a/src/embed-extractor/EmbedExtractors.ts b/src/embed-extractor/EmbedExtractors.ts index 3989186..7ccfa18 100644 --- a/src/embed-extractor/EmbedExtractors.ts +++ b/src/embed-extractor/EmbedExtractors.ts @@ -1,20 +1,31 @@ +import TTLCache from '@isaacs/ttlcache'; import { EmbedExtractor } from './types'; import { Context, UrlResult } from '../types'; export class EmbedExtractors { private readonly embedExtractors: EmbedExtractor[]; + private readonly cache: TTLCache; + constructor(embedExtractors: EmbedExtractor[]) { this.embedExtractors = embedExtractors; + this.cache = new TTLCache({ max: 1024, ttl: 900000 }); // 15m } readonly handle = async (ctx: Context, url: URL, language: string): Promise => { - const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url)); + let urlResult = this.cache.get(url.href); + if (urlResult) { + return urlResult; + } + const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url)); if (undefined === embedExtractor) { throw new Error(`No embed extractor found that supports url ${url}`); } - return embedExtractor.extract(ctx, url, language); + urlResult = await embedExtractor.extract(ctx, url, language); + this.cache.set(url.href, urlResult); + + return urlResult; }; } diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index c39df23..9c47242 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,17 +1,17 @@ import { FetchInterface, FetchOptions } from 'make-fetch-happen'; +import TTLCache from '@isaacs/ttlcache'; import UserAgent from 'user-agents'; -import { LRUCache } from 'lru-cache'; import { logInfo } from './log'; import { Context } from '../types'; export class Fetcher { private readonly fetch: FetchInterface; - private readonly cache: LRUCache; + private readonly cache: TTLCache; constructor(fetch: FetchInterface) { this.fetch = fetch; - this.cache = new LRUCache({ max: 128 }); + this.cache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h } readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise => {