From f5121bb1fbf5914d1ca9a9e17e3726c69d0d1e2d Mon Sep 17 00:00:00 2001
From: WebStreamr <210764791+webstreamr@users.noreply.github.com>
Date: Sat, 28 Jun 2025 17:18:10 +0000
Subject: [PATCH] chore(fetcher): use LRUCache for HTTP responses

---
 package-lock.json    | 22 ++++++++++++++++------
 package.json         |  1 +
 src/utils/Fetcher.ts |  3 ++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 8469644..899a92a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -16,6 +16,7 @@
         "express": "^5.1.0",
         "fetch-socks": "^1.3.2",
         "http-cache-semantics": "^4.2.0",
+        "lru-cache": "^11.1.0",
         "randomstring": "^1.3.1",
         "slugify": "^1.6.6",
         "tough-cookie": "^5.1.2",
@@ -154,6 +155,16 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "yallist": "^3.0.2"
+      }
+    },
     "node_modules/@babel/helper-module-imports": {
       "version": "7.27.1",
       "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz",
@@ -6553,13 +6564,12 @@
       }
     },
     "node_modules/lru-cache": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-      "dev": true,
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz",
+      "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==",
       "license": "ISC",
-      "dependencies": {
-        "yallist": "^3.0.2"
+      "engines": {
+        "node": "20 || >=22"
       }
     },
     "node_modules/make-dir": {
diff --git a/package.json b/package.json
index 3a61fe7..d96b820 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
     "express": "^5.1.0",
     "fetch-socks": "^1.3.2",
     "http-cache-semantics": "^4.2.0",
+    "lru-cache": "^11.1.0",
     "randomstring": "^1.3.1",
     "slugify": "^1.6.6",
     "tough-cookie": "^5.1.2",
diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts
index 5d506d1..bdef40a 100644
--- a/src/utils/Fetcher.ts
+++ b/src/utils/Fetcher.ts
@@ -1,6 +1,7 @@
 import { Dispatcher, ProxyAgent } from 'undici';
 import { socksDispatcher } from 'fetch-socks';
 import CachePolicy from 'http-cache-semantics';
+import { LRUCache } from 'lru-cache';
 import TTLCache from '@isaacs/ttlcache';
 import winston from 'winston';
 import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
@@ -61,7 +62,7 @@ export class Fetcher {
   private readonly logger: winston.Logger;
   private readonly dispatcher: Dispatcher | undefined;
 
-  private readonly httpCache = new TTLCache<string, HttpCacheItem>({ max: 1024 });
+  private readonly httpCache = new LRUCache<string, HttpCacheItem>({ max: 1024 });
   private readonly rateLimitedCache = new TTLCache<string, undefined>({ max: 1024 });
   private readonly semaphores = new Map<string, SemaphoreInterface>();
   private readonly hostUserAgentMap = new Map<string, string>();