Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore(fetcher): use LRUCache for HTTP responses

Browse source
This commit is contained in:
WebStreamr 2025-06-28 17:18:10 +00:00
parent 881fed53a4
commit f5121bb1fb
No known key found for this signature in database
3 changed files with 19 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
package-lock.json generated
View file

@ -16,6 +16,7 @@
"express": "^5.1.0",
"fetch-socks": "^1.3.2",
"http-cache-semantics": "^4.2.0",
"lru-cache": "^11.1.0",
"randomstring": "^1.3.1",
"slugify": "^1.6.6",
"tough-cookie": "^5.1.2",
@ -154,6 +155,16 @@
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dev": true,
"license": "ISC",
"dependencies": {
"yallist": "^3.0.2"
}
},
"node_modules/@babel/helper-module-imports": {
"version": "7.27.1",
"resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz",
@ -6553,13 +6564,12 @@
}
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dev": true,
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz",
"integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==",
"license": "ISC",
"dependencies": {
"yallist": "^3.0.2"
"engines": {
"node": "20 || >=22"
}
},
"node_modules/make-dir": {

1
package.json
View file

@ -38,6 +38,7 @@
"express": "^5.1.0",
"fetch-socks": "^1.3.2",
"http-cache-semantics": "^4.2.0",
"lru-cache": "^11.1.0",
"randomstring": "^1.3.1",
"slugify": "^1.6.6",
"tough-cookie": "^5.1.2",

3
src/utils/Fetcher.ts
View file

@ -1,6 +1,7 @@
import { Dispatcher, ProxyAgent } from 'undici';
import { socksDispatcher } from 'fetch-socks';
import CachePolicy from 'http-cache-semantics';
import { LRUCache } from 'lru-cache';
import TTLCache from '@isaacs/ttlcache';
import winston from 'winston';
import { Mutex, Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
@ -61,7 +62,7 @@ export class Fetcher {
private readonly logger: winston.Logger;
private readonly dispatcher: Dispatcher | undefined;
private readonly httpCache = new TTLCache<string, HttpCacheItem>({ max: 1024 });
private readonly httpCache = new LRUCache<string, HttpCacheItem>({ max: 1024 });
private readonly rateLimitedCache = new TTLCache<string, undefined>({ max: 1024 });
private readonly semaphores = new Map<string, SemaphoreInterface>();
private readonly hostUserAgentMap = new Map<string, string>();