Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

perf: cache all non-error requests for 15m, tune axios caching

Browse source
This commit is contained in:
WebStreamr 2026-02-23 11:57:07 +00:00
parent 964e979469
commit ae10810caa
No known key found for this signature in database
2 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/index.ts
View file

@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto';
import axios from 'axios';
import { setupCache } from 'axios-cache-interceptor';
import { buildMemoryStorage, setupCache } from 'axios-cache-interceptor';
import axiosRetry from 'axios-retry';
import express, { NextFunction, Request, Response } from 'express';
// eslint-disable-next-line import/no-named-as-default
@ -39,7 +39,11 @@ process.on('unhandledRejection', (error: Error) => {
logger.error(`Unhandled rejection: ${error}, cause: ${error.cause}, stack: ${error.stack}`);
});
const cachedAxios = setupCache(axios);
const cachedAxios = setupCache(axios, {
interpretHeader: true,
storage: buildMemoryStorage(false, 3 * 60 * 60 * 1000, 4096, 12 * 60 * 60 * 1000),
ttl: 15 * 60 * 1000, // 15m
});
axiosRetry(cachedAxios, { retries: 3, retryDelay: () => 333 });
const fetcher = new Fetcher(cachedAxios, logger);

11
src/utils/Fetcher.ts
View file

@ -54,7 +54,8 @@ export class Fetcher {
private readonly DEFAULT_QUEUE_LIMIT = 50;
private readonly DEFAULT_QUEUE_TIMEOUT = 10000;
private readonly DEFAULT_TIMEOUTS_COUNT_THROW = 30;
private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h
private readonly TIMEOUT_CACHE_TTL = 60 * 60 * 1000; // 1h
private readonly FLARESOLVERR_CACHE_TTL = 15 * 60 * 1000; // 15m
private readonly MAX_WAIT_RETRY_AFTER = 10000;
private readonly axios: AxiosInstance;
@ -73,6 +74,7 @@ export class Fetcher {
private readonly httpStatus = new Map<string, Record<number, number>>();
private readonly httpStatusMutex = new Mutex();
private readonly flareSolverrCache = new Cacheable({ primary: new Keyv({ store: new CacheableMemory({ lruSize: 1024 }) }) });
private readonly flareSolverrMutexes = new Map<string, Mutex>();
public constructor(axios: AxiosInstance, logger: winston.Logger) {
@ -244,6 +246,12 @@ export class Fetcher {
throw new BlockedError(url, BlockedReason.cloudflare_challenge, response.headers);
}
const cachedResponseData = await this.flareSolverrCache.get<string>(url.href);
if (cachedResponseData) {
response.data = cachedResponseData;
return response;
}
const session = `${envGetAppId()}_${url.host}`;
let mutex = this.flareSolverrMutexes.get(session);
@ -297,6 +305,7 @@ export class Fetcher {
this.hostUserAgentMap.set(url.host, challengeResult.solution.userAgent);
response.data = challengeResult.solution.response;
await this.flareSolverrCache.set<string>(url.href, response.data, this.FLARESOLVERR_CACHE_TTL);
return response;
}