Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

perf(fetcher): simplify FlareSolverr integration by just using a request per host

Browse source
This commit is contained in:
WebStreamr 2026-02-03 09:56:45 +00:00
parent 92c45d553e
commit 31da8aa92c
No known key found for this signature in database
2 changed files with 24 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -78,10 +78,6 @@ Optional. Comma separated list of sources which should be disabled. E.g. `frembe
Optional. If domains show Cloudflare challenges, FlareSolverr can be used to work around them. E.g. `http://flaresolverr:8191`
Proxy configuration is passed-through and only a single session is used to save resources. Byparr is not supported.
#### `FLARESOLVERR_SESSIONS`
Optional. Configure how many FlareSolverr sessions can be open at the same time. Default: `5`
#### `MANIFEST_ID`
Optional. Add-on manifest ID. Default: `webstreamr`

38
src/utils/Fetcher.ts
View file

@ -73,6 +73,8 @@ export class Fetcher {
private readonly httpStatus = new Map<string, Record<number, number>>();
private readonly httpStatusMutex = new Mutex();
private readonly flareSolverrMutexes = new Map<string, Mutex>();
public constructor(axios: AxiosInstance, logger: winston.Logger) {
this.axios = axios;
this.logger = logger;
@ -242,22 +244,30 @@ export class Fetcher {
throw new BlockedError(url, BlockedReason.cloudflare_challenge, response.headers);
}
this.logger.info(`Query FlareSolverr for ${url.href}`, ctx);
const session = `${envGetAppId()}_${url.host}`;
const flareSolverrSessions = parseInt(envGet('FLARESOLVERR_SESSIONS') ?? '5');
const data = {
cmd: 'request.get',
url: url.href,
session: `${envGetAppId()}_${Math.floor(Math.random() * flareSolverrSessions)}`,
session_ttl_minutes: 60,
maxTimeout: 15000,
cookies: (await this.cookieJar.getCookies(url.href)).map(cookie => ({ name: cookie.key, value: cookie.value })),
disableMedia: true,
...(proxyUrl && { proxy: { url: proxyUrl.href } }),
};
let mutex = this.flareSolverrMutexes.get(session);
if (!mutex) {
mutex = new Mutex();
this.flareSolverrMutexes.set(session, mutex);
}
const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000, queueLimit: flareSolverrSessions };
const challengeResult = JSON.parse((await this.queuedFetch(ctx, new URL('/v1', flareSolverrEndpoint), requestConfig)).data) as FlareSolverrResult;
const challengeResult = await mutex.runExclusive(async () => {
this.logger.info(`Query FlareSolverr for ${url.href}`, ctx);
const data = {
cmd: 'request.get',
url: url.href,
session,
session_ttl_minutes: 60,
maxTimeout: 15000,
disableMedia: true,
...(proxyUrl && { proxy: { url: proxyUrl.href } }),
};
const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000 };
return JSON.parse((await this.queuedFetch(ctx, new URL('/v1', flareSolverrEndpoint), requestConfig)).data) as FlareSolverrResult;
});
if (challengeResult.status !== 'ok') {
this.logger.warn(`FlareSolverr issue: ${JSON.stringify(challengeResult)}`, ctx);