From ae5829dcb0f8064e939c06d6457f891b1c7051c8 Mon Sep 17 00:00:00 2001
From: WebStreamr <210764791+webstreamr@users.noreply.github.com>
Date: Mon, 2 Feb 2026 20:05:18 +0000
Subject: [PATCH] perf(fetcher): improve FlareSolverr integration

- multiple sessions which are destroyed after being idle for 60 minutes
- fix timeout handling
- pass cookies to share between sessions
- disable media
---
 README.md            |  4 ++++
 src/utils/Fetcher.ts | 12 ++++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7320d35..989de95 100644
--- a/README.md
+++ b/README.md
@@ -78,6 +78,10 @@ Optional. Comma separated list of sources which should be disabled. E.g. `frembe
 Optional. If domains show Cloudflare challenges, FlareSolverr can be used to work around them. E.g. `http://flaresolverr:8191`
 Proxy configuration is passed-through and only a single session is used to save resources. Byparr is not supported.
 
+#### `FLARESOLVERR_SESSIONS`
+
+Optional. Configure how many FlareSolverr sessions can be open at the same time. Default: `5`
+
 #### `MANIFEST_ID`
 
 Optional. Add-on manifest ID. Default: `webstreamr`
diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts
index c23aff2..5d086dd 100644
--- a/src/utils/Fetcher.ts
+++ b/src/utils/Fetcher.ts
@@ -11,7 +11,7 @@ import { Cookie, CookieJar } from 'tough-cookie';
 import winston from 'winston';
 import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
 import { BlockedReason, Context } from '../types';
-import { envGet } from './env';
+import { envGet, envGetAppId } from './env';
 
 interface FlareSolverrResult {
   status: string;
@@ -194,7 +194,7 @@ export class Fetcher {
         ...(proxyUrl && this.getProxyConfig(proxyUrl)),
         ...(!proxyUrl && { httpsAgent: new HttpsAgent({ rejectUnauthorized: false }) }),
         url: finalUrl.href,
-        timeout: this.DEFAULT_TIMEOUT,
+        timeout: requestConfig?.timeout ?? this.DEFAULT_TIMEOUT,
         transformResponse: [data => data],
         validateStatus: () => true,
       });
@@ -244,15 +244,19 @@ export class Fetcher {
 
       this.logger.info(`Query FlareSolverr for ${url.href}`, ctx);
 
+      const flareSolverrSessions = parseInt(envGet('FLARESOLVERR_SESSIONS') ?? '5');
       const data = {
         cmd: 'request.get',
         url: url.href,
-        session: 'default',
+        session: `${envGetAppId()}_${Math.floor(Math.random() * flareSolverrSessions)}`,
+        session_ttl_minutes: 60,
         maxTimeout: 15000,
+        cookies: (await this.cookieJar.getCookies(url.href)).map(cookie => ({ name: cookie.key, value: cookie.value })),
+        disableMedia: true,
         ...(proxyUrl && { proxy: { url: proxyUrl.href } }),
       };
 
-      const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000, queueLimit: 1 };
+      const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000, queueLimit: flareSolverrSessions };
       const challengeResult = JSON.parse((await this.queuedFetch(ctx, new URL('/v1', flareSolverrEndpoint), requestConfig)).data) as FlareSolverrResult;
 
       if (challengeResult.status !== 'ok') {