diff --git a/README.md b/README.md index 989de95..7320d35 100644 --- a/README.md +++ b/README.md @@ -78,10 +78,6 @@ Optional. Comma separated list of sources which should be disabled. E.g. `frembe Optional. If domains show Cloudflare challenges, FlareSolverr can be used to work around them. E.g. `http://flaresolverr:8191` Proxy configuration is passed-through and only a single session is used to save resources. Byparr is not supported. -#### `FLARESOLVERR_SESSIONS` - -Optional. Configure how many FlareSolverr sessions can be open at the same time. Default: `5` - #### `MANIFEST_ID` Optional. Add-on manifest ID. Default: `webstreamr` diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 5d086dd..cb02e68 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -73,6 +73,8 @@ export class Fetcher { private readonly httpStatus = new Map>(); private readonly httpStatusMutex = new Mutex(); + private readonly flareSolverrMutexes = new Map(); + public constructor(axios: AxiosInstance, logger: winston.Logger) { this.axios = axios; this.logger = logger; @@ -242,22 +244,30 @@ export class Fetcher { throw new BlockedError(url, BlockedReason.cloudflare_challenge, response.headers); } - this.logger.info(`Query FlareSolverr for ${url.href}`, ctx); + const session = `${envGetAppId()}_${url.host}`; - const flareSolverrSessions = parseInt(envGet('FLARESOLVERR_SESSIONS') ?? '5'); - const data = { - cmd: 'request.get', - url: url.href, - session: `${envGetAppId()}_${Math.floor(Math.random() * flareSolverrSessions)}`, - session_ttl_minutes: 60, - maxTimeout: 15000, - cookies: (await this.cookieJar.getCookies(url.href)).map(cookie => ({ name: cookie.key, value: cookie.value })), - disableMedia: true, - ...(proxyUrl && { proxy: { url: proxyUrl.href } }), - }; + let mutex = this.flareSolverrMutexes.get(session); + if (!mutex) { + mutex = new Mutex(); + this.flareSolverrMutexes.set(session, mutex); + } - const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000, queueLimit: flareSolverrSessions }; - const challengeResult = JSON.parse((await this.queuedFetch(ctx, new URL('/v1', flareSolverrEndpoint), requestConfig)).data) as FlareSolverrResult; + const challengeResult = await mutex.runExclusive(async () => { + this.logger.info(`Query FlareSolverr for ${url.href}`, ctx); + + const data = { + cmd: 'request.get', + url: url.href, + session, + session_ttl_minutes: 60, + maxTimeout: 15000, + disableMedia: true, + ...(proxyUrl && { proxy: { url: proxyUrl.href } }), + }; + + const requestConfig: CustomRequestConfig = { method: 'POST', data, headers: { 'Content-Type': 'application/json' }, timeout: 15000, queueTimeout: 60000 }; + return JSON.parse((await this.queuedFetch(ctx, new URL('/v1', flareSolverrEndpoint), requestConfig)).data) as FlareSolverrResult; + }); if (challengeResult.status !== 'ok') { this.logger.warn(`FlareSolverr issue: ${JSON.stringify(challengeResult)}`, ctx);