diff --git a/src/types.ts b/src/types.ts index 43795f2..ccbf450 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,7 +15,7 @@ export type Config = Partial export type CountryCode = 'de' | 'en' | 'es' | 'fr' | 'it' | 'mx'; -export type BlockedReason = 'cloudflare_challenge' | 'unknown'; +export type BlockedReason = 'cloudflare_challenge' | 'flaresolverr_failed' | 'unknown'; export interface Meta { bytes?: number; diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 8818f98..352927d 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -80,6 +80,33 @@ describe('fetch', () => { } }); + test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => { + process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x'; + fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); + fetchMock.post('http://localhost:8191/v1/x', { body: { status: 'ok', solution: { response: 'some response' } } }); + + const response = await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/')); + + expect(response).toBe('some response'); + delete process.env['FLARESOLVERR_ENDPOINT']; + }); + + test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => { + process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y'; + fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); + fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } }); + + try { + await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/')); + fail(); + } catch (error) { + expect(error).toBeInstanceOf(BlockedError); + expect(error).toMatchObject({ reason: 'flaresolverr_failed' }); + } + + delete process.env['FLARESOLVERR_ENDPOINT']; + }); + test('converts generic forbidden to custom BlockedError', async () => { fetchMock.get('https://some-forbidden-url.test/', { status: 403 }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 93ef0fe..3c14cc6 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -4,6 +4,7 @@ import winston from 'winston'; import { Context, TIMEOUT } from '../types'; import { BlockedError, NotFoundError, QueueIsFullError } from '../error'; import { clearTimeout } from 'node:timers'; +import { envGet } from './env'; interface HttpCacheItem { policy: CachePolicy; @@ -61,7 +62,7 @@ export class Fetcher { }, }); - private readonly handleHttpCacheItem = (httpCacheItem: HttpCacheItem): HttpCacheItem => { + private readonly handleHttpCacheItem = async (ctx: Context, httpCacheItem: HttpCacheItem, url: URL): Promise => { if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) { return httpCacheItem; } @@ -73,7 +74,28 @@ export class Fetcher { const responseHeaders = httpCacheItem.policy.responseHeaders(); if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') { - throw new BlockedError('cloudflare_challenge', httpCacheItem.policy.responseHeaders()); + const flareSolverrEndpoint = envGet('FLARESOLVERR_ENDPOINT'); + if (!flareSolverrEndpoint) { + throw new BlockedError('cloudflare_challenge', httpCacheItem.policy.responseHeaders()); + } + + this.logger.info(`Query FlareSolverr for ${url.href}`, ctx); + + const body = { cmd: 'request.get', url: url.href, session: 'default' }; + const challengeResult = await (await fetch(new URL(flareSolverrEndpoint), { method: 'POST', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' } })).json(); + if (challengeResult.status !== 'ok') { + throw new BlockedError('flaresolverr_failed', {}); + } + + httpCacheItem.body = challengeResult.solution.response; + + const ttl = this.determineTtl(httpCacheItem); + /* istanbul ignore next */ + if (ttl > 0) { + this.httpCache.set(url.href, httpCacheItem, { ttl }); + } + + return httpCacheItem; } if (httpCacheItem.status === 403) { @@ -83,6 +105,14 @@ export class Fetcher { throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}, response headers: ${JSON.stringify(responseHeaders)}`); }; + private readonly determineTtl = (httpCacheItem: HttpCacheItem): number => { + if (httpCacheItem.status === 200) { + return Math.max(httpCacheItem.policy.timeToLive(), 900000); // 15m at least + } + + return httpCacheItem.policy.timeToLive(); + }; + private readonly headersToObject = (headers: Headers): Record => { const obj: Record = {}; @@ -101,7 +131,7 @@ export class Fetcher { let httpCacheItem: HttpCacheItem | undefined = this.httpCache.get(url.href); if (httpCacheItem) { this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); - return this.handleHttpCacheItem(httpCacheItem); + return this.handleHttpCacheItem(ctx, httpCacheItem, url); } const response = await this.queuedFetch(ctx, url, newInit); @@ -111,18 +141,12 @@ export class Fetcher { httpCacheItem = { policy, status: response.status, statusText: response.statusText, body }; - let ttl; - if (response.ok) { - ttl = Math.max(policy.timeToLive(), 900000); // 15m at least - } else { - ttl = policy.timeToLive(); - } - + const ttl = this.determineTtl(httpCacheItem); if (ttl > 0) { this.httpCache.set(url.href, httpCacheItem, { ttl }); } - return this.handleHttpCacheItem(httpCacheItem); + return this.handleHttpCacheItem(ctx, httpCacheItem, url); }; private readonly fetchWithTimeout = async (ctx: Context, url: URL, init?: RequestInit): Promise => {