From aa4c6725944d7fffc071acfcb27d57f1141e869c Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Wed, 4 Jun 2025 18:10:21 +0000 Subject: [PATCH] chore(fetcher): persist and use FlareSolverr cookie and user agent doesn't seem to work, but this is best practice anyway.. --- package-lock.json | 31 ++++++++++++++++++ package.json | 1 + src/utils/Fetcher.test.ts | 30 ++++++++++++++++-- src/utils/Fetcher.ts | 67 ++++++++++++++++++++++++++++++--------- 4 files changed, 111 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index fa2bb25..a36cc1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", + "tough-cookie": "^5.1.2", "unpacker": "^1.0.1", "uuid": "^11.1.0", "winston": "^3.17.0" @@ -6700,6 +6701,24 @@ "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", "license": "MIT" }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -6739,6 +6758,18 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/triple-beam": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", diff --git a/package.json b/package.json index 1579db3..6c848c4 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", + "tough-cookie": "^5.1.2", "unpacker": "^1.0.1", "uuid": "^11.1.0", "winston": "^3.17.0" diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 352927d..06b2414 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -30,7 +30,7 @@ describe('fetch', () => { 'Forwarded': 'for=127.0.0.1', 'Priority': 'u=0', 'Referer': 'https://some-url.test', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36', 'X-Forwarded-For': '127.0.0.1', 'X-Forwarded-Proto': 'https', 'X-Real-IP': '127.0.0.1', @@ -83,11 +83,35 @@ describe('fetch', () => { test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => { process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x'; fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); - fetchMock.post('http://localhost:8191/v1/x', { body: { status: 'ok', solution: { response: 'some response' } } }); + fetchMock.post( + 'http://localhost:8191/v1/x', + { + body: { + status: 'ok', + solution: { + response: 'some response', + cookies: [ + { name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000, domain: 'some-cloudflare-flaresolverr-success-url.test' }, + { name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000, domain: 'some-other.domain' }, + ], + userAgent: 'user agent that works', + }, + }, + }, + ); const response = await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/')); - expect(response).toBe('some response'); + + // The next request has to use the cookie and user agent we got + await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/')); + expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({ + headers: { + 'Cookie': 'cf_clearance=some_value', + 'User-Agent': 'user agent that works', + }, + }); + delete process.env['FLARESOLVERR_ENDPOINT']; }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index ef0759f..ab2f45a 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,6 +1,7 @@ import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; +import { Cookie, CookieJar } from 'tough-cookie'; import { Context, TIMEOUT } from '../types'; import { BlockedError, NotFoundError, QueueIsFullError } from '../error'; import { clearTimeout } from 'node:timers'; @@ -18,6 +19,17 @@ interface HostQueue { count: number; } +interface FlareResolverCookie { + domain: string; + expiry: number; + httpOnly: boolean; + name: string; + path: string; + sameSite: string; + secure: boolean; + value: string; +} + export class Fetcher { private readonly logger: winston.Logger; private readonly queueLimit: number; @@ -25,6 +37,8 @@ export class Fetcher { private readonly httpCache: TTLCache; private readonly hostQueues = new Map(); + private readonly hostUserAgentMap = new Map(); + private readonly cookieJar = new CookieJar(); constructor(logger: winston.Logger, queueLimit?: number, timeout?: number) { this.logger = logger; @@ -46,21 +60,26 @@ export class Fetcher { return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders(); }; - public readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => ({ - ...init, - headers: { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en', - 'Forwarded': `for=${ctx.ip}`, - 'Priority': 'u=0', - 'Referer': `${ctx.referer?.href ?? url.origin}`, - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3', - 'X-Forwarded-For': ctx.ip, - 'X-Forwarded-Proto': url.protocol.slice(0, -1), - 'X-Real-IP': ctx.ip, - ...init?.headers, - }, - }); + public readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => { + const cookieString = this.cookieJar.getCookieStringSync(url.href); + + return { + ...init, + headers: { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + ...(cookieString && { Cookie: cookieString }), + 'Forwarded': `for=${ctx.ip}`, + 'Priority': 'u=0', + 'Referer': `${ctx.referer?.href ?? url.origin}`, + 'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36', + 'X-Forwarded-For': ctx.ip, + 'X-Forwarded-Proto': url.protocol.slice(0, -1), + 'X-Real-IP': ctx.ip, + ...init?.headers, + }, + }; + }; private readonly handleHttpCacheItem = async (ctx: Context, httpCacheItem: HttpCacheItem, url: URL): Promise => { if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) { @@ -91,6 +110,24 @@ export class Fetcher { throw new BlockedError('flaresolverr_failed', {}); } + challengeResult.solution.cookies.forEach((cookie: FlareResolverCookie) => { + if (!['cf_clearance'].includes(cookie.name)) { + return; + } + + this.cookieJar.setCookie( + new Cookie({ + key: cookie.name, + value: cookie.value, + expires: new Date(cookie.expiry * 1000), + domain: cookie.domain.replace(/^.+/, ''), + }), + url.href, + ); + }); + + this.hostUserAgentMap.set(url.host, challengeResult.solution.userAgent); + httpCacheItem.body = challengeResult.solution.response; const ttl = this.determineTtl(httpCacheItem);