chore(fetcher): persist and use FlareSolverr cookie and user agent
doesn't seem to work, but this is best practice anyway..
This commit is contained in:
WebStreamr
parent
0d3a5b8919
commit
aa4c672594
4 changed files with 111 additions and 18 deletions
31
package-lock.json
generated
31
package-lock.json
generated
|
|
@ -17,6 +17,7 @@
|
|||
"http-cache-semantics": "^4.2.0",
|
||||
"randomstring": "^1.3.1",
|
||||
"slugify": "^1.6.6",
|
||||
"tough-cookie": "^5.1.2",
|
||||
"unpacker": "^1.0.1",
|
||||
"uuid": "^11.1.0",
|
||||
"winston": "^3.17.0"
|
||||
|
|
@ -6700,6 +6701,24 @@
|
|||
"integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",
|
||||
"integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^6.1.86"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts-core": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz",
|
||||
"integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tmpl": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz",
|
||||
|
|
@ -6739,6 +6758,18 @@
|
|||
"nodetouch": "bin/nodetouch.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
|
||||
"integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tldts": "^6.1.32"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/triple-beam": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz",
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@
|
|||
"http-cache-semantics": "^4.2.0",
|
||||
"randomstring": "^1.3.1",
|
||||
"slugify": "^1.6.6",
|
||||
"tough-cookie": "^5.1.2",
|
||||
"unpacker": "^1.0.1",
|
||||
"uuid": "^11.1.0",
|
||||
"winston": "^3.17.0"
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ describe('fetch', () => {
|
|||
'Forwarded': 'for=127.0.0.1',
|
||||
'Priority': 'u=0',
|
||||
'Referer': 'https://some-url.test',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3',
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
|
||||
'X-Forwarded-For': '127.0.0.1',
|
||||
'X-Forwarded-Proto': 'https',
|
||||
'X-Real-IP': '127.0.0.1',
|
||||
|
|
@ -83,11 +83,35 @@ describe('fetch', () => {
|
|||
test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => {
|
||||
process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x';
|
||||
fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
fetchMock.post('http://localhost:8191/v1/x', { body: { status: 'ok', solution: { response: 'some response' } } });
|
||||
fetchMock.post(
|
||||
'http://localhost:8191/v1/x',
|
||||
{
|
||||
body: {
|
||||
status: 'ok',
|
||||
solution: {
|
||||
response: 'some response',
|
||||
cookies: [
|
||||
{ name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000, domain: 'some-cloudflare-flaresolverr-success-url.test' },
|
||||
{ name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000, domain: 'some-other.domain' },
|
||||
],
|
||||
userAgent: 'user agent that works',
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const response = await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
|
||||
|
||||
expect(response).toBe('some response');
|
||||
|
||||
// The next request has to use the cookie and user agent we got
|
||||
await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
|
||||
expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Cookie': 'cf_clearance=some_value',
|
||||
'User-Agent': 'user agent that works',
|
||||
},
|
||||
});
|
||||
|
||||
delete process.env['FLARESOLVERR_ENDPOINT'];
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import CachePolicy from 'http-cache-semantics';
|
||||
import TTLCache from '@isaacs/ttlcache';
|
||||
import winston from 'winston';
|
||||
import { Cookie, CookieJar } from 'tough-cookie';
|
||||
import { Context, TIMEOUT } from '../types';
|
||||
import { BlockedError, NotFoundError, QueueIsFullError } from '../error';
|
||||
import { clearTimeout } from 'node:timers';
|
||||
|
|
@ -18,6 +19,17 @@ interface HostQueue {
|
|||
count: number;
|
||||
}
|
||||
|
||||
interface FlareResolverCookie {
|
||||
domain: string;
|
||||
expiry: number;
|
||||
httpOnly: boolean;
|
||||
name: string;
|
||||
path: string;
|
||||
sameSite: string;
|
||||
secure: boolean;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export class Fetcher {
|
||||
private readonly logger: winston.Logger;
|
||||
private readonly queueLimit: number;
|
||||
|
|
@ -25,6 +37,8 @@ export class Fetcher {
|
|||
|
||||
private readonly httpCache: TTLCache<string, HttpCacheItem>;
|
||||
private readonly hostQueues = new Map<string, HostQueue>();
|
||||
private readonly hostUserAgentMap = new Map<string, string>();
|
||||
private readonly cookieJar = new CookieJar();
|
||||
|
||||
constructor(logger: winston.Logger, queueLimit?: number, timeout?: number) {
|
||||
this.logger = logger;
|
||||
|
|
@ -46,21 +60,26 @@ export class Fetcher {
|
|||
return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders();
|
||||
};
|
||||
|
||||
public readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => ({
|
||||
...init,
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Forwarded': `for=${ctx.ip}`,
|
||||
'Priority': 'u=0',
|
||||
'Referer': `${ctx.referer?.href ?? url.origin}`,
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3',
|
||||
'X-Forwarded-For': ctx.ip,
|
||||
'X-Forwarded-Proto': url.protocol.slice(0, -1),
|
||||
'X-Real-IP': ctx.ip,
|
||||
...init?.headers,
|
||||
},
|
||||
});
|
||||
public readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => {
|
||||
const cookieString = this.cookieJar.getCookieStringSync(url.href);
|
||||
|
||||
return {
|
||||
...init,
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
...(cookieString && { Cookie: cookieString }),
|
||||
'Forwarded': `for=${ctx.ip}`,
|
||||
'Priority': 'u=0',
|
||||
'Referer': `${ctx.referer?.href ?? url.origin}`,
|
||||
'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
|
||||
'X-Forwarded-For': ctx.ip,
|
||||
'X-Forwarded-Proto': url.protocol.slice(0, -1),
|
||||
'X-Real-IP': ctx.ip,
|
||||
...init?.headers,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
private readonly handleHttpCacheItem = async (ctx: Context, httpCacheItem: HttpCacheItem, url: URL): Promise<HttpCacheItem> => {
|
||||
if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) {
|
||||
|
|
@ -91,6 +110,24 @@ export class Fetcher {
|
|||
throw new BlockedError('flaresolverr_failed', {});
|
||||
}
|
||||
|
||||
challengeResult.solution.cookies.forEach((cookie: FlareResolverCookie) => {
|
||||
if (!['cf_clearance'].includes(cookie.name)) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.cookieJar.setCookie(
|
||||
new Cookie({
|
||||
key: cookie.name,
|
||||
value: cookie.value,
|
||||
expires: new Date(cookie.expiry * 1000),
|
||||
domain: cookie.domain.replace(/^.+/, ''),
|
||||
}),
|
||||
url.href,
|
||||
);
|
||||
});
|
||||
|
||||
this.hostUserAgentMap.set(url.host, challengeResult.solution.userAgent);
|
||||
|
||||
httpCacheItem.body = challengeResult.solution.response;
|
||||
|
||||
const ttl = this.determineTtl(httpCacheItem);
|
||||
|
|
|
|||
Loading…
Reference in a new issue