import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; import { Context } from '../types'; import { CloudflareChallengeError, NotFoundError } from '../error'; import { clearTimeout } from 'node:timers'; interface HttpCacheItem { policy: CachePolicy; status: number; statusText: string; body: string } export class Fetcher { private readonly logger: winston.Logger; private readonly httpCache: TTLCache; constructor(logger: winston.Logger) { this.logger = logger; this.httpCache = new TTLCache(); } readonly text = async (ctx: Context, url: URL, init?: RequestInit): Promise => { return (await this.cachedFetch(ctx, url, init)).body; }; readonly textPost = async (ctx: Context, url: URL, data: unknown, init?: RequestInit): Promise => { return (await this.cachedFetch(ctx, url, { ...init, method: 'POST', body: JSON.stringify(data) })).body; }; readonly head = async (ctx: Context, url: URL, init?: RequestInit): Promise => { return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders(); }; private readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => ({ ...init, headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en', 'Forwarded': `for=${ctx.ip}`, 'Priority': 'u=0', 'Referer': `${ctx.referer?.href ?? url.origin}`, 'X-Forwarded-For': ctx.ip, 'X-Forwarded-Proto': url.protocol.slice(0, -1), 'X-Real-IP': ctx.ip, ...init?.headers, }, }); private readonly handleHttpCacheItem = (httpCacheItem: HttpCacheItem): HttpCacheItem => { if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) { return httpCacheItem; } if (httpCacheItem.status === 404) { throw new NotFoundError(); } if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') { throw new CloudflareChallengeError(); } throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}`); }; private readonly headersToObject = (headers: Headers): Record => { const obj: Record = {}; headers.forEach((value, name) => { obj[name] = value; }); return obj; }; private readonly cachedFetch = async (ctx: Context, url: URL, init?: RequestInit): Promise => { const newInit = this.getInit(ctx, url, init); const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} }; let httpCacheItem: HttpCacheItem | undefined = this.httpCache.get(url.href); if (httpCacheItem) { this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); return this.handleHttpCacheItem(httpCacheItem); } this.logger.info(`Fetch ${request.method} ${url}`, ctx); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 10000); let response; try { response = await fetch(url, { ...newInit, signal: controller.signal }); } finally { clearTimeout(timer); } const body = await response.text(); const policy = new CachePolicy(request, { status: response.status, headers: this.headersToObject(response.headers) }, { shared: false }); httpCacheItem = { policy, status: response.status, statusText: response.statusText, body }; let ttl; if (response.ok) { ttl = Math.max(policy.timeToLive(), 900000); // 15m at least } else { ttl = policy.timeToLive(); } if (ttl > 0) { this.httpCache.set(url.href, httpCacheItem, { ttl }); } return this.handleHttpCacheItem(httpCacheItem); }; }