diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 49a15cb..8532ba4 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -17,6 +17,20 @@ describe('fetch', () => { setGlobalDispatcher(mockAgent); }); + test('fetch', async () => { + const mockPool = mockAgent.get('https://some-fetch-url.test'); + mockPool.intercept({ path: '/' }).reply(200, 'some text'); + + expect(await fetcher.fetch(ctx, new URL('https://some-fetch-url.test/'))).toStrictEqual({ + body: 'some text', + headers: { age: '0', date: 'Wed, 11 Apr 1990 12:34:56 GMT' }, + status: 200, + statusText: 'OK', + ttl: 900000, + url: 'https://some-fetch-url.test/', + }); + }); + test('text passes successful response through setting headers', async () => { const mockPool = mockAgent.get('https://some-url.test'); mockPool.intercept({ path: '/' }).reply(200, 'some text'); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index f28790a..0086f89 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -11,12 +11,13 @@ import { noCache } from './config'; import { getProxyAgent, getProxyForUrl } from './dispatcher'; import { envGet } from './env'; -interface HttpCacheItem { +export interface HttpCacheItem { + body: string; headers: CachePolicy.Headers; status: number; statusText: string; - body: string; ttl: number; + url: string; } interface FlareSolverrResult { @@ -89,6 +90,10 @@ export class Fetcher { }; }; + public async fetch(ctx: Context, url: URL, init?: CustomRequestInit): Promise { + return await this.cachedFetch(ctx, url, init); + }; + public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise { return (await this.cachedFetch(ctx, url, init)).body; }; @@ -269,7 +274,7 @@ export class Fetcher { const policy = new CachePolicy(request, { status: response.status, headers: this.headersToObject(response.headers) }, { shared: false }); const ttl = this.determineCacheTtl(response.status, policy, init); - httpCacheItem = { headers: policy.responseHeaders(), status: response.status, statusText: response.statusText, body, ttl }; + httpCacheItem = { headers: policy.responseHeaders(), status: response.status, statusText: response.statusText, body, ttl, url: response.url }; await this.cacheSet(cacheKey, httpCacheItem); diff --git a/src/utils/FetcherMock.ts b/src/utils/FetcherMock.ts index 5e25c22..5ba00ec 100644 --- a/src/utils/FetcherMock.ts +++ b/src/utils/FetcherMock.ts @@ -7,7 +7,7 @@ import { RequestInit } from 'undici'; import winston from 'winston'; import { Context } from '../types'; import { envGet } from './env'; -import { Fetcher } from './Fetcher'; +import { Fetcher, HttpCacheItem } from './Fetcher'; export class FetcherMock extends Fetcher { private readonly fixturePath: string; @@ -18,22 +18,37 @@ export class FetcherMock extends Fetcher { this.fixturePath = fixturePath; } + public override async fetch(ctx: Context, url: URL, init?: RequestInit): Promise { + let path: string; + + if (init?.method === 'POST') { + const body = init.body as string; + path = `${this.fixturePath}/post-${this.slugifyUrl(url)}-${slugify(body)}`; + } else if (init?.method === 'HEAD') { + path = `${this.fixturePath}/head-${this.slugifyUrl(url)}`; + } else { + path = `${this.fixturePath}/${this.slugifyUrl(url)}`; + } + + return this.fetchInternal(path, ctx, url, init); + }; + public override async text(ctx: Context, url: URL, init?: RequestInit): Promise { const path = `${this.fixturePath}/${this.slugifyUrl(url)}`; - return this.fetch(path, ctx, url, init); + return (await this.fetchInternal(path, ctx, url, init)).body; }; public override async textPost(ctx: Context, url: URL, body: string, init?: RequestInit): Promise { const path = `${this.fixturePath}/post-${this.slugifyUrl(url)}-${slugify(body)}`; - return this.fetch(path, ctx, url, { ...init, method: 'POST', body }); + return (await this.fetchInternal(path, ctx, url, { ...init, method: 'POST', body })).body; }; public override async head(ctx: Context, url: URL, init?: RequestInit): Promise { const path = `${this.fixturePath}/head-${this.slugifyUrl(url)}`; - return JSON.parse(await this.fetch(path, ctx, url, { ...init, method: 'HEAD' })); + return (await this.fetchInternal(path, ctx, url, { ...init, method: 'HEAD' })).headers; }; private readonly slugifyUrl = (url: URL): string => { @@ -46,13 +61,24 @@ export class FetcherMock extends Fetcher { return slugifiedUrl; }; - private readonly fetch = async (path: string, ctx: Context, url: URL, init?: RequestInit): Promise => { + private readonly fetchInternal = async (path: string, ctx: Context, url: URL, init?: RequestInit): Promise => { const errorPath = `${path}.error`; + const isHead = init?.method === 'HEAD'; + if (fs.existsSync(errorPath)) { throw new Error(fs.readFileSync(errorPath).toString()); } else if (fs.existsSync(path)) { - return fs.readFileSync(path).toString(); + const data = fs.readFileSync(path).toString(); + + return { + body: isHead ? '' : data, + headers: isHead ? JSON.parse(data) : {}, + status: 200, + statusText: 'OK', + ttl: 0, + url: url.href, + }; } else { let response; try { @@ -74,7 +100,7 @@ export class FetcherMock extends Fetcher { } let result; - if (init?.method === 'HEAD') { + if (isHead) { const headers: Record = {}; response.headers.forEach((value, key) => { @@ -88,7 +114,14 @@ export class FetcherMock extends Fetcher { fs.writeFileSync(path, result); - return result; + return { + body: isHead ? '' : result, + headers: isHead ? JSON.parse(result) : {}, + status: 200, + statusText: 'OK', + ttl: 0, + url: url.href, + }; } }; }