chore(fetcher): add generic fetch also returning response url

src/utils/Fetcher.test.ts

@@ -17,6 +17,20 @@ describe('fetch', () => {
     setGlobalDispatcher(mockAgent);
   });
 
+  test('fetch', async () => {
+    const mockPool = mockAgent.get('https://some-fetch-url.test');
+    mockPool.intercept({ path: '/' }).reply(200, 'some text');
+
+    expect(await fetcher.fetch(ctx, new URL('https://some-fetch-url.test/'))).toStrictEqual({
+      body: 'some text',
+      headers: { age: '0', date: 'Wed, 11 Apr 1990 12:34:56 GMT' },
+      status: 200,
+      statusText: 'OK',
+      ttl: 900000,
+      url: 'https://some-fetch-url.test/',
+    });
+  });
+
   test('text passes successful response through setting headers', async () => {
     const mockPool = mockAgent.get('https://some-url.test');
     mockPool.intercept({ path: '/' }).reply(200, 'some text');

src/utils/Fetcher.ts

@@ -11,12 +11,13 @@ import { noCache } from './config';
 import { getProxyAgent, getProxyForUrl } from './dispatcher';
 import { envGet } from './env';
 
-interface HttpCacheItem {
+export interface HttpCacheItem {
+  body: string;
   headers: CachePolicy.Headers;
   status: number;
   statusText: string;
-  body: string;
   ttl: number;
+  url: string;
 }
 
 interface FlareSolverrResult {
@@ -89,6 +90,10 @@ export class Fetcher {
     };
   };
 
+  public async fetch(ctx: Context, url: URL, init?: CustomRequestInit): Promise<HttpCacheItem> {
+    return await this.cachedFetch(ctx, url, init);
+  };
+
   public async text(ctx: Context, url: URL, init?: CustomRequestInit): Promise<string> {
     return (await this.cachedFetch(ctx, url, init)).body;
   };
@@ -269,7 +274,7 @@ export class Fetcher {
     const policy = new CachePolicy(request, { status: response.status, headers: this.headersToObject(response.headers) }, { shared: false });
     const ttl = this.determineCacheTtl(response.status, policy, init);
 
-    httpCacheItem = { headers: policy.responseHeaders(), status: response.status, statusText: response.statusText, body, ttl };
+    httpCacheItem = { headers: policy.responseHeaders(), status: response.status, statusText: response.statusText, body, ttl, url: response.url };
 
     await this.cacheSet(cacheKey, httpCacheItem);
 

src/utils/FetcherMock.ts

@@ -7,7 +7,7 @@ import { RequestInit } from 'undici';
 import winston from 'winston';
 import { Context } from '../types';
 import { envGet } from './env';
-import { Fetcher } from './Fetcher';
+import { Fetcher, HttpCacheItem } from './Fetcher';
 
 export class FetcherMock extends Fetcher {
   private readonly fixturePath: string;
@@ -18,22 +18,37 @@ export class FetcherMock extends Fetcher {
     this.fixturePath = fixturePath;
   }
 
+  public override async fetch(ctx: Context, url: URL, init?: RequestInit): Promise<HttpCacheItem> {
+    let path: string;
+
+    if (init?.method === 'POST') {
+      const body = init.body as string;
+      path = `${this.fixturePath}/post-${this.slugifyUrl(url)}-${slugify(body)}`;
+    } else if (init?.method === 'HEAD') {
+      path = `${this.fixturePath}/head-${this.slugifyUrl(url)}`;
+    } else {
+      path = `${this.fixturePath}/${this.slugifyUrl(url)}`;
+    }
+
+    return this.fetchInternal(path, ctx, url, init);
+  };
+
   public override async text(ctx: Context, url: URL, init?: RequestInit): Promise<string> {
     const path = `${this.fixturePath}/${this.slugifyUrl(url)}`;
 
-    return this.fetch(path, ctx, url, init);
+    return (await this.fetchInternal(path, ctx, url, init)).body;
   };
 
   public override async textPost(ctx: Context, url: URL, body: string, init?: RequestInit): Promise<string> {
     const path = `${this.fixturePath}/post-${this.slugifyUrl(url)}-${slugify(body)}`;
 
-    return this.fetch(path, ctx, url, { ...init, method: 'POST', body });
+    return (await this.fetchInternal(path, ctx, url, { ...init, method: 'POST', body })).body;
   };
 
   public override async head(ctx: Context, url: URL, init?: RequestInit): Promise<CachePolicy.Headers> {
     const path = `${this.fixturePath}/head-${this.slugifyUrl(url)}`;
 
-    return JSON.parse(await this.fetch(path, ctx, url, { ...init, method: 'HEAD' }));
+    return (await this.fetchInternal(path, ctx, url, { ...init, method: 'HEAD' })).headers;
   };
 
   private readonly slugifyUrl = (url: URL): string => {
@@ -46,13 +61,24 @@ export class FetcherMock extends Fetcher {
     return slugifiedUrl;
   };
 
-  private readonly fetch = async (path: string, ctx: Context, url: URL, init?: RequestInit): Promise<string> => {
+  private readonly fetchInternal = async (path: string, ctx: Context, url: URL, init?: RequestInit): Promise<HttpCacheItem> => {
     const errorPath = `${path}.error`;
 
+    const isHead = init?.method === 'HEAD';
+
     if (fs.existsSync(errorPath)) {
       throw new Error(fs.readFileSync(errorPath).toString());
     } else if (fs.existsSync(path)) {
-      return fs.readFileSync(path).toString();
+      const data = fs.readFileSync(path).toString();
+
+      return {
+        body: isHead ? '' : data,
+        headers: isHead ? JSON.parse(data) : {},
+        status: 200,
+        statusText: 'OK',
+        ttl: 0,
+        url: url.href,
+      };
     } else {
       let response;
       try {
@@ -74,7 +100,7 @@ export class FetcherMock extends Fetcher {
       }
 
       let result;
-      if (init?.method === 'HEAD') {
+      if (isHead) {
         const headers: Record<string, string> = {};
 
         response.headers.forEach((value, key) => {
@@ -88,7 +114,14 @@ export class FetcherMock extends Fetcher {
 
       fs.writeFileSync(path, result);
 
-      return result;
+      return {
+        body: isHead ? '' : result,
+        headers: isHead ? JSON.parse(result) : {},
+        status: 200,
+        statusText: 'OK',
+        ttl: 0,
+        url: url.href,
+      };
     }
   };
 }