refactor(fetcher): switch to axios for hopefully better errors

package.json

@@ -31,11 +31,11 @@
   },
   "dependencies": {
     "@isaacs/ttlcache": "^1.4.1",
+    "axios-cache-interceptor": "^1.8.0",
     "bytes": "^3.1.2",
     "cheerio": "^1.0.0",
     "country-emoji": "^1.5.6",
     "express": "^5.1.0",
-    "make-fetch-happen": "^14.0.3",
     "slugify": "^1.6.6",
     "unpacker": "^1.0.1",
     "user-agents": "^1.1.536",
@@ -51,6 +51,7 @@
     "@types/node": "^22.15.3",
     "@types/stremio-addon-sdk": "^1.6.11",
     "@types/user-agents": "^1.0.4",
+    "axios-mock-adapter": "^2.1.0",
     "babel-jest": "^29.7.0",
     "eslint": "^9.26.0",
     "jest": "^29.7.0",

src/index.ts

@@ -1,12 +1,11 @@
 import express, { NextFunction, Request, Response } from 'express';
-import makeFetchHappen from 'make-fetch-happen';
+import axios from 'axios';
+import { setupCache } from 'axios-cache-interceptor';
 import winston from 'winston';
 import { FrenchCloud, Handler, KinoKiste, MeineCloud, MostraGuarda, VerHdLink } from './handler';
 import { EmbedExtractorRegistry } from './embed-extractor';
 import { ConfigureController, ManifestController, StreamController } from './controller';
 import { Fetcher, StreamResolver } from './utils';
-import fs from 'node:fs';
-import * as os from 'node:os';
 
 const logger = winston.createLogger({
   transports: [
@@ -20,13 +19,7 @@ const logger = winston.createLogger({
   ],
 });
 
-const fetcher = new Fetcher(
-  makeFetchHappen.defaults({
-    cachePath: `${fs.realpathSync(os.tmpdir())}/webstreamr`,
-    maxSockets: 5,
-  }),
-  logger,
-);
+const fetcher = new Fetcher(setupCache(axios), logger);
 
 const embedExtractors = new EmbedExtractorRegistry(fetcher);
 

src/utils/Fetcher.test.ts

@@ -1,36 +1,28 @@
-import makeFetchHappen from 'make-fetch-happen';
 import winston from 'winston';
+import axios from 'axios';
+import AxiosMockAdapter from 'axios-mock-adapter';
 import { Fetcher } from './Fetcher';
 import { Context } from '../types';
 
-const mockedFetch = jest.fn();
-jest.mock('make-fetch-happen', () => ({
-  __esModule: true,
-  default: {
-    defaults: () => mockedFetch,
-  },
-}));
+const axiosMock = new AxiosMockAdapter(axios);
 
-const fetcher = new Fetcher(makeFetchHappen.defaults(), winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
+const fetcher = new Fetcher(axios, winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
 
 describe('fetch', () => {
   const ctx: Context = { ip: '127.0.0.1' };
 
-  test('text throws if the response is not OK', async () => {
-    mockedFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'some error happened' });
-
-    await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toThrow(new Error('HTTP error: 500 - some error happened'));
-  });
-
   test('text passes successful response through setting headers', async () => {
-    mockedFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('some text') });
+    axiosMock.onGet().reply(200, 'some text');
+    const axiosSpy = jest.spyOn(axios, 'get');
 
-    const responseText = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
+    const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/'));
+    const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
 
-    expect(responseText).toBe('some text');
-    expect(mockedFetch).toHaveBeenCalledWith(
-      'https://some-url.test/',
-      {
+    expect(responseText1).toBe('some text');
+    expect(responseText2).toStrictEqual(responseText1);
+
+    expect(axiosSpy).toHaveBeenCalledWith(
+      'https://some-url.test/', {
         headers: {
           'User-Agent': expect.not.stringMatching(/jest/),
           'Forwarded': 'for=127.0.0.1',

src/utils/Fetcher.ts

@@ -1,31 +1,31 @@
-import { FetchInterface, FetchOptions } from 'make-fetch-happen';
+import { AxiosInstance, AxiosRequestConfig } from 'axios';
 import TTLCache from '@isaacs/ttlcache';
 import UserAgent from 'user-agents';
 import winston from 'winston';
 import { Context } from '../types';
 
 export class Fetcher {
-  private readonly fetch: FetchInterface;
+  private readonly axios: AxiosInstance;
 
   private readonly logger: winston.Logger;
 
   private readonly ipUserAgentCache: TTLCache<string, string>;
 
-  constructor(fetch: FetchInterface, logger: winston.Logger) {
-    this.fetch = fetch;
+  constructor(axios: AxiosInstance, logger: winston.Logger) {
+    this.axios = axios;
     this.logger = logger;
     this.ipUserAgentCache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h
   }
 
-  readonly text = async (ctx: Context, url: URL, opts?: FetchOptions): Promise<string> => {
+  readonly text = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<string> => {
     this.logger.info(`Fetch ${url}`);
 
-    const response = await this.fetch(
+    const response = await this.axios.get(
       url.href,
       {
-        ...opts,
+        ...config,
         headers: {
-          ...opts?.headers,
+          ...config?.headers,
           'User-Agent': this.createUserAgentForIp(ctx.ip),
           'Forwarded': `for=${ctx.ip}`,
           'X-Forwarded-For': ctx.ip,
@@ -35,11 +35,7 @@ export class Fetcher {
       },
     );
 
-    if (!response.ok) {
-      throw new Error(`HTTP error: ${response.status} - ${response.statusText}`);
-    }
-
-    return await response.text();
+    return response.data;
   };
 
   private readonly createUserAgentForIp = (ip: string): string => {

src/utils/__mocks__/Fetcher.ts

@@ -1,17 +1,16 @@
-import { FetchOptions } from 'make-fetch-happen';
-import makeFetchHappen from 'make-fetch-happen';
 import fs from 'node:fs';
+import Axios, { AxiosRequestConfig } from 'axios';
 import slugify from 'slugify';
 import { Context } from '../../types';
 
 export class Fetcher {
-  readonly text = async (_ctx: Context, url: URL, opts?: FetchOptions): Promise<string> => {
+  readonly text = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<string> => {
     const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(url.href)}`;
 
     if (fs.existsSync(path)) {
       return fs.readFileSync(path).toString();
     } else {
-      const text = await (await makeFetchHappen.defaults()(url.href, opts)).text();
+      const text = (await Axios.create().get(url.href, config)).data;
 
       fs.writeFileSync(path, text);