webstreamr-github/src/utils/Fetcher.test.ts

import winston from 'winston';
import fetchMock from 'fetch-mock';
import { Fetcher } from './Fetcher';
import { Context } from '../types';
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError } from '../error';
fetchMock.mockGlobal();

const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));

describe('fetch', () => {
  const ctx: Context = { id: 'id', ip: '127.0.0.1', config: { de: 'on' } };

  afterEach(() => {
    fetchMock.clearHistory();
  });

  test('text passes successful response through setting headers', async () => {
    fetchMock.get('https://some-url.test/', 'some text');

    const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/'));
    const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });

    expect(responseText1).toBe('some text');
    expect(responseText2).toStrictEqual(responseText1);

    expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
      headers: {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en',
        'Forwarded': 'for=127.0.0.1',
        'Priority': 'u=0',
        'User-Agent': 'node',
        'X-Forwarded-For': '127.0.0.1',
        'X-Forwarded-Proto': 'https',
        'X-Real-IP': '127.0.0.1',
      },
    });
  });

  test('textPost ', async () => {
    fetchMock.post('https://some-post-url.test/', 'some text');

    expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), JSON.stringify({ foo: 'bar' }))).toBe('some text');
  });

  test('head', async () => {
    fetchMock.head('https://some-head-url.test/', { status: 200, headers: { 'X-Fake-Response': 'foo' } });

    expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
  });

  test('converts 404 to custom NotFoundError', async () => {
    fetchMock.get('https://some-404-url.test/', 404);

    await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
  });

  test('converts Cloudflare challenge block to custom BlockedError', async () => {
    fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });

    try {
      await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(BlockedError);
      expect(error).toMatchObject({ reason: 'cloudflare_challenge' });
    }
  });

  test('uses FlareSolverr to solve Cloudflare challenge if configured and succeeds', async () => {
    process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/x';
    fetchMock.get('https://some-cloudflare-flaresolverr-success-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
    fetchMock.post(
      'http://localhost:8191/v1/x',
      {
        body: {
          status: 'ok',
          solution: {
            response: 'some response',
            cookies: [
              { name: 'cf_clearance', value: 'some_value', expires: Date.now() / 1000 + 12345, domain: 'some-cloudflare-flaresolverr-success-url.test' },
              { name: 'irrelevant', value: 'some_other_value', expires: Date.now() / 1000 + 12345, domain: 'some-other.domain' },
            ],
            userAgent: 'user agent that works',
          },
        },
      },
    );

    const response = await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
    expect(response).toBe('some response');

    // The next request has to use the cookie and user agent we got
    await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-success-url.test/'));
    expect(fetchMock.callHistory.callLogs[2]?.args[1]).toMatchObject({
      headers: {
        'Cookie': 'cf_clearance=some_value',
        'User-Agent': 'user agent that works',
      },
    });

    delete process.env['FLARESOLVERR_ENDPOINT'];
  });

  test('uses FlareSolverr to solve Cloudflare challenge if configured and fails with custom BlockedError', async () => {
    process.env['FLARESOLVERR_ENDPOINT'] = 'http://localhost:8191/v1/y';
    fetchMock.get('https://some-cloudflare-flaresolverr-fail-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
    fetchMock.post('http://localhost:8191/v1/y', { body: { status: 'nok' } });

    try {
      await fetcher.text(ctx, new URL('https://some-cloudflare-flaresolverr-fail-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(BlockedError);
      expect(error).toMatchObject({ reason: 'flaresolverr_failed' });
    }

    delete process.env['FLARESOLVERR_ENDPOINT'];
  });

  test('converts generic forbidden to custom BlockedError', async () => {
    fetchMock.get('https://some-forbidden-url.test/', { status: 403 });

    try {
      await fetcher.text(ctx, new URL('https://some-forbidden-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(BlockedError);
      expect(error).toMatchObject({ reason: 'unknown' });
    }
  });

  test('converts 429 to custom TooManyRequestsError', async () => {
    fetchMock.get('https://some-rate-limited-url.test/', { status: 429 });

    try {
      await fetcher.text(ctx, new URL('https://some-rate-limited-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(TooManyRequestsError);
      expect(error).toMatchObject({ retryAfter: NaN });
    }
  });

  test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => {
    fetchMock.get('https://some-rate-limited-retry-after-url.test/', { status: 429, headers: { 'retry-after': '10' } });

    try {
      await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(TooManyRequestsError);
      expect(error).toMatchObject({ retryAfter: 10 });
    }

    try {
      await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(TooManyRequestsError);
    }
  });

  test('passes through other error as HttpError', async () => {
    fetchMock.get('https://some-error-url.test/', { status: 500, headers: { 'x-foo': 'bar' } });

    try {
      await fetcher.text(ctx, new URL('https://some-error-url.test/'));
      fail();
    } catch (error) {
      expect(error).toBeInstanceOf(HttpError);
      expect(error).toMatchObject({ status: 500, headers: { 'x-foo': 'bar' } });
    }
  });

  test('passes through exceptions', async () => {
    fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') });

    await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError);
  });

  test('times out', async () => {
    fetchMock.get('https://some-timeout-url.test/', 200, { delay: 20 });

    await expect(fetcher.text(ctx, new URL('https://some-timeout-url.test/'), { timeout: 10 })).rejects.toBeInstanceOf(TimeoutError);
  });

  test('queues requests and throws if queue is full', async () => {
    fetchMock.get('https://some-full-queue-url.test/', 200, { delay: 20 });

    const allPromises = Promise.all([
      fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
      fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
      fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
      fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
      fetcher.text(ctx, new URL('https://some-full-queue-url.test/'), { queueLimit: 1, queueTimeout: 10 }),
    ]);

    await expect(allPromises).rejects.toBeInstanceOf(QueueIsFullError);
  });
});