chore(extractor): use HEAD instead of GET to check external URLs

src/extractor/ExternalUrl.ts

@@ -19,7 +19,7 @@ export class ExternalUrl implements Extractor {
 
   readonly extract = async (ctx: Context, url: URL, countryCode: string) => {
     // We only want to make sure that the URL is accessible
-    await this.fetcher.text(ctx, url);
+    await this.fetcher.head(ctx, url);
 
     return {
       url: url,

src/utils/Fetcher.test.ts

@@ -74,4 +74,27 @@ describe('fetch', () => {
       },
     );
   });
+
+  test('head passes successful response through setting headers', async () => {
+    axiosMock.onHead().reply(200, undefined, { 'X-Fake-Response': 'foo' });
+    const axiosSpy = jest.spyOn(axios, 'head');
+
+    const responseHeaders1 = await fetcher.head(ctx, new URL('https://some-url.test/'));
+    const responseHeaders2 = await fetcher.head(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
+
+    expect(responseHeaders1).toMatchObject({ 'X-Fake-Response': 'foo' });
+    expect(responseHeaders2).toStrictEqual(responseHeaders1);
+
+    expect(axiosSpy).toHaveBeenCalledWith(
+      'https://some-url.test/',
+      {
+        headers: {
+          'User-Agent': expect.not.stringMatching(/jest/),
+          'Referer': 'https://some-url.test',
+        },
+        responseType: 'text',
+        timeout: 15000,
+      },
+    );
+  });
 });

src/utils/Fetcher.ts

@@ -1,4 +1,4 @@
-import { AxiosInstance, AxiosRequestConfig } from 'axios';
+import { AxiosInstance, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios';
 import TTLCache from '@isaacs/ttlcache';
 import UserAgent from 'user-agents';
 import winston from 'winston';
@@ -31,6 +31,14 @@ export class Fetcher {
     return response.data;
   };
 
+  readonly head = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<RawAxiosResponseHeaders | AxiosResponseHeaders> => {
+    this.logger.info(`Fetch head ${url}`, ctx);
+
+    const response = await this.axios.head(url.href, this.getConfig(ctx, url, config));
+
+    return response.headers;
+  };
+
   private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => {
     return {
       responseType: 'text',

src/utils/__fixtures__/Fetcher/head-https:johnalwayssame.comecqy9oue7sv0g

@@ -0,0 +1 @@
+{"server":"ddos-guard","connection":"keep-alive","keep-alive":"timeout=60","set-cookie":["__ddg8_=gmw8dCG6blfocUE2; Domain=.johnalwayssame.com; Path=/; Expires=Thu, 22-May-2025 16:46:44 GMT","__ddg10_=1747931204; Domain=.johnalwayssame.com; Path=/; Expires=Thu, 22-May-2025 16:46:44 GMT","__ddg9_=130.61.236.204; Domain=.johnalwayssame.com; Path=/; Expires=Thu, 22-May-2025 16:46:44 GMT","__ddg1_=BgkoUHamR633bMYVIX7f; Domain=.johnalwayssame.com; HttpOnly; Path=/; Expires=Fri, 22-May-2026 16:26:44 GMT","XSRF-TOKEN=eyJpdiI6InNsano3R3B5dW41QkpHNzZOd2lvcXc9PSIsInZhbHVlIjoiS3ZVTnZTRXJKc2F2OFh3MDFFWjhvZHpVOThVZnlZV2UzcXdxRVNOY2hWYTIvUUJCZnBtZXZiR295L0ozNXFGS0VmMlJ0VldpSmRKOE91Y2pjT0cxVVhXUDdvTG1HaDlFaldWaHIyTEJ2UysrMVJ4WGViUng5dGVEZmlqZHZqZ2EiLCJtYWMiOiJiYTJjY2U4ZGI2NjIwNTFhNDVhYzFiOWY5NzcxNTkxZWRlMGQxNmJhZTFiN2JiMTgxYjRhMjI5MmIyMzI1ZmM1IiwidGFnIjoiIn0%3D; expires=Thu, 22 May 2025 17:56:44 GMT; Max-Age=5400; path=/; secure; samesite=none; partitioned","voe_session=eyJpdiI6IjJPR1JsTHR6Y1VaSTJ1N08vQm9mbWc9PSIsInZhbHVlIjoiMnlLU244dVVyazVKUHBPWXdjdWg1bW1IY1U4Qkx6WmNSSHVXamRVMnpwOWRnRWdTT2VFTExYelN1ZU9mVTJJN0QvYy9BdGRwQ3ZPWjlGaS96TkpDaVY5S3VuQytoNUpZTHRoUy92OFcwWWgzWlZKbFlpTktXZThFTjZkK2xkbUIiLCJtYWMiOiI3YzU1OTZjZTg1OWJkOGYyN2VmMGNhY2Q5MzE0MzA4MTMwZjdiMDlkNmUxNzczZDkzNjI5MmQ3NWRmMTIzZTEwIiwidGFnIjoiIn0%3D; expires=Thu, 22 May 2025 17:56:44 GMT; Max-Age=5400; path=/; secure; httponly; samesite=none; partitioned"],"strict-transport-security":"max-age=2628000","content-security-policy":"upgrade-insecure-requests;","content-type":"text/html; charset=UTF-8","vary":"Accept-Encoding","cache-control":"no-cache, private","date":"Thu, 22 May 2025 16:26:44 GMT"}

src/utils/__fixtures__/Fetcher/head-https:mixdrop.age3nzwveprim63or6

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=UTF-8","connection":"keep-alive","server":"cloudflare","expires":"Thu, 19 Nov 1981 08:52:00 GMT","cache-control":"no-store, no-cache, must-revalidate","pragma":"no-cache","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","cf-cache-status":"DYNAMIC","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=%2FXbIAsQRSGxA0wnx8arRq5DMZd1Y0Ta8fD4BgmvhZtMHPbNqh67JS2Z93kneJdciD6Gr4Olhbu0n6fF4cH7ZVOZDvNz05S1QwA8%3D\"}]}","set-cookie":["PHPSESSID=rhtto1gok9odeq9p12dfble0qo; SameSite=None; Secure; Path=/"],"cf-ray":"943dad4c9d2f364b-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:mixdrop.agel7v73zqrfdj19z

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=UTF-8","connection":"keep-alive","server":"cloudflare","expires":"Thu, 19 Nov 1981 08:52:00 GMT","cache-control":"no-store, no-cache, must-revalidate","pragma":"no-cache","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","cf-cache-status":"DYNAMIC","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=CDKjYdtj2bDLLHYXqP%2BDiezHbUv3evtF3P6dLr8DBvwLnG8ZsDWDdw%2BDwSlwSklQIbIrh5g4yz9G7LSqAD3bvo4oR9tDhajV92I%3D\"}]}","set-cookie":["PHPSESSID=7j46e49jabt96ea6tu426cb88c; SameSite=None; Secure; Path=/"],"cf-ray":"943dad4c3acb9280-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:mixdrop.agevk196d6xfzwwo1

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=UTF-8","connection":"keep-alive","server":"cloudflare","expires":"Thu, 19 Nov 1981 08:52:00 GMT","cache-control":"no-store, no-cache, must-revalidate","pragma":"no-cache","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","cf-cache-status":"DYNAMIC","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=vR%2BavpQfik4lUW5zXJ8NR2skrj2BTHirUBPSy7%2FbUQMUtKaER%2Fgb6sFItzEZtbQ9t6rTSahW2ZGHWeaRsXES%2F%2BJ7uDpMoaITmHE%3D\"}]}","set-cookie":["PHPSESSID=fr7a9ikrus94vpbotbar1qe3ka; SameSite=None; Secure; Path=/"],"cf-ray":"943dad4c8c6ba5fa-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:mixdrop.agevn0wx308fq984q

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=UTF-8","connection":"keep-alive","server":"cloudflare","expires":"Thu, 19 Nov 1981 08:52:00 GMT","cache-control":"no-store, no-cache, must-revalidate","pragma":"no-cache","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","cf-cache-status":"DYNAMIC","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=5oLwXcL7W3j%2Bd2KfE3TD%2Bc5QSe6BYknpcuDOwWhtf9wgkGyLt69SIRM%2B4DP1R5dARDDXCUnZRa6GrXZRVMImKpnvNe9OQ%2BggMqI%3D\"}]}","set-cookie":["PHPSESSID=r1b5f4eugimpli63mralrsbej0; SameSite=None; Secure; Path=/"],"cf-ray":"943dad4b89569f35-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:mixdrop.agexokzmv61hrwe8o

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=UTF-8","connection":"keep-alive","server":"cloudflare","expires":"Thu, 19 Nov 1981 08:52:00 GMT","cache-control":"no-store, no-cache, must-revalidate","pragma":"no-cache","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","cf-cache-status":"DYNAMIC","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=UE%2FLNdFE9jYY64kQoujhEsYwVAHxsUKqS9vbB81HZiwLLkaokCRdQItgr7hlPwCsc4wGd2tZ6HMC17L8YELeZCi%2FOV8elCyEZmo%3D\"}]}","set-cookie":["PHPSESSID=56msn0nr16tnl8oitbrjtr25kr; SameSite=None; Secure; Path=/"],"cf-ray":"943dad4b894f2c29-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:netu.frembed.arte0DFgfkcXOsDP

@@ -0,0 +1 @@
+{"date":"Thu, 22 May 2025 16:26:44 GMT","content-type":"text/html; charset=utf-8","connection":"keep-alive","nel":"{\"report_to\":\"cf-nel\",\"success_fraction\":0.0,\"max_age\":604800}","vary":"Accept-Encoding","x-robots-tag":"'none, noindex, nofollow, noarchive, nosnippet, noodp, notranslate, noimageindex'","x-content-type-options":"nosniff","x-xss-protection":"1; mode=block;","p3p":"policyref=\"http://www.example.com/w3c/p3p.xml\", CP=\"CURa ADMa DEVa CONo HISa OUR IND DSP ALL COR\"","link":"<//netu.frembed.art>; rel=preconnect; crossorigin, <//global.stun.twilio.com>; rel=dns-prefetch; crossorigin, <//counter.yadro.ru>; rel=preconnect; crossorigin, <//imasdk.googleapis.com>; rel=preconnect; crossorigin, <//stun2.l.google.com>; rel=dns-prefetch; crossorigin, <//unpkg.com>; rel=preconnect; crossorigin, <//mc.yandex.ru>; rel=preconnect; crossorigin, <//cdn.jsdelivr.net>; rel=preconnect; crossorigin, <//wss.commentsmodule.com>; rel=dns-prefetch; crossorigin, <//www.gstatic.com>; rel=preconnect; crossorigin, <//imasdk.googleapis.com>; rel=preconnect; crossorigin, <//storage.googleapis.com>; rel=preconnect; crossorigin, <//www.google.com>; rel=preconnect; crossorigin,<//a.labadena.com>; rel=preconnect; crossorigin, <//deliver.vkcdnservice.com>; rel=preconnect; crossorigin,<//vkcdnservice.appspot.com.storage.googleapis.com>; rel=preconnect; crossorigin, <//www.google.com>; rel=preconnect; crossorigin, <//www.recaptcha.net>; rel=preconnect; crossorigin, <//cdnjs.cloudflare.com>; rel=preconnect; crossorigin","pragma":"no-cache","x-origin-location":"player","cache-control":"no-store, no-cache, must-revalidate, max-age=0, post-check=0, pre-check=0","report-to":"{\"group\":\"cf-nel\",\"max_age\":604800,\"endpoints\":[{\"url\":\"https://a.nel.cloudflare.com/report/v4?s=K%2FQQ%2F3J2cjuKJy0%2FLlwJRHNnsu9UnaDAz9hQ0m%2FYtTJAkSWvR%2FWyEclPsUZrNTteC2btE%2BE0yOCcKJAnGYI4XfeelWrU6G9sZ8XyiSsK\"}]}","server":"cloudflare","x-cache-status-inferno":"MISS","x-inferno-location":"player","x-inferno-limit-req":"PASSED","cf-cache-status":"DYNAMIC","cf-ray":"943dad4d2f92699b-FRA","alt-svc":"h3=\":443\"; ma=86400"}

src/utils/__fixtures__/Fetcher/head-https:streamtape.comeBjp2vjrdBxsK82.error

@@ -0,0 +1 @@
+{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}}

src/utils/__fixtures__/Fetcher/head-https:streamtape.comegjA1OQ4klyHxgJ.error

@@ -0,0 +1 @@
+{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}}

src/utils/__fixtures__/Fetcher/https:johnalwayssame.comecqy9oue7sv0g

@@ -1,23 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <title>Redirecting...</title>
-</head>
-<body>
-<script>
-        if (typeof localStorage !== 'undefined') {
-            const permanentToken = localStorage.getItem('permanentToken');
-            if (permanentToken) {
-                const currentUrl = new URL(window.location.href);
-                currentUrl.searchParams.set('permanentToken', permanentToken);
-                window.location.href = currentUrl.toString();
-            } else {
-                window.location.href = 'https://kellywhatcould.com/e/cqy9oue7sv0g';
-            }
-        } else {
-            window.location.href = 'https://kellywhatcould.com/e/cqy9oue7sv0g';
-        }
-    </script>
-</body>
-</html>

src/utils/__mocks__/Fetcher.ts

@@ -1,5 +1,5 @@
 import fs from 'node:fs';
-import Axios, { AxiosError, AxiosRequestConfig } from 'axios';
+import Axios, { AxiosError, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios';
 import slugify from 'slugify';
 import { Context } from '../../types';
 
@@ -16,7 +16,13 @@ export class Fetcher {
     return this.fixtureWrapper(path, async () => (await Axios.create().post(url.href, data, this.getConfig(config))).data);
   };
 
-  private readonly fixtureWrapper = async (path: string, callable: () => Promise<string>) => {
+  readonly head = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<RawAxiosResponseHeaders | AxiosResponseHeaders> => {
+    const path = `${__dirname}/../__fixtures__/Fetcher/head-${slugify(url.href)}`;
+
+    return JSON.parse(await this.fixtureWrapper(path, async () => JSON.stringify((await Axios.create().head(url.href, this.getConfig(config))).headers)));
+  };
+
+  private readonly fixtureWrapper = async (path: string, callable: () => Promise<string>): Promise<string> => {
     const errorPath = `${path}.error`;
 
     if (fs.existsSync(errorPath)) {