feat(fetcher): work only with URL objects, set 2 more proxy headers

src/embed-extractor/Dropload.ts

@@ -20,7 +20,7 @@ export class Dropload implements EmbedExtractor {
 
   readonly extract = async (ctx: Context, url: URL, language: string) => {
     const normalizedUrl = url.toString().replace('/e/', '').replace('/embed-', '/');
-    const html = await this.fetcher.text(ctx, normalizedUrl);
+    const html = await this.fetcher.text(ctx, new URL(normalizedUrl));
 
     const heightMatch = html.match(/\d{3,}x(\d{3,}),/) as string[];
 

src/embed-extractor/SuperVideo.ts

@@ -20,7 +20,7 @@ export class SuperVideo implements EmbedExtractor {
 
   readonly extract = async (ctx: Context, url: URL, language: string) => {
     const normalizedUrl = url.toString().replace('/e/', '/').replace('/embed-', '/');
-    const html = await this.fetcher.text(ctx, normalizedUrl);
+    const html = await this.fetcher.text(ctx, new URL(normalizedUrl));
 
     const heightAndSizeMatch = html.match(/\d{3,}x(\d{3,}), ([\d.]+ ?[GM]B)/) as string[];
 

src/handler/KinoKiste.ts

@@ -48,13 +48,15 @@ export class KinoKiste implements Handler {
     );
   };
 
-  private fetchSeriesPageUrl = async (ctx: Context, imdbId: ImdbId): Promise<string | undefined> => {
-    const html = await this.fetcher.text(ctx, `https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`);
+  private fetchSeriesPageUrl = async (ctx: Context, imdbId: ImdbId): Promise<URL | undefined> => {
+    const html = await this.fetcher.text(ctx, new URL(`https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`));
 
     const $ = cheerio.load(html);
 
-    return $('.item-video a[href]:first')
+    const url = $('.item-video a[href]:first')
       .map((_i, el) => $(el).attr('href'))
       .get(0);
+
+    return url !== undefined ? new URL(url) : url;
   };
 }

src/handler/MeineCloud.ts

@@ -26,7 +26,7 @@ export class MeineCloud implements Handler {
       return Promise.resolve([]);
     }
 
-    const html = await this.fetcher.text(ctx, `https://meinecloud.click/movie/${parseImdbId(id).id}`);
+    const html = await this.fetcher.text(ctx, new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`));
 
     const $ = cheerio.load(html);
 

src/utils/Fetcher.test.ts

@@ -19,21 +19,23 @@ describe('fetch', () => {
   test('text throws if the response is not OK', async () => {
     mockedFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'some error happened' });
 
-    await expect(fetcher.text(ctx, 'https://some-url.test')).rejects.toThrow(new Error('HTTP error: 500 - some error happened'));
+    await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toThrow(new Error('HTTP error: 500 - some error happened'));
   });
 
   test('text passes successful response through setting headers', async () => {
     mockedFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('some text') });
 
-    const responseText = await fetcher.text(ctx, 'https://some-url.test', { headers: { 'User-Agent': 'jest' } });
+    const responseText = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
 
     expect(responseText).toBe('some text');
     expect(mockedFetch).toHaveBeenCalledWith(
-      'https://some-url.test',
+      'https://some-url.test/',
       {
         headers: {
           'User-Agent': expect.not.stringMatching(/jest/),
+          'Forwarded': 'for=127.0.0.1',
           'X-Forwarded-For': '127.0.0.1',
+          'X-Forwarded-Proto': 'https',
           'X-Real-IP': '127.0.0.1',
         },
       },

src/utils/Fetcher.ts

@@ -17,17 +17,19 @@ export class Fetcher {
     this.ipUserAgentCache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h
   }
 
-  readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise<string> => {
-    this.logger.info(`Fetch ${uriOrRequest}`);
+  readonly text = async (ctx: Context, url: URL, opts?: FetchOptions): Promise<string> => {
+    this.logger.info(`Fetch ${url}`);
 
     const response = await this.fetch(
-      uriOrRequest,
+      url.href,
       {
         ...opts,
         headers: {
           ...opts?.headers,
           'User-Agent': this.createUserAgentForIp(ctx.ip),
+          'Forwarded': `for=${ctx.ip}`,
           'X-Forwarded-For': ctx.ip,
+          'X-Forwarded-Proto': url.protocol.slice(0, -1),
           'X-Real-IP': ctx.ip,
         },
       },

src/utils/__mocks__/Fetcher.ts

@@ -5,13 +5,13 @@ import slugify from 'slugify';
 import { Context } from '../../types';
 
 export class Fetcher {
-  readonly text = async (_ctx: Context, uriOrRequest: string, opts?: FetchOptions): Promise<string> => {
-    const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(uriOrRequest)}`;
+  readonly text = async (_ctx: Context, url: URL, opts?: FetchOptions): Promise<string> => {
+    const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(url.href)}`;
 
     if (fs.existsSync(path)) {
       return fs.readFileSync(path).toString();
     } else {
-      const text = await (await makeFetchHappen.defaults()(uriOrRequest, opts)).text();
+      const text = await (await makeFetchHappen.defaults()(url.href, opts)).text();
 
       fs.writeFileSync(path, text);