feat: introduce Context and use it to set X-Forwarded-For in requests

src/embed-extractor/Dropload.ts

@@ -1,5 +1,6 @@
 import { EmbedExtractor } from './types';
 import { extractUrlFromPacked, Fetcher, iso2ToFlag, scanFromResolution } from '../utils';
+import { Context } from '../types';
 
 export class Dropload implements EmbedExtractor {
   readonly id = 'dropload';
@@ -14,9 +15,9 @@ export class Dropload implements EmbedExtractor {
 
   readonly supports = (url: URL): boolean => null !== url.host.match(/dropload/);
 
-  readonly extract = async (url: URL, language: string) => {
+  readonly extract = async (ctx: Context, url: URL, language: string) => {
     const normalizedUrl = url.toString().replace('/e/', '').replace('/embed-', '/');
-    const html = await this.fetcher.text(normalizedUrl);
+    const html = await this.fetcher.text(ctx, normalizedUrl);
 
     const resolution = scanFromResolution((html.match(/(\d{3,}x\d{3,}),/) as string[])[1] as string);
 

src/embed-extractor/EmbedExtractors.test.ts

@@ -1,9 +1,13 @@
 import { EmbedExtractors } from './EmbedExtractors';
+import { Context } from '../types';
 
 describe('EmbedExtractors', () => {
+  const ctx: Context = { ip: '127.0.0.1' };
+
   test('throws when no embed extractor can be found', () => {
     const embedExtractors = new EmbedExtractors([]);
 
-    expect(embedExtractors.handle(new URL('https://some-url.test'), 'en')).rejects.toThrow('No embed extractor found that supports url https://some-url.test');
+    expect(embedExtractors.handle(ctx, new URL('https://some-url.test'), 'en'))
+      .rejects.toThrow('No embed extractor found that supports url https://some-url.test');
   });
 });

src/embed-extractor/EmbedExtractors.ts

@@ -1,5 +1,5 @@
 import { EmbedExtractor } from './types';
-import { StreamWithMeta } from '../types';
+import { Context, StreamWithMeta } from '../types';
 
 export class EmbedExtractors {
   private readonly embedExtractors: EmbedExtractor[];
@@ -8,13 +8,13 @@ export class EmbedExtractors {
     this.embedExtractors = embedExtractors;
   }
 
-  readonly handle = async (url: URL, language: string): Promise<StreamWithMeta> => {
+  readonly handle = async (ctx: Context, url: URL, language: string): Promise<StreamWithMeta> => {
     const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url));
 
     if (undefined === embedExtractor) {
       throw new Error(`No embed extractor found that supports url ${url}`);
     }
 
-    return embedExtractor.extract(url, language);
+    return embedExtractor.extract(ctx, url, language);
   };
 }

src/embed-extractor/SuperVideo.ts

@@ -1,5 +1,6 @@
 import { EmbedExtractor } from './types';
 import { extractUrlFromPacked, Fetcher, iso2ToFlag, scanFromResolution } from '../utils';
+import { Context } from '../types';
 
 export class SuperVideo implements EmbedExtractor {
   readonly id = 'supervideo';
@@ -14,9 +15,9 @@ export class SuperVideo implements EmbedExtractor {
 
   readonly supports = (url: URL): boolean => null !== url.host.match(/supervideo/);
 
-  readonly extract = async (url: URL, language: string) => {
+  readonly extract = async (ctx: Context, url: URL, language: string) => {
     const normalizedUrl = url.toString().replace('/e/', '/').replace('/embed-', '/');
-    const html = await this.fetcher.text(normalizedUrl);
+    const html = await this.fetcher.text(ctx, normalizedUrl);
 
     const resolutionAndSizeMatch = html.match(/(\d{3,}x\d{3,}), ([\d.]+) ?([GM]B)/) as string[];
     const resolution = scanFromResolution(resolutionAndSizeMatch[1] as string);

src/embed-extractor/types.ts

@@ -1,4 +1,4 @@
-import { StreamWithMeta } from '../types';
+import { Context, StreamWithMeta } from '../types';
 
 export interface EmbedExtractor {
   readonly id: string;
@@ -7,5 +7,5 @@ export interface EmbedExtractor {
 
   readonly supports: (url: URL) => boolean;
 
-  readonly extract: (url: URL, language: string) => Promise<StreamWithMeta>;
+  readonly extract: (ctx: Context, url: URL, language: string) => Promise<StreamWithMeta>;
 }

src/handler/KinoKiste.test.ts

@@ -1,27 +1,29 @@
 import { KinoKiste } from './KinoKiste';
 import { Dropload, EmbedExtractors, SuperVideo } from '../embed-extractor';
 import { Fetcher } from '../utils';
+import { Context } from '../types';
 jest.mock('../utils/Fetcher');
 
 // @ts-expect-error No constructor args needed
 const fetcher = new Fetcher();
 const kinokiste = new KinoKiste(fetcher, new EmbedExtractors([new Dropload(fetcher), new SuperVideo(fetcher)]));
+const ctx: Context = { ip: '127.0.0.1' };
 
 describe('KinoKiste', () => {
   test('does not handle non imdb series', async () => {
-    const streams = await kinokiste.handle('kitsu:123');
+    const streams = await kinokiste.handle(ctx, 'kitsu:123');
 
     expect(streams).toHaveLength(0);
   });
 
   test('handles non-existent series gracefully', async () => {
-    const streams = await kinokiste.handle('tt12345678:1:1');
+    const streams = await kinokiste.handle(ctx, 'tt12345678:1:1');
 
     expect(streams).toHaveLength(0);
   });
 
   test('handle imdb black mirror s2e4', async () => {
-    const streams = await kinokiste.handle('tt2085059:2:4');
+    const streams = await kinokiste.handle(ctx, 'tt2085059:2:4');
 
     expect(streams).toHaveLength(2);
     expect(streams[0]).toStrictEqual({

src/handler/KinoKiste.ts

@@ -2,6 +2,7 @@ import * as cheerio from 'cheerio';
 import { Handler } from './types';
 import { ImdbId, fulfillAllPromises, parseImdbId, Fetcher } from '../utils';
 import { EmbedExtractors } from '../embed-extractor';
+import { Context } from '../types';
 
 export class KinoKiste implements Handler {
   readonly id = 'kinokiste';
@@ -20,19 +21,19 @@ export class KinoKiste implements Handler {
     this.embedExtractors = embedExtractors;
   }
 
-  readonly handle = async (id: string) => {
+  readonly handle = async (ctx: Context, id: string) => {
     if (!id.startsWith('tt')) {
       return Promise.resolve([]);
     }
 
     const imdbId = parseImdbId(id);
 
-    const seriesPageUrl = await this.fetchSeriesPageUrl(imdbId);
+    const seriesPageUrl = await this.fetchSeriesPageUrl(ctx, imdbId);
     if (!seriesPageUrl) {
       return Promise.resolve([]);
     }
 
-    const html = await this.fetcher.text(seriesPageUrl);
+    const html = await this.fetcher.text(ctx, seriesPageUrl);
 
     const $ = cheerio.load(html);
 
@@ -43,12 +44,12 @@ export class KinoKiste implements Handler {
         .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
         .toArray()
         .filter(embedUrl => embedUrl.host.match(/(dropload|supervideo)/))
-        .map(embedUrl => this.embedExtractors.handle(embedUrl, 'de')),
+        .map(embedUrl => this.embedExtractors.handle(ctx, embedUrl, 'de')),
     );
   };
 
-  private fetchSeriesPageUrl = async (imdbId: ImdbId): Promise<string | undefined> => {
-    const html = await this.fetcher.text(`https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`);
+  private fetchSeriesPageUrl = async (ctx: Context, imdbId: ImdbId): Promise<string | undefined> => {
+    const html = await this.fetcher.text(ctx, `https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`);
 
     const $ = cheerio.load(html);
 

src/handler/MeineCloud.test.ts

@@ -1,27 +1,29 @@
 import { MeineCloud } from './MeineCloud';
 import { Fetcher } from '../utils';
 import { Dropload, EmbedExtractors, SuperVideo } from '../embed-extractor';
+import { Context } from '../types';
 jest.mock('../utils/Fetcher');
 
 // @ts-expect-error No constructor args needed
 const fetcher = new Fetcher();
 const meinecloud = new MeineCloud(fetcher, new EmbedExtractors([new Dropload(fetcher), new SuperVideo(fetcher)]));
+const ctx: Context = { ip: '127.0.0.1' };
 
 describe('MeineCloud', () => {
   test('does not handle non imdb movies', async () => {
-    const streams = await meinecloud.handle('kitsu:123');
+    const streams = await meinecloud.handle(ctx, 'kitsu:123');
 
     expect(streams).toHaveLength(0);
   });
 
   test('handles non-existent movies gracefully', async () => {
-    const streams = await meinecloud.handle('tt12345678');
+    const streams = await meinecloud.handle(ctx, 'tt12345678');
 
     expect(streams).toHaveLength(0);
   });
 
   test('handle imdb the devil\'s bath', async () => {
-    const streams = await meinecloud.handle('tt29141112');
+    const streams = await meinecloud.handle(ctx, 'tt29141112');
 
     expect(streams).toHaveLength(2);
     expect(streams[0]).toStrictEqual({

src/handler/MeineCloud.ts

@@ -2,6 +2,7 @@ import * as cheerio from 'cheerio';
 import { Handler } from './types';
 import { Fetcher, fulfillAllPromises, parseImdbId } from '../utils';
 import { EmbedExtractors } from '../embed-extractor';
+import { Context } from '../types';
 
 export class MeineCloud implements Handler {
   readonly id = 'meinecloud';
@@ -20,12 +21,12 @@ export class MeineCloud implements Handler {
     this.embedExtractors = embedExtractors;
   }
 
-  readonly handle = async (id: string) => {
+  readonly handle = async (ctx: Context, id: string) => {
     if (!id.startsWith('tt')) {
       return Promise.resolve([]);
     }
 
-    const html = await this.fetcher.text(`https://meinecloud.click/movie/${parseImdbId(id).id}`);
+    const html = await this.fetcher.text(ctx, `https://meinecloud.click/movie/${parseImdbId(id).id}`);
 
     const $ = cheerio.load(html);
 
@@ -34,7 +35,7 @@ export class MeineCloud implements Handler {
         .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
         .toArray()
         .filter(embedUrl => embedUrl.host.match(/(dropload|supervideo)/))
-        .map(embedUrl => this.embedExtractors.handle(embedUrl, 'de')),
+        .map(embedUrl => this.embedExtractors.handle(ctx, embedUrl, 'de')),
     );
   };
 }

src/handler/types.ts

@@ -1,4 +1,4 @@
-import { StreamWithMeta } from '../types';
+import { Context, StreamWithMeta } from '../types';
 
 export interface Handler {
   readonly id: string;
@@ -9,5 +9,5 @@ export interface Handler {
 
   readonly languages: string[];
 
-  readonly handle: (id: string) => Promise<StreamWithMeta[]>;
+  readonly handle: (ctx: Context, id: string) => Promise<StreamWithMeta[]>;
 }

src/index.ts

@@ -9,6 +9,7 @@ import fs from 'node:fs';
 import * as os from 'node:os';
 
 const addon = express();
+addon.set('trust proxy', true);
 
 const fetcher = new Fetcher(makeFetchHappen.defaults({
   cachePath: `${fs.realpathSync(os.tmpdir())}/webstreamr`,
@@ -100,7 +101,7 @@ addon.get('/:config/stream/:type/:id.json', async function (req: Request, res: R
       return;
     }
 
-    const handlerStreams = await handler.handle(id);
+    const handlerStreams = await handler.handle({ ip: req.ip as string }, id);
     logInfo(`${handler.id} returned ${handlerStreams.length} streams`);
 
     streams.push(...handlerStreams);

src/types.ts

@@ -1,5 +1,7 @@
 import { Manifest, ManifestConfig, Stream } from 'stremio-addon-sdk';
 
+export interface Context { ip: string }
+
 export type ManifestWithConfig = Manifest & { config: ManifestConfig[] };
 
 export type Config = Record<string, string>;

src/utils/Fetcher.test.ts

@@ -1,5 +1,6 @@
 import { Fetcher } from './Fetcher';
 import makeFetchHappen from 'make-fetch-happen';
+import { Context } from '../types';
 
 global.console = {
   ...console,
@@ -18,17 +19,23 @@ jest.mock('make-fetch-happen', () => ({
 const fetcher = new Fetcher(makeFetchHappen.defaults());
 
 describe('fetch', () => {
+  const ctx: Context = { ip: '127.0.0.1' };
+
   test('text throws if the response is not OK', async () => {
     mockedFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'some error happened' });
 
-    await expect(fetcher.text('https://some-url.test')).rejects.toThrow(new Error('HTTP error: 500 - some error happened'));
+    await expect(fetcher.text(ctx, 'https://some-url.test')).rejects.toThrow(new Error('HTTP error: 500 - some error happened'));
   });
 
-  test('text passes response through if it is OK', async () => {
+  test('text passes successful response through setting headers', async () => {
     mockedFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('some text') });
 
-    const responseText = await fetcher.text('https://some-url.test');
+    const responseText = await fetcher.text(ctx, 'https://some-url.test', { headers: { 'User-Agent': 'jest' } });
 
     expect(responseText).toBe('some text');
+    expect(mockedFetch).toHaveBeenCalledWith(
+      'https://some-url.test',
+      { headers: { 'User-Agent': 'jest', 'X-Forwarded-For': '127.0.0.1' } },
+    );
   });
 });

src/utils/Fetcher.ts

@@ -1,5 +1,6 @@
 import { FetchInterface, FetchOptions } from 'make-fetch-happen';
 import { logInfo } from './log';
+import { Context } from '../types';
 
 export class Fetcher {
   private readonly fetch: FetchInterface;
@@ -8,10 +9,20 @@ export class Fetcher {
     this.fetch = fetch;
   }
 
-  readonly text = async (uriOrRequest: string | Request, opts?: FetchOptions): Promise<string> => {
+  readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise<string> => {
     logInfo(`Fetch ${uriOrRequest}`);
 
-    const response = await this.fetch(uriOrRequest, opts);
+    const response = await this.fetch(
+      uriOrRequest,
+      {
+        ...opts,
+        headers: {
+          ...opts?.headers,
+          'X-Forwarded-For': ctx.ip,
+        },
+      },
+    );
+
     if (!response.ok) {
       throw new Error(`HTTP error: ${response.status} - ${response.statusText}`);
     }

src/utils/__mocks__/Fetcher.ts

@@ -2,9 +2,10 @@ import { FetchOptions } from 'make-fetch-happen';
 import makeFetchHappen from 'make-fetch-happen';
 import fs from 'node:fs';
 import slugify from 'slugify';
+import { Context } from '../../types';
 
 export class Fetcher {
-  readonly text = async (uriOrRequest: string, opts?: FetchOptions): Promise<string> => {
+  readonly text = async (_ctx: Context, uriOrRequest: string, opts?: FetchOptions): Promise<string> => {
     const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(uriOrRequest)}`;
 
     if (fs.existsSync(path)) {