diff --git a/src/embed-extractor/Dropload.ts b/src/embed-extractor/Dropload.ts index 09acbe3..ca80d5c 100644 --- a/src/embed-extractor/Dropload.ts +++ b/src/embed-extractor/Dropload.ts @@ -20,7 +20,7 @@ export class Dropload implements EmbedExtractor { readonly extract = async (ctx: Context, url: URL, language: string) => { const normalizedUrl = url.toString().replace('/e/', '').replace('/embed-', '/'); - const html = await this.fetcher.text(ctx, normalizedUrl); + const html = await this.fetcher.text(ctx, new URL(normalizedUrl)); const heightMatch = html.match(/\d{3,}x(\d{3,}),/) as string[]; diff --git a/src/embed-extractor/SuperVideo.ts b/src/embed-extractor/SuperVideo.ts index 15275c2..d508f2d 100644 --- a/src/embed-extractor/SuperVideo.ts +++ b/src/embed-extractor/SuperVideo.ts @@ -20,7 +20,7 @@ export class SuperVideo implements EmbedExtractor { readonly extract = async (ctx: Context, url: URL, language: string) => { const normalizedUrl = url.toString().replace('/e/', '/').replace('/embed-', '/'); - const html = await this.fetcher.text(ctx, normalizedUrl); + const html = await this.fetcher.text(ctx, new URL(normalizedUrl)); const heightAndSizeMatch = html.match(/\d{3,}x(\d{3,}), ([\d.]+ ?[GM]B)/) as string[]; diff --git a/src/handler/KinoKiste.ts b/src/handler/KinoKiste.ts index 5da2b4a..b539f2a 100644 --- a/src/handler/KinoKiste.ts +++ b/src/handler/KinoKiste.ts @@ -48,13 +48,15 @@ export class KinoKiste implements Handler { ); }; - private fetchSeriesPageUrl = async (ctx: Context, imdbId: ImdbId): Promise => { - const html = await this.fetcher.text(ctx, `https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`); + private fetchSeriesPageUrl = async (ctx: Context, imdbId: ImdbId): Promise => { + const html = await this.fetcher.text(ctx, new URL(`https://kinokiste.live/serien/?do=search&subaction=search&story=${imdbId.id}`)); const $ = cheerio.load(html); - return $('.item-video a[href]:first') + const url = $('.item-video a[href]:first') .map((_i, el) => $(el).attr('href')) .get(0); + + return url !== undefined ? new URL(url) : url; }; } diff --git a/src/handler/MeineCloud.ts b/src/handler/MeineCloud.ts index 49011fb..9b4d7e8 100644 --- a/src/handler/MeineCloud.ts +++ b/src/handler/MeineCloud.ts @@ -26,7 +26,7 @@ export class MeineCloud implements Handler { return Promise.resolve([]); } - const html = await this.fetcher.text(ctx, `https://meinecloud.click/movie/${parseImdbId(id).id}`); + const html = await this.fetcher.text(ctx, new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`)); const $ = cheerio.load(html); diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index c7cdb3f..5863e1e 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -19,21 +19,23 @@ describe('fetch', () => { test('text throws if the response is not OK', async () => { mockedFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'some error happened' }); - await expect(fetcher.text(ctx, 'https://some-url.test')).rejects.toThrow(new Error('HTTP error: 500 - some error happened')); + await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toThrow(new Error('HTTP error: 500 - some error happened')); }); test('text passes successful response through setting headers', async () => { mockedFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('some text') }); - const responseText = await fetcher.text(ctx, 'https://some-url.test', { headers: { 'User-Agent': 'jest' } }); + const responseText = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } }); expect(responseText).toBe('some text'); expect(mockedFetch).toHaveBeenCalledWith( - 'https://some-url.test', + 'https://some-url.test/', { headers: { 'User-Agent': expect.not.stringMatching(/jest/), + 'Forwarded': 'for=127.0.0.1', 'X-Forwarded-For': '127.0.0.1', + 'X-Forwarded-Proto': 'https', 'X-Real-IP': '127.0.0.1', }, }, diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index c7fb5b1..c0097c6 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -17,17 +17,19 @@ export class Fetcher { this.ipUserAgentCache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h } - readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise => { - this.logger.info(`Fetch ${uriOrRequest}`); + readonly text = async (ctx: Context, url: URL, opts?: FetchOptions): Promise => { + this.logger.info(`Fetch ${url}`); const response = await this.fetch( - uriOrRequest, + url.href, { ...opts, headers: { ...opts?.headers, 'User-Agent': this.createUserAgentForIp(ctx.ip), + 'Forwarded': `for=${ctx.ip}`, 'X-Forwarded-For': ctx.ip, + 'X-Forwarded-Proto': url.protocol.slice(0, -1), 'X-Real-IP': ctx.ip, }, }, diff --git a/src/utils/__mocks__/Fetcher.ts b/src/utils/__mocks__/Fetcher.ts index 33cedd4..f387575 100644 --- a/src/utils/__mocks__/Fetcher.ts +++ b/src/utils/__mocks__/Fetcher.ts @@ -5,13 +5,13 @@ import slugify from 'slugify'; import { Context } from '../../types'; export class Fetcher { - readonly text = async (_ctx: Context, uriOrRequest: string, opts?: FetchOptions): Promise => { - const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(uriOrRequest)}`; + readonly text = async (_ctx: Context, url: URL, opts?: FetchOptions): Promise => { + const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(url.href)}`; if (fs.existsSync(path)) { return fs.readFileSync(path).toString(); } else { - const text = await (await makeFetchHappen.defaults()(uriOrRequest, opts)).text(); + const text = await (await makeFetchHappen.defaults()(url.href, opts)).text(); fs.writeFileSync(path, text);