From 02b73b41d7ea08f299907510cfe88daa7ad25d9c Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Thu, 29 May 2025 17:36:43 +0000 Subject: [PATCH] chore(fetcher): use more realistic referer/origin header values --- src/handler/CineHDPlus.ts | 2 +- src/handler/Eurostreaming.ts | 2 +- src/handler/Frembed.ts | 7 +++++-- src/handler/FrenchCloud.ts | 5 +++-- src/handler/KinoKiste.ts | 2 +- src/handler/MeineCloud.ts | 5 +++-- src/handler/MostraGuarda.ts | 5 +++-- src/handler/VerHdLink.ts | 5 +++-- src/types.ts | 1 + src/utils/Fetcher.test.ts | 17 +++++++++++++++++ src/utils/Fetcher.ts | 7 +++++-- 11 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/handler/CineHDPlus.ts b/src/handler/CineHDPlus.ts index e52641a..4a0fed4 100644 --- a/src/handler/CineHDPlus.ts +++ b/src/handler/CineHDPlus.ts @@ -51,7 +51,7 @@ export class CineHDPlus implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/cinehdplus/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode, title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode, title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), ); }; diff --git a/src/handler/Eurostreaming.ts b/src/handler/Eurostreaming.ts index e41b5c9..06e0c5d 100644 --- a/src/handler/Eurostreaming.ts +++ b/src/handler/Eurostreaming.ts @@ -49,7 +49,7 @@ export class Eurostreaming implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/eurostreaming/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'it', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode: 'it', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), ); }; diff --git a/src/handler/Frembed.ts b/src/handler/Frembed.ts index f57a255..b4dcc83 100644 --- a/src/handler/Frembed.ts +++ b/src/handler/Frembed.ts @@ -28,7 +28,8 @@ export class Frembed implements Handler { const tmdbId = await getTmdbIdFromImdbId(ctx, this.fetcher, parseImdbId(id)); - const response = await this.apiCall(ctx, new URL(`https://frembed.club/api/series?id=${tmdbId.id}&sa=${tmdbId.series}&epi=${tmdbId.episode}&idType=tmdb`)); + const apiUrl = new URL(`https://frembed.club/api/series?id=${tmdbId.id}&sa=${tmdbId.series}&epi=${tmdbId.episode}&idType=tmdb`); + const response = await this.apiCall(ctx, apiUrl); if (!response) { return []; } @@ -46,7 +47,9 @@ export class Frembed implements Handler { } } - return Promise.all(urls.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'fr', title: `${json['title']} ${tmdbId.series}x${tmdbId.episode}` }))); + return Promise.all( + urls.map(url => this.extractorRegistry.handle({ ...ctx, referer: apiUrl }, url, { countryCode: 'fr', title: `${json['title']} ${tmdbId.series}x${tmdbId.episode}` })), + ); }; private readonly apiCall = async (ctx: Context, url: URL): Promise => { diff --git a/src/handler/FrenchCloud.ts b/src/handler/FrenchCloud.ts index 2c9e707..cc2bac5 100644 --- a/src/handler/FrenchCloud.ts +++ b/src/handler/FrenchCloud.ts @@ -26,7 +26,8 @@ export class FrenchCloud implements Handler { return []; } - const html = await this.fetcher.text(ctx, new URL(`https://frenchcloud.cam/movie/${parseImdbId(id).id}`)); + const pageUrl = new URL(`https://frenchcloud.cam/movie/${parseImdbId(id).id}`); + const html = await this.fetcher.text(ctx, pageUrl); const $ = cheerio.load(html); @@ -35,7 +36,7 @@ export class FrenchCloud implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/frenchcloud/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'fr' })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'fr' })), ); }; } diff --git a/src/handler/KinoKiste.ts b/src/handler/KinoKiste.ts index 307bdf0..ceae6f3 100644 --- a/src/handler/KinoKiste.ts +++ b/src/handler/KinoKiste.ts @@ -46,7 +46,7 @@ export class KinoKiste implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/kinokiste/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'de', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode: 'de', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })), ); }; diff --git a/src/handler/MeineCloud.ts b/src/handler/MeineCloud.ts index 5607a61..1528173 100644 --- a/src/handler/MeineCloud.ts +++ b/src/handler/MeineCloud.ts @@ -26,7 +26,8 @@ export class MeineCloud implements Handler { return []; } - const html = await this.fetcher.text(ctx, new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`)); + const pageUrl = new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`); + const html = await this.fetcher.text(ctx, pageUrl); const $ = cheerio.load(html); @@ -35,7 +36,7 @@ export class MeineCloud implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/meinecloud/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'de' })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'de' })), ); }; } diff --git a/src/handler/MostraGuarda.ts b/src/handler/MostraGuarda.ts index 4518991..10aaedf 100644 --- a/src/handler/MostraGuarda.ts +++ b/src/handler/MostraGuarda.ts @@ -26,7 +26,8 @@ export class MostraGuarda implements Handler { return []; } - const html = await this.fetcher.text(ctx, new URL(`https://mostraguarda.stream/movie/${parseImdbId(id).id}`)); + const pageUrl = new URL(`https://mostraguarda.stream/movie/${parseImdbId(id).id}`); + const html = await this.fetcher.text(ctx, pageUrl); const $ = cheerio.load(html); @@ -35,7 +36,7 @@ export class MostraGuarda implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/mostraguarda/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'it' })), + .map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'it' })), ); }; } diff --git a/src/handler/VerHdLink.ts b/src/handler/VerHdLink.ts index 1023d95..a242bbb 100644 --- a/src/handler/VerHdLink.ts +++ b/src/handler/VerHdLink.ts @@ -26,7 +26,8 @@ export class VerHdLink implements Handler { return []; } - const html = await this.fetcher.text(ctx, new URL(`https://verhdlink.cam/movie/${parseImdbId(id).id}`)); + const pageUrl = new URL(`https://verhdlink.cam/movie/${parseImdbId(id).id}`); + const html = await this.fetcher.text(ctx, pageUrl); const $ = cheerio.load(html); @@ -46,7 +47,7 @@ export class VerHdLink implements Handler { .map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://'))) .toArray() .filter(url => !url.host.match(/verhdlink/)) - .map(url => this.extractorRegistry.handle(ctx, url, { countryCode })); + .map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode })); }), ); }; diff --git a/src/types.ts b/src/types.ts index a905d3b..c75386a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,6 +4,7 @@ export interface Context { id: string; ip: string; config: Config; + referer?: URL; } export type ManifestWithConfig = Manifest & { config: ManifestConfig[] }; diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index a51fa81..9d3ebc4 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -119,6 +119,23 @@ describe('fetch', () => { ); }); + test('uses context referer', async () => { + axiosMock.onGet().reply(200, 'some text'); + const axiosSpy = jest.spyOn(axios, 'get'); + + await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-url.test/')); + + expect(axiosSpy).toHaveBeenCalledWith( + 'https://some-url.test/', + expect.objectContaining({ + headers: expect.objectContaining({ + Origin: 'https://example.com', + Referer: 'https://example.com/foo/bar', + }), + }), + ); + }); + test('converts 404 to custom NotFoundError', async () => { axiosMock.onGet().reply(404); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 106bcba..43ae271 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -35,14 +35,17 @@ export class Fetcher { }; private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => { + const origin = ctx.referer?.origin ?? url.origin; + const referer = ctx.referer?.href ?? url.origin; + return { responseType: 'text', timeout: 15000, ...config, headers: { 'Forwarded': `for=${ctx.ip}`, - 'Origin': `${url.origin}`, - 'Referer': `${url.origin}`, + 'Origin': `${origin}`, + 'Referer': `${referer}`, 'User-Agent': this.createUserAgentForIp(ctx.ip), 'X-Forwarded-For': ctx.ip, 'X-Forwarded-Proto': url.protocol.slice(0, -1),