From f11c591dc544f41720d5c9002bd5216449c60e3b Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 14 Jun 2025 13:44:32 +0200 Subject: [PATCH] chore: do not pass referer through context --- src/extractor/DoodStream.ts | 4 ++-- src/extractor/KinoGer.ts | 2 +- src/extractor/VidSrc.ts | 2 +- src/extractor/__snapshots__/DoodStream.test.ts.snap | 6 +++--- src/extractor/__snapshots__/KinoGer.test.ts.snap | 4 ++-- src/source/Frembed.ts | 2 +- src/source/KinoGer.ts | 2 +- src/source/Soaper.ts | 4 ++-- src/source/__snapshots__/Frembed.test.ts.snap | 8 -------- src/source/__snapshots__/KinoGer.test.ts.snap | 8 -------- src/source/__snapshots__/Soaper.test.ts.snap | 3 --- src/source/types.ts | 1 - src/types.ts | 1 - src/utils/Fetcher.test.ts | 13 ------------- src/utils/Fetcher.ts | 4 ---- src/utils/StreamResolver.ts | 6 +----- src/utils/__snapshots__/StreamResolver.test.ts.snap | 4 ++-- 17 files changed, 16 insertions(+), 58 deletions(-) diff --git a/src/extractor/DoodStream.ts b/src/extractor/DoodStream.ts index 273aa17..5fc9266 100644 --- a/src/extractor/DoodStream.ts +++ b/src/extractor/DoodStream.ts @@ -46,7 +46,7 @@ export class DoodStream extends Extractor { const mp4Url = new URL(`${baseUrl}${randomstring.generate(10)}?token=${token}&expiry=${Date.now()}`); - const mp4Head = await this.fetcher.head(ctx, mp4Url, { headers: { Referer: 'http://dood.to' } }); + const mp4Head = await this.fetcher.head(ctx, mp4Url, { headers: { Referer: url.origin } }); return [ { @@ -61,7 +61,7 @@ export class DoodStream extends Extractor { ...(height && { height }), }, requestHeaders: { - Referer: 'http://dood.to/', + Referer: url.origin, }, }, ]; diff --git a/src/extractor/KinoGer.ts b/src/extractor/KinoGer.ts index 4b032d0..cb7b419 100644 --- a/src/extractor/KinoGer.ts +++ b/src/extractor/KinoGer.ts @@ -50,7 +50,7 @@ export class KinoGer extends Extractor { ...(height && { height }), }, requestHeaders: { - Referer: ctx.referer?.href as string, + Referer: url.origin, }, }, ]; diff --git a/src/extractor/VidSrc.ts b/src/extractor/VidSrc.ts index a155081..d6bc1b5 100644 --- a/src/extractor/VidSrc.ts +++ b/src/extractor/VidSrc.ts @@ -46,7 +46,7 @@ export class VidSrc extends Extractor { } const m3u8Url = new URL(fileMatch[1] as string); - const height = await guessFromPlaylist(ctx, this.fetcher, m3u8Url, { noReferer: true }); + const height = await guessFromPlaylist(ctx, this.fetcher, m3u8Url); return { url: m3u8Url, diff --git a/src/extractor/__snapshots__/DoodStream.test.ts.snap b/src/extractor/__snapshots__/DoodStream.test.ts.snap index 5f9ced1..c68ada1 100644 --- a/src/extractor/__snapshots__/DoodStream.test.ts.snap +++ b/src/extractor/__snapshots__/DoodStream.test.ts.snap @@ -10,7 +10,7 @@ exports[`DoodStream can guess height from title 1`] = ` "title": "Black Mirror S04E02 FRENCH 720p WEB x264-RiPiT", }, "requestHeaders": { - "Referer": "http://dood.to/", + "Referer": "http://dood.to", }, "sourceId": "doodstream_fr", "ttl": 900000, @@ -29,7 +29,7 @@ exports[`DoodStream dood.to 1`] = ` "title": "des-teufels-bad-2024", }, "requestHeaders": { - "Referer": "http://dood.to/", + "Referer": "http://dood.to", }, "sourceId": "doodstream_de", "ttl": 900000, @@ -47,7 +47,7 @@ exports[`DoodStream doodster 1`] = ` "title": "Brokeback Mountain 2005", }, "requestHeaders": { - "Referer": "http://dood.to/", + "Referer": "http://dood.to", }, "sourceId": "doodstream_de", "ttl": 900000, diff --git a/src/extractor/__snapshots__/KinoGer.test.ts.snap b/src/extractor/__snapshots__/KinoGer.test.ts.snap index 7fca167..c2e8f92 100644 --- a/src/extractor/__snapshots__/KinoGer.test.ts.snap +++ b/src/extractor/__snapshots__/KinoGer.test.ts.snap @@ -9,7 +9,7 @@ exports[`KinoGer Blood & Sinners 1`] = ` "title": "Blood.and.Sinners.2025.mkv", }, "requestHeaders": { - "Referer": undefined, + "Referer": "https://kinoger.re", }, "sourceId": "kinoger_de", "ttl": 900000, @@ -28,7 +28,7 @@ exports[`KinoGer Dead City 1`] = ` "title": "the.walking.dead.dead.city.s02e05.german.dl.1080p.web.x264-wvf.mkv", }, "requestHeaders": { - "Referer": undefined, + "Referer": "https://kinoger.re", }, "sourceId": "kinoger_de", "ttl": 900000, diff --git a/src/source/Frembed.ts b/src/source/Frembed.ts index e1b0b28..e80ff74 100644 --- a/src/source/Frembed.ts +++ b/src/source/Frembed.ts @@ -36,6 +36,6 @@ export class Frembed implements Source { } } - return urls.map(url => ({ countryCode: CountryCode.fr, referer: apiUrl, title: `${json['title']} ${tmdbId.season}x${tmdbId.episode}`, url })); + return urls.map(url => ({ countryCode: CountryCode.fr, title: `${json['title']} ${tmdbId.season}x${tmdbId.episode}`, url })); }; } diff --git a/src/source/KinoGer.ts b/src/source/KinoGer.ts index 7df47f6..2a18244 100644 --- a/src/source/KinoGer.ts +++ b/src/source/KinoGer.ts @@ -40,7 +40,7 @@ export class KinoGer implements Source { return Array.from(html.matchAll(/\.show\(.*/g)) .map(showJsMatch => this.findEpisodeUrlInShowJs(showJsMatch[0], seasonIndex, episodeIndex)) .filter((url): url is URL => url !== undefined && !['kinoger.be', 'kinoger.ru'].includes(url.host)) - .map(url => ({ countryCode: CountryCode.de, referer: pageUrl, title, url })); + .map(url => ({ countryCode: CountryCode.de, title, url })); }; private readonly findEpisodeUrlInShowJs = (showJs: string, seasonIndex: number, episodeIndex: number): URL | undefined => { diff --git a/src/source/Soaper.ts b/src/source/Soaper.ts index c433be4..2ab9de0 100644 --- a/src/source/Soaper.ts +++ b/src/source/Soaper.ts @@ -44,12 +44,12 @@ export class Soaper implements Source { const episodeUrl = new URL(episodePageHref, this.baseUrl); const title = `${keyword} ${tmdbId.season}x${tmdbId.episode}`; - return [{ countryCode: CountryCode.en, referer: episodeUrl, title, url: episodeUrl }]; + return [{ countryCode: CountryCode.en, title, url: episodeUrl }]; } const title = `${keyword} (${year})`; - return [{ countryCode: CountryCode.en, referer: pageUrl, title, url: pageUrl }]; + return [{ countryCode: CountryCode.en, title, url: pageUrl }]; }; private readonly fetchPageUrl = async (ctx: Context, keyword: string, year: number, hrefPrefix: string): Promise => { diff --git a/src/source/__snapshots__/Frembed.test.ts.snap b/src/source/__snapshots__/Frembed.test.ts.snap index e9e4708..5a5d938 100644 --- a/src/source/__snapshots__/Frembed.test.ts.snap +++ b/src/source/__snapshots__/Frembed.test.ts.snap @@ -4,25 +4,21 @@ exports[`Frembed handle imdb black mirror s4e2 1`] = ` [ { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://do7go.com/e/dfx8me4un4ul", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://netu.fanstream.us/e/0DFgfkcXOsDP", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://johnalwayssame.com/e/cqy9oue7sv0g", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://ds2play.com/e/fzfvfq3ngig0", }, @@ -33,25 +29,21 @@ exports[`Frembed handle tmdb black mirror s4e2 1`] = ` [ { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://do7go.com/e/dfx8me4un4ul", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://netu.fanstream.us/e/0DFgfkcXOsDP", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://johnalwayssame.com/e/cqy9oue7sv0g", }, { "countryCode": "fr", - "referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb", "title": "Black Mirror 4x2", "url": "https://ds2play.com/e/fzfvfq3ngig0", }, diff --git a/src/source/__snapshots__/KinoGer.test.ts.snap b/src/source/__snapshots__/KinoGer.test.ts.snap index fbb5e0e..dfb569d 100644 --- a/src/source/__snapshots__/KinoGer.test.ts.snap +++ b/src/source/__snapshots__/KinoGer.test.ts.snap @@ -4,19 +4,16 @@ exports[`KinoGer handle imdb blood and sinners 1`] = ` [ { "countryCode": "de", - "referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html", "title": "Blood & Sinners (2025)", "url": "https://fsst.online/embed/900576/", }, { "countryCode": "de", - "referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html", "title": "Blood & Sinners (2025)", "url": "https://kinoger.pw/e/6zeREaMlrqREZPa", }, { "countryCode": "de", - "referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html", "title": "Blood & Sinners (2025)", "url": "https://kinoger.re/#ge5fhb", }, @@ -27,13 +24,11 @@ exports[`KinoGer handle imdb dead city s2e5 1`] = ` [ { "countryCode": "de", - "referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html", "title": "The Walking Dead: Dead City 2x5", "url": "https://supervideo.cc/e/bj6szat1pval", }, { "countryCode": "de", - "referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html", "title": "The Walking Dead: Dead City 2x5", "url": "https://kinoger.re/#x6tsx9", }, @@ -44,13 +39,11 @@ exports[`KinoGer handle imdb dead city s2e6 1`] = ` [ { "countryCode": "de", - "referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html", "title": "The Walking Dead: Dead City 2x6", "url": "https://fsst.online/embed/901994/", }, { "countryCode": "de", - "referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html", "title": "The Walking Dead: Dead City 2x6", "url": "https://kinoger.re/#ep1tcf", }, @@ -63,7 +56,6 @@ exports[`KinoGer handle no fsst via brokeback mountain 1`] = ` [ { "countryCode": "de", - "referer": "https://kinoger.com/stream/14781-brokeback-mountain-stream.html", "title": "Brokeback Mountain (2005)", "url": "https://dooodster.com/e/1cfcevn6dg8shrfvht22odxw2lty18hr", }, diff --git a/src/source/__snapshots__/Soaper.test.ts.snap b/src/source/__snapshots__/Soaper.test.ts.snap index 7d28805..90c1640 100644 --- a/src/source/__snapshots__/Soaper.test.ts.snap +++ b/src/source/__snapshots__/Soaper.test.ts.snap @@ -4,7 +4,6 @@ exports[`Soaper handle imdb black mirror s4e2 1`] = ` [ { "countryCode": "en", - "referer": "https://soaper.live/episode_5KDq78eGp1.html", "title": "Black Mirror 4x2", "url": "https://soaper.live/episode_5KDq78eGp1.html", }, @@ -15,7 +14,6 @@ exports[`Soaper handle imdb full metal jacket 1`] = ` [ { "countryCode": "en", - "referer": "https://soaper.live/movie_d8kdeypDY9.html", "title": "Full Metal Jacket (1987)", "url": "https://soaper.live/movie_d8kdeypDY9.html", }, @@ -26,7 +24,6 @@ exports[`Soaper handle tmdb black mirror s4e2 1`] = ` [ { "countryCode": "en", - "referer": "https://soaper.live/episode_5KDq78eGp1.html", "title": "Black Mirror 4x2", "url": "https://soaper.live/episode_5KDq78eGp1.html", }, diff --git a/src/source/types.ts b/src/source/types.ts index 9a324e9..ff01782 100644 --- a/src/source/types.ts +++ b/src/source/types.ts @@ -4,7 +4,6 @@ import { Id } from '../utils'; export interface SourceResult { countryCode: CountryCode; - referer?: URL; title?: string; url: URL; } diff --git a/src/types.ts b/src/types.ts index 40cc5f2..124ff96 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,7 +6,6 @@ export interface Context { id: string; ip: string; config: Config; - referer?: URL; } export type ManifestWithConfig = Manifest & { config: ManifestConfig[] }; diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index f90b93b..a03b5cd 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -29,7 +29,6 @@ describe('fetch', () => { 'Accept-Language': 'en', 'Forwarded': 'for=127.0.0.1', 'Priority': 'u=0', - 'Referer': 'https://some-url.test', 'User-Agent': 'node', 'X-Forwarded-For': '127.0.0.1', 'X-Forwarded-Proto': 'https', @@ -50,18 +49,6 @@ describe('fetch', () => { expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' }); }); - test('uses context referer', async () => { - fetchMock.get('https://some-referer-url.test/', 'some text'); - - await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-referer-url.test/')); - - expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({ - headers: { - Referer: 'https://example.com/foo/bar', - }, - }); - }); - test('converts 404 to custom NotFoundError', async () => { fetchMock.get('https://some-404-url.test/', 404); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index eb4b493..794f302 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -42,7 +42,6 @@ interface FlareSolverrResult { export type CustomRequestInit = RequestInit & { noFlareSolverr?: boolean; - noReferer?: boolean; queueLimit?: number; queueErrorLimit?: number; timeout?: number; @@ -79,8 +78,6 @@ export class Fetcher { private getInit(ctx: Context, url: URL, init?: CustomRequestInit): CustomRequestInit { const cookieString = this.cookieJar.getCookieStringSync(url.href); - const noReferer = init?.noReferer ?? false; - return { ...init, headers: { @@ -89,7 +86,6 @@ export class Fetcher { ...(cookieString && { Cookie: cookieString }), 'Forwarded': `for=${ctx.ip}`, 'Priority': 'u=0', - ...(!noReferer && { Referer: `${ctx.referer?.href ?? url.origin}` }), 'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'node', 'X-Forwarded-For': ctx.ip, 'X-Forwarded-Proto': url.protocol.slice(0, -1), diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index 9980820..4ca8aef 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -51,11 +51,7 @@ export class StreamResolver { this.logger.info(`${handler.id} returned ${handleResults.length} urls`, ctx); const handlerUrlResults = await Promise.all( - handleResults.map(async ({ countryCode, referer, title, url }) => { - const newCtx = { ...ctx, ...(referer && { referer }) }; - - return await this.extractorRegistry.handle(newCtx, url, countryCode, title); - }), + handleResults.map(({ countryCode, title, url }) => this.extractorRegistry.handle(ctx, url, countryCode, title)), ); urlResults.push(...handlerUrlResults.flat()); diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index fc5c7d2..5a6aa9b 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -200,7 +200,7 @@ exports[`resolve returns sorted results 1`] = ` "notWebReady": true, "proxyHeaders": { "request": { - "Referer": "http://dood.to/", + "Referer": "http://dood.to", }, }, "videoSize": 791702409, @@ -218,7 +218,7 @@ exports[`resolve returns sorted results 1`] = ` "notWebReady": true, "proxyHeaders": { "request": { - "Referer": "http://dood.to/", + "Referer": "http://dood.to", }, }, },