chore: do not pass referer through context

This commit is contained in:
WebStreamr 2025-06-14 13:44:32 +02:00
parent 2853bff284
commit f11c591dc5
No known key found for this signature in database
17 changed files with 16 additions and 58 deletions

View file

@ -46,7 +46,7 @@ export class DoodStream extends Extractor {
const mp4Url = new URL(`${baseUrl}${randomstring.generate(10)}?token=${token}&expiry=${Date.now()}`);
const mp4Head = await this.fetcher.head(ctx, mp4Url, { headers: { Referer: 'http://dood.to' } });
const mp4Head = await this.fetcher.head(ctx, mp4Url, { headers: { Referer: url.origin } });
return [
{
@ -61,7 +61,7 @@ export class DoodStream extends Extractor {
...(height && { height }),
},
requestHeaders: {
Referer: 'http://dood.to/',
Referer: url.origin,
},
},
];

View file

@ -50,7 +50,7 @@ export class KinoGer extends Extractor {
...(height && { height }),
},
requestHeaders: {
Referer: ctx.referer?.href as string,
Referer: url.origin,
},
},
];

View file

@ -46,7 +46,7 @@ export class VidSrc extends Extractor {
}
const m3u8Url = new URL(fileMatch[1] as string);
const height = await guessFromPlaylist(ctx, this.fetcher, m3u8Url, { noReferer: true });
const height = await guessFromPlaylist(ctx, this.fetcher, m3u8Url);
return {
url: m3u8Url,

View file

@ -10,7 +10,7 @@ exports[`DoodStream can guess height from title 1`] = `
"title": "Black Mirror S04E02 FRENCH 720p WEB x264-RiPiT",
},
"requestHeaders": {
"Referer": "http://dood.to/",
"Referer": "http://dood.to",
},
"sourceId": "doodstream_fr",
"ttl": 900000,
@ -29,7 +29,7 @@ exports[`DoodStream dood.to 1`] = `
"title": "des-teufels-bad-2024",
},
"requestHeaders": {
"Referer": "http://dood.to/",
"Referer": "http://dood.to",
},
"sourceId": "doodstream_de",
"ttl": 900000,
@ -47,7 +47,7 @@ exports[`DoodStream doodster 1`] = `
"title": "Brokeback Mountain 2005",
},
"requestHeaders": {
"Referer": "http://dood.to/",
"Referer": "http://dood.to",
},
"sourceId": "doodstream_de",
"ttl": 900000,

View file

@ -9,7 +9,7 @@ exports[`KinoGer Blood & Sinners 1`] = `
"title": "Blood.and.Sinners.2025.mkv",
},
"requestHeaders": {
"Referer": undefined,
"Referer": "https://kinoger.re",
},
"sourceId": "kinoger_de",
"ttl": 900000,
@ -28,7 +28,7 @@ exports[`KinoGer Dead City 1`] = `
"title": "the.walking.dead.dead.city.s02e05.german.dl.1080p.web.x264-wvf.mkv",
},
"requestHeaders": {
"Referer": undefined,
"Referer": "https://kinoger.re",
},
"sourceId": "kinoger_de",
"ttl": 900000,

View file

@ -36,6 +36,6 @@ export class Frembed implements Source {
}
}
return urls.map(url => ({ countryCode: CountryCode.fr, referer: apiUrl, title: `${json['title']} ${tmdbId.season}x${tmdbId.episode}`, url }));
return urls.map(url => ({ countryCode: CountryCode.fr, title: `${json['title']} ${tmdbId.season}x${tmdbId.episode}`, url }));
};
}

View file

@ -40,7 +40,7 @@ export class KinoGer implements Source {
return Array.from(html.matchAll(/\.show\(.*/g))
.map(showJsMatch => this.findEpisodeUrlInShowJs(showJsMatch[0], seasonIndex, episodeIndex))
.filter((url): url is URL => url !== undefined && !['kinoger.be', 'kinoger.ru'].includes(url.host))
.map(url => ({ countryCode: CountryCode.de, referer: pageUrl, title, url }));
.map(url => ({ countryCode: CountryCode.de, title, url }));
};
private readonly findEpisodeUrlInShowJs = (showJs: string, seasonIndex: number, episodeIndex: number): URL | undefined => {

View file

@ -44,12 +44,12 @@ export class Soaper implements Source {
const episodeUrl = new URL(episodePageHref, this.baseUrl);
const title = `${keyword} ${tmdbId.season}x${tmdbId.episode}`;
return [{ countryCode: CountryCode.en, referer: episodeUrl, title, url: episodeUrl }];
return [{ countryCode: CountryCode.en, title, url: episodeUrl }];
}
const title = `${keyword} (${year})`;
return [{ countryCode: CountryCode.en, referer: pageUrl, title, url: pageUrl }];
return [{ countryCode: CountryCode.en, title, url: pageUrl }];
};
private readonly fetchPageUrl = async (ctx: Context, keyword: string, year: number, hrefPrefix: string): Promise<URL | undefined> => {

View file

@ -4,25 +4,21 @@ exports[`Frembed handle imdb black mirror s4e2 1`] = `
[
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://do7go.com/e/dfx8me4un4ul",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://netu.fanstream.us/e/0DFgfkcXOsDP",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://johnalwayssame.com/e/cqy9oue7sv0g",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://ds2play.com/e/fzfvfq3ngig0",
},
@ -33,25 +29,21 @@ exports[`Frembed handle tmdb black mirror s4e2 1`] = `
[
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://do7go.com/e/dfx8me4un4ul",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://netu.fanstream.us/e/0DFgfkcXOsDP",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://johnalwayssame.com/e/cqy9oue7sv0g",
},
{
"countryCode": "fr",
"referer": "https://frembed.space/api/series?id=42009&sa=4&epi=2&idType=tmdb",
"title": "Black Mirror 4x2",
"url": "https://ds2play.com/e/fzfvfq3ngig0",
},

View file

@ -4,19 +4,16 @@ exports[`KinoGer handle imdb blood and sinners 1`] = `
[
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html",
"title": "Blood & Sinners (2025)",
"url": "https://fsst.online/embed/900576/",
},
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html",
"title": "Blood & Sinners (2025)",
"url": "https://kinoger.pw/e/6zeREaMlrqREZPa",
},
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/18809-blood-sinners-stream.html",
"title": "Blood & Sinners (2025)",
"url": "https://kinoger.re/#ge5fhb",
},
@ -27,13 +24,11 @@ exports[`KinoGer handle imdb dead city s2e5 1`] = `
[
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html",
"title": "The Walking Dead: Dead City 2x5",
"url": "https://supervideo.cc/e/bj6szat1pval",
},
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html",
"title": "The Walking Dead: Dead City 2x5",
"url": "https://kinoger.re/#x6tsx9",
},
@ -44,13 +39,11 @@ exports[`KinoGer handle imdb dead city s2e6 1`] = `
[
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html",
"title": "The Walking Dead: Dead City 2x6",
"url": "https://fsst.online/embed/901994/",
},
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/12971-the-walking-dead-dead-city-stream.html",
"title": "The Walking Dead: Dead City 2x6",
"url": "https://kinoger.re/#ep1tcf",
},
@ -63,7 +56,6 @@ exports[`KinoGer handle no fsst via brokeback mountain 1`] = `
[
{
"countryCode": "de",
"referer": "https://kinoger.com/stream/14781-brokeback-mountain-stream.html",
"title": "Brokeback Mountain (2005)",
"url": "https://dooodster.com/e/1cfcevn6dg8shrfvht22odxw2lty18hr",
},

View file

@ -4,7 +4,6 @@ exports[`Soaper handle imdb black mirror s4e2 1`] = `
[
{
"countryCode": "en",
"referer": "https://soaper.live/episode_5KDq78eGp1.html",
"title": "Black Mirror 4x2",
"url": "https://soaper.live/episode_5KDq78eGp1.html",
},
@ -15,7 +14,6 @@ exports[`Soaper handle imdb full metal jacket 1`] = `
[
{
"countryCode": "en",
"referer": "https://soaper.live/movie_d8kdeypDY9.html",
"title": "Full Metal Jacket (1987)",
"url": "https://soaper.live/movie_d8kdeypDY9.html",
},
@ -26,7 +24,6 @@ exports[`Soaper handle tmdb black mirror s4e2 1`] = `
[
{
"countryCode": "en",
"referer": "https://soaper.live/episode_5KDq78eGp1.html",
"title": "Black Mirror 4x2",
"url": "https://soaper.live/episode_5KDq78eGp1.html",
},

View file

@ -4,7 +4,6 @@ import { Id } from '../utils';
export interface SourceResult {
countryCode: CountryCode;
referer?: URL;
title?: string;
url: URL;
}

View file

@ -6,7 +6,6 @@ export interface Context {
id: string;
ip: string;
config: Config;
referer?: URL;
}
export type ManifestWithConfig = Manifest & { config: ManifestConfig[] };

View file

@ -29,7 +29,6 @@ describe('fetch', () => {
'Accept-Language': 'en',
'Forwarded': 'for=127.0.0.1',
'Priority': 'u=0',
'Referer': 'https://some-url.test',
'User-Agent': 'node',
'X-Forwarded-For': '127.0.0.1',
'X-Forwarded-Proto': 'https',
@ -50,18 +49,6 @@ describe('fetch', () => {
expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
});
test('uses context referer', async () => {
fetchMock.get('https://some-referer-url.test/', 'some text');
await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-referer-url.test/'));
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
headers: {
Referer: 'https://example.com/foo/bar',
},
});
});
test('converts 404 to custom NotFoundError', async () => {
fetchMock.get('https://some-404-url.test/', 404);

View file

@ -42,7 +42,6 @@ interface FlareSolverrResult {
export type CustomRequestInit = RequestInit & {
noFlareSolverr?: boolean;
noReferer?: boolean;
queueLimit?: number;
queueErrorLimit?: number;
timeout?: number;
@ -79,8 +78,6 @@ export class Fetcher {
private getInit(ctx: Context, url: URL, init?: CustomRequestInit): CustomRequestInit {
const cookieString = this.cookieJar.getCookieStringSync(url.href);
const noReferer = init?.noReferer ?? false;
return {
...init,
headers: {
@ -89,7 +86,6 @@ export class Fetcher {
...(cookieString && { Cookie: cookieString }),
'Forwarded': `for=${ctx.ip}`,
'Priority': 'u=0',
...(!noReferer && { Referer: `${ctx.referer?.href ?? url.origin}` }),
'User-Agent': this.hostUserAgentMap.get(url.host) ?? 'node',
'X-Forwarded-For': ctx.ip,
'X-Forwarded-Proto': url.protocol.slice(0, -1),

View file

@ -51,11 +51,7 @@ export class StreamResolver {
this.logger.info(`${handler.id} returned ${handleResults.length} urls`, ctx);
const handlerUrlResults = await Promise.all(
handleResults.map(async ({ countryCode, referer, title, url }) => {
const newCtx = { ...ctx, ...(referer && { referer }) };
return await this.extractorRegistry.handle(newCtx, url, countryCode, title);
}),
handleResults.map(({ countryCode, title, url }) => this.extractorRegistry.handle(ctx, url, countryCode, title)),
);
urlResults.push(...handlerUrlResults.flat());

View file

@ -200,7 +200,7 @@ exports[`resolve returns sorted results 1`] = `
"notWebReady": true,
"proxyHeaders": {
"request": {
"Referer": "http://dood.to/",
"Referer": "http://dood.to",
},
},
"videoSize": 791702409,
@ -218,7 +218,7 @@ exports[`resolve returns sorted results 1`] = `
"notWebReady": true,
"proxyHeaders": {
"request": {
"Referer": "http://dood.to/",
"Referer": "http://dood.to",
},
},
},