chore(fetcher): use more realistic referer/origin header values
This commit is contained in:
WebStreamr
parent
0638ce0a8c
commit
02b73b41d7
11 changed files with 43 additions and 15 deletions
|
|
@ -51,7 +51,7 @@ export class CineHDPlus implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/cinehdplus/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode, title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode, title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ export class Eurostreaming implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/eurostreaming/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'it', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode: 'it', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ export class Frembed implements Handler {
|
|||
|
||||
const tmdbId = await getTmdbIdFromImdbId(ctx, this.fetcher, parseImdbId(id));
|
||||
|
||||
const response = await this.apiCall(ctx, new URL(`https://frembed.club/api/series?id=${tmdbId.id}&sa=${tmdbId.series}&epi=${tmdbId.episode}&idType=tmdb`));
|
||||
const apiUrl = new URL(`https://frembed.club/api/series?id=${tmdbId.id}&sa=${tmdbId.series}&epi=${tmdbId.episode}&idType=tmdb`);
|
||||
const response = await this.apiCall(ctx, apiUrl);
|
||||
if (!response) {
|
||||
return [];
|
||||
}
|
||||
|
|
@ -46,7 +47,9 @@ export class Frembed implements Handler {
|
|||
}
|
||||
}
|
||||
|
||||
return Promise.all(urls.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'fr', title: `${json['title']} ${tmdbId.series}x${tmdbId.episode}` })));
|
||||
return Promise.all(
|
||||
urls.map(url => this.extractorRegistry.handle({ ...ctx, referer: apiUrl }, url, { countryCode: 'fr', title: `${json['title']} ${tmdbId.series}x${tmdbId.episode}` })),
|
||||
);
|
||||
};
|
||||
|
||||
private readonly apiCall = async (ctx: Context, url: URL): Promise<string | undefined> => {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ export class FrenchCloud implements Handler {
|
|||
return [];
|
||||
}
|
||||
|
||||
const html = await this.fetcher.text(ctx, new URL(`https://frenchcloud.cam/movie/${parseImdbId(id).id}`));
|
||||
const pageUrl = new URL(`https://frenchcloud.cam/movie/${parseImdbId(id).id}`);
|
||||
const html = await this.fetcher.text(ctx, pageUrl);
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
|
|
@ -35,7 +36,7 @@ export class FrenchCloud implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/frenchcloud/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'fr' })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'fr' })),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ export class KinoKiste implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/kinokiste/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'de', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: seriesPageUrl }, url, { countryCode: 'de', title: `${title.trim()} ${imdbId.series}x${imdbId.episode}` })),
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ export class MeineCloud implements Handler {
|
|||
return [];
|
||||
}
|
||||
|
||||
const html = await this.fetcher.text(ctx, new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`));
|
||||
const pageUrl = new URL(`https://meinecloud.click/movie/${parseImdbId(id).id}`);
|
||||
const html = await this.fetcher.text(ctx, pageUrl);
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
|
|
@ -35,7 +36,7 @@ export class MeineCloud implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/meinecloud/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'de' })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'de' })),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ export class MostraGuarda implements Handler {
|
|||
return [];
|
||||
}
|
||||
|
||||
const html = await this.fetcher.text(ctx, new URL(`https://mostraguarda.stream/movie/${parseImdbId(id).id}`));
|
||||
const pageUrl = new URL(`https://mostraguarda.stream/movie/${parseImdbId(id).id}`);
|
||||
const html = await this.fetcher.text(ctx, pageUrl);
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
|
|
@ -35,7 +36,7 @@ export class MostraGuarda implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/mostraguarda/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode: 'it' })),
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode: 'it' })),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ export class VerHdLink implements Handler {
|
|||
return [];
|
||||
}
|
||||
|
||||
const html = await this.fetcher.text(ctx, new URL(`https://verhdlink.cam/movie/${parseImdbId(id).id}`));
|
||||
const pageUrl = new URL(`https://verhdlink.cam/movie/${parseImdbId(id).id}`);
|
||||
const html = await this.fetcher.text(ctx, pageUrl);
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
|
|
@ -46,7 +47,7 @@ export class VerHdLink implements Handler {
|
|||
.map((_i, el) => new URL(($(el).attr('data-link') as string).replace(/^(https:)?\/\//, 'https://')))
|
||||
.toArray()
|
||||
.filter(url => !url.host.match(/verhdlink/))
|
||||
.map(url => this.extractorRegistry.handle(ctx, url, { countryCode }));
|
||||
.map(url => this.extractorRegistry.handle({ ...ctx, referer: pageUrl }, url, { countryCode }));
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ export interface Context {
|
|||
id: string;
|
||||
ip: string;
|
||||
config: Config;
|
||||
referer?: URL;
|
||||
}
|
||||
|
||||
export type ManifestWithConfig = Manifest & { config: ManifestConfig[] };
|
||||
|
|
|
|||
|
|
@ -119,6 +119,23 @@ describe('fetch', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('uses context referer', async () => {
|
||||
axiosMock.onGet().reply(200, 'some text');
|
||||
const axiosSpy = jest.spyOn(axios, 'get');
|
||||
|
||||
await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-url.test/'));
|
||||
|
||||
expect(axiosSpy).toHaveBeenCalledWith(
|
||||
'https://some-url.test/',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Origin: 'https://example.com',
|
||||
Referer: 'https://example.com/foo/bar',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('converts 404 to custom NotFoundError', async () => {
|
||||
axiosMock.onGet().reply(404);
|
||||
|
||||
|
|
|
|||
|
|
@ -35,14 +35,17 @@ export class Fetcher {
|
|||
};
|
||||
|
||||
private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => {
|
||||
const origin = ctx.referer?.origin ?? url.origin;
|
||||
const referer = ctx.referer?.href ?? url.origin;
|
||||
|
||||
return {
|
||||
responseType: 'text',
|
||||
timeout: 15000,
|
||||
...config,
|
||||
headers: {
|
||||
'Forwarded': `for=${ctx.ip}`,
|
||||
'Origin': `${url.origin}`,
|
||||
'Referer': `${url.origin}`,
|
||||
'Origin': `${origin}`,
|
||||
'Referer': `${referer}`,
|
||||
'User-Agent': this.createUserAgentForIp(ctx.ip),
|
||||
'X-Forwarded-For': ctx.ip,
|
||||
'X-Forwarded-Proto': url.protocol.slice(0, -1),
|
||||
|
|
|
|||
Loading…
Reference in a new issue