Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore(extractor): decrease timeout and ignore errors for external URLs

Browse source
This commit is contained in:
WebStreamr 2025-06-11 08:17:35 +00:00
parent faad6a2562
commit 19395f1921
No known key found for this signature in database
8 changed files with 27 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/extractor/ExternalUrl.ts
View file

@ -20,8 +20,12 @@ export class ExternalUrl implements Extractor {
readonly normalize = (url: URL): URL => url;
readonly extract = async (ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> => {
// We only want to make sure that the URL is accessible
await this.fetcher.head(ctx, url, { noFlareSolverr: true });
try {
// Make sure the URL is accessible, but avoid causing noise and delays doing this
await this.fetcher.head(ctx, url, { noFlareSolverr: true, timeout: 1000 });
} catch {
return [];
}
return [
{

6
src/extractor/ExtractorRegistry.test.ts
View file

@ -45,4 +45,10 @@ describe('ExtractorRegistry', () => {
expect(urlResults1).toStrictEqual([]);
expect(urlResults2).toStrictEqual([]);
});
test('returns external url for error', async () => {
const urlResults = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), { countryCode: 'de' });
expect(urlResults).toMatchSnapshot();
});
});

15
src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap
View file

@ -15,17 +15,20 @@ exports[`ExtractorRegistry return external URLs by default 1`] = `
]
`;
exports[`ExtractorRegistry returns error result from extractor 1`] = `
exports[`ExtractorRegistry returns error result from extractor 1`] = `[]`;
exports[`ExtractorRegistry returns external url for error 1`] = `
[
{
"error": [Error: TypeError: fetch failed],
"error": [Error: Fetcher error: 403: Forbidden
],
"isExternal": true,
"label": "some-url.test",
"label": "dropload.io",
"meta": {
"countryCode": "en",
"countryCode": "de",
},
"sourceId": "external",
"url": "https://some-url.test/",
"sourceId": "dropload",
"url": "https://dropload.io/mocked-blocked.html",
},
]
`;

28
src/handler/__snapshots__/Frembed.test.ts.snap
View file

@ -44,19 +44,7 @@ exports[`Frembed handle imdb black mirror s4e2 1`] = `
"url": "https://johnalwayssame.com/e/cqy9oue7sv0g",
},
],
[
{
"error": [Error: TypeError: fetch failed],
"isExternal": true,
"label": "ds2play.com",
"meta": {
"countryCode": "fr",
"title": "Black Mirror 4x2",
},
"sourceId": "external",
"url": "https://ds2play.com/e/fzfvfq3ngig0",
},
],
[],
]
`;
@ -104,18 +92,6 @@ exports[`Frembed handle tmdb black mirror s4e2 1`] = `
"url": "https://johnalwayssame.com/e/cqy9oue7sv0g",
},
],
[
{
"error": [Error: TypeError: fetch failed],
"isExternal": true,
"label": "ds2play.com",
"meta": {
"countryCode": "fr",
"title": "Black Mirror 4x2",
},
"sourceId": "external",
"url": "https://ds2play.com/e/fzfvfq3ngig0",
},
],
[],
]
`;

13
src/handler/__snapshots__/FrenchCloud.test.ts.snap
View file

@ -42,18 +42,7 @@ exports[`FrenchCloud handle imdb the devil's bath 1`] = `
"url": "https://mixdrop.ag/e/l7v73zqrfdj19z",
},
],
[
{
"error": [Error: Fetcher error: 404: Not Found],
"isExternal": true,
"label": "streamtape.com",
"meta": {
"countryCode": "fr",
},
"sourceId": "external",
"url": "https://streamtape.com/e/gjA1OQ4klyHxgJ",
},
],
[],
[
{
"label": "DoodStream",

14
src/handler/__snapshots__/KinoGer.test.ts.snap
View file

@ -34,19 +34,7 @@ exports[`KinoGer handle imdb blood and sinners 1`] = `
"url": "https://www.secvideo1.online/get_file/16/fae180e05d8d7af50fff44c7a24232b522db464935/900000/900576/900576.mp4/",
},
],
[
{
"error": [Error: Fetcher error: 404: Not Found],
"isExternal": true,
"label": "kinoger.pw",
"meta": {
"countryCode": "de",
"title": "Blood & Sinners (2025)",
},
"sourceId": "external",
"url": "https://kinoger.pw/e/6zeREaMlrqREZPa",
},
],
[],
[
{
"label": "KinoGer",

13
src/handler/__snapshots__/VerHdLink.test.ts.snap
View file

@ -42,18 +42,7 @@ exports[`VerHdLink handle titanic 1`] = `
"url": "https://mixdrop.ag/e/vn0wx308fq984q",
},
],
[
{
"error": [Error: Fetcher error: 404: Not Found],
"isExternal": true,
"label": "streamtape.com",
"meta": {
"countryCode": "mx",
},
"sourceId": "external",
"url": "https://streamtape.com/e/Bjp2vjrdBxsK82",
},
],
[],
[],
[
{

1
src/utils/__fixtures__/Fetcher/https:dropload.iomocked-blocked.html.error generated Normal file
View file

@ -0,0 +1 @@
Fetcher error: 403: Forbidden