Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

fix(extractor): remove flaky HubCloud head requests

Browse source 
potentially non-working URLs might be returned now from time to time, but that's still better than not returning working ones because of flakyness or blocking. should also improve performance slightly.

reverts changes from c819dd6
This commit is contained in:
WebStreamr 2025-11-01 20:28:19 +00:00
parent a36d152b61
commit 053dce0729
No known key found for this signature in database
7 changed files with 30 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/extractor/HubCloud.ts
View file

@ -23,7 +23,7 @@ export class HubCloud extends Extractor {
const linksHtml = await this.fetcher.text(ctx, new URL(redirectUrlMatch[1] as string), { headers: { Referer: url.href } });
const $ = cheerio.load(linksHtml);
const urlResults = [
return Promise.all([
...$('a')
.filter((_i, el) => {
const text = $(el).text();
@ -62,22 +62,6 @@ export class HubCloud extends Extractor {
},
};
}).toArray(),
];
return this.asyncFilter(urlResults, async ({ url }) => {
try {
await this.fetcher.head(ctx, url);
} catch {
return false;
}
return true;
});
]);
};
private async asyncFilter<T>(array: T[], asyncCallback: (item: T) => Promise<boolean>): Promise<T[]> {
const results = await Promise.all(array.map(asyncCallback));
return array.filter((_, index) => results[index]);
}
}

1
src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.devapifile2kFQBQiodownload generated
View file

@ -1 +0,0 @@
{"accept-ranges":"bytes","access-control-allow-origin":"*","cache-control":"public, max-age=31536000","connection":"close","content-disposition":"attachment; filename=\"Dark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv\"","content-length":"3386258053","content-security-policy":"default-src 'self'; script-src 'none'","content-type":"video/x-matroska","date":"Tue, 16 Sep 2025 12:25:06 GMT","last-modified":"Sun, 18 May 2025 10:50:26 GMT","strict-transport-security":"max-age=31536000","x-ratelimit-limit":"3000","x-ratelimit-remaining":"2998","x-ratelimit-reset":"1758025560","x-robots-tag":"noindex, nofollow"}

1
src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.netapifileHqAiguQV generated
View file

@ -1 +0,0 @@
{"accept-ranges":"bytes","access-control-allow-origin":"*","cache-control":"public, max-age=31536000","connection":"close","content-disposition":"inline; filename=\"Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv\"","content-length":"5733546178","content-security-policy":"default-src 'self'; script-src 'none'","content-type":"video/x-matroska","date":"Tue, 16 Sep 2025 12:25:05 GMT","last-modified":"Sun, 25 May 2025 11:45:11 GMT","strict-transport-security":"max-age=31536000","x-ratelimit-limit":"3000","x-ratelimit-remaining":"2998","x-ratelimit-reset":"1758025560","x-robots-tag":"noindex, nofollow"}

1
src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev-e8f5101f73a4cf8b912d84c41e08628f.error generated
View file

@ -1 +0,0 @@
Fetcher error: 403: Forbidden

1
src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.devDexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkvtoken1758025505.error generated
View file

@ -1 +0,0 @@
Fetcher error: 403: Forbidden

1
src/extractor/__fixtures__/HubCloud/head-https:pub-c0c400d353df4c68ae3a2d5e089ecf60.r2.devDark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv generated
View file

@ -1 +0,0 @@
{"accept-ranges":"bytes","cache-control":"public, max-age=604800","cf-ray":"980058374e5337d8-FRA","connection":"close","content-disposition":"attachment; filename*=UTF-8''Dark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv","content-language":"S-B0t","content-length":"3386258051","content-type":"application/octet-stream","date":"Tue, 16 Sep 2025 12:25:06 GMT","etag":"\"26f5e98cf3cc3ef929a660fbf44fe115-404\"","last-modified":"Sun, 18 May 2025 11:02:51 GMT","server":"cloudflare","vary":"Accept-Encoding"}

29
src/extractor/__snapshots__/HubCloud.test.ts.snap
View file

@ -1,6 +1,21 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
exports[`HubCloud handle crayon shin-chan 1993 1`] = `[]`;
exports[`HubCloud handle crayon shin-chan 1993 1`] = `
[
{
"format": "unknown",
"label": "HubCloud (FSL)",
"meta": {
"bytes": 28550795100,
"countryCodes": [],
"title": "Crayon Shinchan Action Kamen vs Demon (1993) 1080p BluRay REMUX AVC [Hindi DD 2.0 + Japanese FLAC 2.0] (PitiPati-Ionicboy) (4kHDHub.com).mkv",
},
"sourceId": "hubcloud_fsl",
"ttl": 259200000,
"url": "https://pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev/Crayon%20Shinchan%20Action%20Kamen%20vs%20Demon%201993%201080p%20BluRay%20REMUX%20AVC%20[Hindi%20DD%202.0%20+%20Japanese%20FLAC%202.0]%20PitiPati-Ionicboy%204kHDHub.com.mkv?token=1758025506",
},
]
`;
exports[`HubCloud handle dark 2017 s03e08 1`] = `
[
@ -33,6 +48,18 @@ exports[`HubCloud handle dark 2017 s03e08 1`] = `
exports[`HubCloud handle dexter original sin 2024 s01e01 1`] = `
[
{
"format": "unknown",
"label": "HubCloud (FSL)",
"meta": {
"bytes": 5733781340,
"countryCodes": [],
"title": "Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv",
},
"sourceId": "hubcloud_fsl",
"ttl": 259200000,
"url": "https://pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev/Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv?token=1758025505",
},
{
"format": "unknown",
"label": "HubCloud (PixelServer)",