From 053dce07297a62b1cae0dcfcf7c753c333b8538e Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 1 Nov 2025 20:28:19 +0000 Subject: [PATCH] fix(extractor): remove flaky HubCloud head requests potentially non-working URLs might be returned now from time to time, but that's still better than not returning working ones because of flakyness or blocking. should also improve performance slightly. reverts changes from c819dd6 --- src/extractor/HubCloud.ts | 20 ++----------- ...ttps:pixeldrain.devapifile2kFQBQiodownload | 1 - .../head-https:pixeldrain.netapifileHqAiguQV | 1 - ...dev-e8f5101f73a4cf8b912d84c41e08628f.error | 1 - ...265.DDP5.1-Tyrell.mkvtoken1758025505.error | 1 - ...DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv | 1 - .../__snapshots__/HubCloud.test.ts.snap | 29 ++++++++++++++++++- 7 files changed, 30 insertions(+), 24 deletions(-) delete mode 100644 src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.devapifile2kFQBQiodownload delete mode 100644 src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.netapifileHqAiguQV delete mode 100644 src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev-e8f5101f73a4cf8b912d84c41e08628f.error delete mode 100644 src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.devDexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkvtoken1758025505.error delete mode 100644 src/extractor/__fixtures__/HubCloud/head-https:pub-c0c400d353df4c68ae3a2d5e089ecf60.r2.devDark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv diff --git a/src/extractor/HubCloud.ts b/src/extractor/HubCloud.ts index 272ccd0..e7b0a5b 100644 --- a/src/extractor/HubCloud.ts +++ b/src/extractor/HubCloud.ts @@ -23,7 +23,7 @@ export class HubCloud extends Extractor { const linksHtml = await this.fetcher.text(ctx, new URL(redirectUrlMatch[1] as string), { headers: { Referer: url.href } }); const $ = cheerio.load(linksHtml); - const urlResults = [ + return Promise.all([ ...$('a') .filter((_i, el) => { const text = $(el).text(); @@ -62,22 +62,6 @@ export class HubCloud extends Extractor { }, }; }).toArray(), - ]; - - return this.asyncFilter(urlResults, async ({ url }) => { - try { - await this.fetcher.head(ctx, url); - } catch { - return false; - } - - return true; - }); + ]); }; - - private async asyncFilter(array: T[], asyncCallback: (item: T) => Promise): Promise { - const results = await Promise.all(array.map(asyncCallback)); - - return array.filter((_, index) => results[index]); - } } diff --git a/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.devapifile2kFQBQiodownload b/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.devapifile2kFQBQiodownload deleted file mode 100644 index f911fbf..0000000 --- a/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.devapifile2kFQBQiodownload +++ /dev/null @@ -1 +0,0 @@ -{"accept-ranges":"bytes","access-control-allow-origin":"*","cache-control":"public, max-age=31536000","connection":"close","content-disposition":"attachment; filename=\"Dark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv\"","content-length":"3386258053","content-security-policy":"default-src 'self'; script-src 'none'","content-type":"video/x-matroska","date":"Tue, 16 Sep 2025 12:25:06 GMT","last-modified":"Sun, 18 May 2025 10:50:26 GMT","strict-transport-security":"max-age=31536000","x-ratelimit-limit":"3000","x-ratelimit-remaining":"2998","x-ratelimit-reset":"1758025560","x-robots-tag":"noindex, nofollow"} \ No newline at end of file diff --git a/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.netapifileHqAiguQV b/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.netapifileHqAiguQV deleted file mode 100644 index 66ffc88..0000000 --- a/src/extractor/__fixtures__/HubCloud/head-https:pixeldrain.netapifileHqAiguQV +++ /dev/null @@ -1 +0,0 @@ -{"accept-ranges":"bytes","access-control-allow-origin":"*","cache-control":"public, max-age=31536000","connection":"close","content-disposition":"inline; filename=\"Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv\"","content-length":"5733546178","content-security-policy":"default-src 'self'; script-src 'none'","content-type":"video/x-matroska","date":"Tue, 16 Sep 2025 12:25:05 GMT","last-modified":"Sun, 25 May 2025 11:45:11 GMT","strict-transport-security":"max-age=31536000","x-ratelimit-limit":"3000","x-ratelimit-remaining":"2998","x-ratelimit-reset":"1758025560","x-robots-tag":"noindex, nofollow"} \ No newline at end of file diff --git a/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev-e8f5101f73a4cf8b912d84c41e08628f.error b/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev-e8f5101f73a4cf8b912d84c41e08628f.error deleted file mode 100644 index 91789e8..0000000 --- a/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev-e8f5101f73a4cf8b912d84c41e08628f.error +++ /dev/null @@ -1 +0,0 @@ -Fetcher error: 403: Forbidden \ No newline at end of file diff --git a/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.devDexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkvtoken1758025505.error b/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.devDexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkvtoken1758025505.error deleted file mode 100644 index 91789e8..0000000 --- a/src/extractor/__fixtures__/HubCloud/head-https:pub-52f57cc1b81f4ecdac02802c1566c33d.r2.devDexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkvtoken1758025505.error +++ /dev/null @@ -1 +0,0 @@ -Fetcher error: 403: Forbidden \ No newline at end of file diff --git a/src/extractor/__fixtures__/HubCloud/head-https:pub-c0c400d353df4c68ae3a2d5e089ecf60.r2.devDark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv b/src/extractor/__fixtures__/HubCloud/head-https:pub-c0c400d353df4c68ae3a2d5e089ecf60.r2.devDark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv deleted file mode 100644 index 29388c0..0000000 --- a/src/extractor/__fixtures__/HubCloud/head-https:pub-c0c400d353df4c68ae3a2d5e089ecf60.r2.devDark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv +++ /dev/null @@ -1 +0,0 @@ -{"accept-ranges":"bytes","cache-control":"public, max-age=604800","cf-ray":"980058374e5337d8-FRA","connection":"close","content-disposition":"attachment; filename*=UTF-8''Dark.S03E08.The.Paradise.1080p.NF.WEB-DL.English.DDP5.1-German.DDP5.1.H.264-4kHdHub.Com.mkv","content-language":"S-B0t","content-length":"3386258051","content-type":"application/octet-stream","date":"Tue, 16 Sep 2025 12:25:06 GMT","etag":"\"26f5e98cf3cc3ef929a660fbf44fe115-404\"","last-modified":"Sun, 18 May 2025 11:02:51 GMT","server":"cloudflare","vary":"Accept-Encoding"} \ No newline at end of file diff --git a/src/extractor/__snapshots__/HubCloud.test.ts.snap b/src/extractor/__snapshots__/HubCloud.test.ts.snap index 2642167..98f2c15 100644 --- a/src/extractor/__snapshots__/HubCloud.test.ts.snap +++ b/src/extractor/__snapshots__/HubCloud.test.ts.snap @@ -1,6 +1,21 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`HubCloud handle crayon shin-chan 1993 1`] = `[]`; +exports[`HubCloud handle crayon shin-chan 1993 1`] = ` +[ + { + "format": "unknown", + "label": "HubCloud (FSL)", + "meta": { + "bytes": 28550795100, + "countryCodes": [], + "title": "Crayon Shinchan Action Kamen vs Demon (1993) 1080p BluRay REMUX AVC [Hindi DD 2.0 + Japanese FLAC 2.0] (PitiPati-Ionicboy) (4kHDHub.com).mkv", + }, + "sourceId": "hubcloud_fsl", + "ttl": 259200000, + "url": "https://pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev/Crayon%20Shinchan%20Action%20Kamen%20vs%20Demon%201993%201080p%20BluRay%20REMUX%20AVC%20[Hindi%20DD%202.0%20+%20Japanese%20FLAC%202.0]%20PitiPati-Ionicboy%204kHDHub.com.mkv?token=1758025506", + }, +] +`; exports[`HubCloud handle dark 2017 s03e08 1`] = ` [ @@ -33,6 +48,18 @@ exports[`HubCloud handle dark 2017 s03e08 1`] = ` exports[`HubCloud handle dexter original sin 2024 s01e01 1`] = ` [ + { + "format": "unknown", + "label": "HubCloud (FSL)", + "meta": { + "bytes": 5733781340, + "countryCodes": [], + "title": "Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv", + }, + "sourceId": "hubcloud_fsl", + "ttl": 259200000, + "url": "https://pub-52f57cc1b81f4ecdac02802c1566c33d.r2.dev/Dexter.Original.Sin.S01E01.2160p.SKST.WEB-DL.H.265.DDP5.1-Tyrell.mkv?token=1758025505", + }, { "format": "unknown", "label": "HubCloud (PixelServer)",