From c1f9de58b0a08f240dbf92dfc1e0fe4905bfb84f Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:56:35 +0000 Subject: [PATCH] chore: do not show external URLs by default --- src/extractor/ExternalUrl.test.ts | 2 +- src/extractor/ExtractorRegistry.test.ts | 8 +- .../ExtractorRegistry.test.ts.snap | 6 +- src/types.ts | 2 +- src/utils/StreamResolver.test.ts | 10 +- .../__snapshots__/StreamResolver.test.ts.snap | 347 +++++++++++------- src/utils/__snapshots__/manifest.test.ts.snap | 8 +- src/utils/config.ts | 2 +- src/utils/manifest.ts | 4 +- 9 files changed, 233 insertions(+), 156 deletions(-) diff --git a/src/extractor/ExternalUrl.test.ts b/src/extractor/ExternalUrl.test.ts index a024400..402f6fd 100644 --- a/src/extractor/ExternalUrl.test.ts +++ b/src/extractor/ExternalUrl.test.ts @@ -7,7 +7,7 @@ import { ExternalUrl } from './ExternalUrl'; const logger = winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }); const extractorRegistry = new ExtractorRegistry(logger, [new ExternalUrl(new FetcherMock(`${__dirname}/__fixtures__/ExternalUrl`))]); -const ctx: Context = { id: 'id', ip: '127.0.0.1', config: {} }; +const ctx: Context = { id: 'id', ip: '127.0.0.1', config: { includeExternalUrls: 'on' } }; describe('ExternalUrl', () => { test('404 - not found', async () => { diff --git a/src/extractor/ExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts index 4dc1f53..8f9d6bc 100644 --- a/src/extractor/ExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -16,14 +16,14 @@ describe('ExtractorRegistry', () => { expect(urlResult).toMatchSnapshot(); }); - test('return external URLs by default', async () => { - const urlResult = await extractorRegistry.handle(ctx, new URL('https://mixdrop.ag/e/3nzwveprim63or6'), CountryCode.de); + test('returns external URLs if enabled by config', async () => { + const urlResult = await extractorRegistry.handle({ ...ctx, config: { ...ctx.config, includeExternalUrls: 'on' } }, new URL('https://mixdrop.ag/e/3nzwveprim63or6'), CountryCode.de); expect(urlResult).toMatchSnapshot(); }); - test('does not return external URLs if disabled by config', async () => { - const urlResult = await extractorRegistry.handle({ ...ctx, config: { ...ctx.config, excludeExternalUrls: 'on' } }, new URL('https://mixdrop.ag/e/l7v73zqrfdj19z'), CountryCode.de); + test('does not return external URLs by default', async () => { + const urlResult = await extractorRegistry.handle(ctx, new URL('https://mixdrop.ag/e/l7v73zqrfdj19z'), CountryCode.de); expect(urlResult).toStrictEqual([]); }); diff --git a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap index fe777c7..3b4bee0 100644 --- a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap +++ b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap @@ -1,6 +1,8 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`ExtractorRegistry return external URLs by default 1`] = ` +exports[`ExtractorRegistry returns error result from extractor 1`] = `[]`; + +exports[`ExtractorRegistry returns external URLs if enabled by config 1`] = ` [ { "isExternal": true, @@ -15,8 +17,6 @@ exports[`ExtractorRegistry return external URLs by default 1`] = ` ] `; -exports[`ExtractorRegistry returns error result from extractor 1`] = `[]`; - exports[`ExtractorRegistry returns external url for error 1`] = ` [ { diff --git a/src/types.ts b/src/types.ts index c13ae27..625128f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,7 +8,7 @@ export interface Context { export type ManifestWithConfig = Manifest & { config: ManifestConfig[] }; -export type Config = Partial>; +export type Config = Partial>; export enum CountryCode { de = 'de', diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 361739b..334678f 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -55,8 +55,12 @@ describe('resolve', () => { const streamResolver = new StreamResolver(logger, new ExtractorRegistry(logger, createExtractors(fetcher))); const streams = await streamResolver.resolve(ctx, [meineCloud, mostraGuarda], 'movie', new ImdbId('tt29141112', undefined, undefined)); + expect(streams.ttl).not.toBeUndefined(); + expect(streams.streams).toMatchSnapshot(); - expect(streams).toMatchSnapshot(); + const streamsWithExternalUrls = await streamResolver.resolve({ ...ctx, config: { ...ctx.config, includeExternalUrls: 'on' } }, [meineCloud, mostraGuarda], 'movie', new ImdbId('tt29141112', undefined, undefined)); + expect(streamsWithExternalUrls.ttl).not.toBeUndefined(); + expect(streamsWithExternalUrls.streams).toMatchSnapshot(); }); test('adds error info', async () => { @@ -173,8 +177,8 @@ describe('resolve', () => { const streams = await streamResolver.resolve(ctx, [new MockHandler()], 'movie', new ImdbId('tt11655566', undefined, undefined)); expect(streams).toMatchSnapshot(); - const streamsWithoutExternalUrls = await streamResolver.resolve({ ...ctx, config: { ...ctx.config, excludeExternalUrls: 'on' } }, [new MockHandler()], 'movie', new ImdbId('tt11655566', undefined, undefined)); - expect(streamsWithoutExternalUrls).toMatchSnapshot(); + const streamsWithExternalUrls = await streamResolver.resolve({ ...ctx, config: { ...ctx.config, includeExternalUrls: 'on' } }, [new MockHandler()], 'movie', new ImdbId('tt11655566', undefined, undefined)); + expect(streamsWithExternalUrls).toMatchSnapshot(); }); test('ignores not found errors', async () => { diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index 2a7116f..41f2aba 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -1,6 +1,69 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing exports[`resolve adds error info 1`] = ` +{ + "streams": [ + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +⚠️ Request was blocked.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +⚠️ Request was blocked.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +🚦 Request was rate-limited. Please try again later or consider self-hosting.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +🚦 Too many recent timeouts. Please try again later.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +❌ Request failed. Request-id: id.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +🐢 Request timed out.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +⏳ Request queue is full. Please try again later or consider self-hosting.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr 🇩🇪", + "title": "🔗 hoster.com +❌ Request failed with status 500 (Internal Server Error). Request-id: id.", + "ytId": "E4WlUXrJgy4", + }, + ], +} +`; + +exports[`resolve adds error info 2`] = ` { "streams": [ { @@ -63,69 +126,6 @@ exports[`resolve adds error info 1`] = ` } `; -exports[`resolve adds error info 2`] = ` -{ - "streams": [ - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -⚠️ Request was blocked.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -⚠️ Request was blocked.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -🚦 Request was rate-limited. Please try again later or consider self-hosting.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -🚦 Too many recent timeouts. Please try again later.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -❌ Request failed. Request-id: id.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -🐢 Request timed out.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -⏳ Request queue is full. Please try again later or consider self-hosting.", - "ytId": "E4WlUXrJgy4", - }, - { - "behaviorHints": {}, - "name": "WebStreamr 🇩🇪", - "title": "🔗 hoster.com -❌ Request failed with status 500 (Internal Server Error). Request-id: id.", - "ytId": "E4WlUXrJgy4", - }, - ], -} -`; - exports[`resolve ignores not found errors 1`] = ` { "streams": [], @@ -160,98 +160,171 @@ exports[`resolve returns info as stream if no sources were configured 1`] = ` `; exports[`resolve returns sorted results 1`] = ` -{ - "streams": [ - { - "behaviorHints": { - "bingeGroup": "webstreamr-dropload_de", - "videoSize": 1395864371, - }, - "name": "WebStreamr 🇩🇪 1080p", - "title": "des-teufels-bad-2024 +[ + { + "behaviorHints": { + "bingeGroup": "webstreamr-dropload_de", + "videoSize": 1395864371, + }, + "name": "WebStreamr 🇩🇪 1080p", + "title": "des-teufels-bad-2024 💾 1.3 GB 🔗 Dropload", - "url": "https://srv34.dropload.io/hls2/01/00197/lyo2h1snpe5c_h/master.m3u8?t=MZSqs3F338ocbM2eTs8X23s585dWcYuilGrNNcidNas&s=1749846846&e=14400&f=987607&srv=srv27&i=0.0&sp=0&ii=50.99.15.93&p1=srv27&p2=srv27", + "url": "https://srv34.dropload.io/hls2/01/00197/lyo2h1snpe5c_h/master.m3u8?t=MZSqs3F338ocbM2eTs8X23s585dWcYuilGrNNcidNas&s=1749846846&e=14400&f=987607&srv=srv27&i=0.0&sp=0&ii=50.99.15.93&p1=srv27&p2=srv27", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-dropload_it", + "videoSize": 1181116006, }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-dropload_it", - "videoSize": 1181116006, - }, - "name": "WebStreamr 🇮🇹 720p", - "title": "the-devils-bath-2024-sd-sub-ita + "name": "WebStreamr 🇮🇹 720p", + "title": "the-devils-bath-2024-sd-sub-ita 💾 1.1 GB 🔗 Dropload", - "url": "https://srv34.dropload.io/hls2/01/00200/xsr90y2ltdyx_h/master.m3u8?t=rsJMeR8rtWZxFi_3DOdlt98U0rvAMincCe4-awDQzHE&s=1749846846&e=14400&f=1004116&srv=srv24&i=0.0&sp=0&ii=50.99.15.93&p1=srv24&p2=srv24", + "url": "https://srv34.dropload.io/hls2/01/00200/xsr90y2ltdyx_h/master.m3u8?t=rsJMeR8rtWZxFi_3DOdlt98U0rvAMincCe4-awDQzHE&s=1749846846&e=14400&f=1004116&srv=srv24&i=0.0&sp=0&ii=50.99.15.93&p1=srv24&p2=srv24", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-supervideo_it", + "videoSize": 1181116006, }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-supervideo_it", - "videoSize": 1181116006, - }, - "name": "WebStreamr 🇮🇹 720p", - "title": "the-devils-bath-2024-sd-sub-ita + "name": "WebStreamr 🇮🇹 720p", + "title": "the-devils-bath-2024-sd-sub-ita 💾 1.1 GB 🔗 SuperVideo", - "url": "https://hfs311.serversicuro.cc/hls/,dnzpcjj427g4a3gyvbth53rxqgffbu4vol4o7gg7pskpzb6josghbkwhnisq,.urlset/master.m3u8", + "url": "https://hfs311.serversicuro.cc/hls/,dnzpcjj427g4a3gyvbth53rxqgffbu4vol4o7gg7pskpzb6josghbkwhnisq,.urlset/master.m3u8", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-supervideo_de", + "videoSize": 1073741824, }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-supervideo_de", - "videoSize": 1073741824, - }, - "name": "WebStreamr 🇩🇪 720p", - "title": "des-teufels-bad-2024 + "name": "WebStreamr 🇩🇪 720p", + "title": "des-teufels-bad-2024 💾 1 GB 🔗 SuperVideo", - "url": "https://hfs309.serversicuro.cc/hls/,dnzpfi3d27g4a3gyvbmh5klwtl65qh654hyjtgupd6p577fze64mz3bsmdyq,.urlset/master.m3u8", - }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-doodstream_de", - "notWebReady": true, - "proxyHeaders": { - "request": { - "Referer": "http://dood.to", - }, + "url": "https://hfs309.serversicuro.cc/hls/,dnzpfi3d27g4a3gyvbmh5klwtl65qh654hyjtgupd6p577fze64mz3bsmdyq,.urlset/master.m3u8", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-doodstream_de", + "notWebReady": true, + "proxyHeaders": { + "request": { + "Referer": "http://dood.to", }, - "videoSize": 791702409, }, - "name": "WebStreamr 🇩🇪", - "title": "des-teufels-bad-2024 + "videoSize": 791702409, + }, + "name": "WebStreamr 🇩🇪", + "title": "des-teufels-bad-2024 💾 755.03 MB 🔗 DoodStream", - "url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000", - }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-doodstream_it", - "notWebReady": true, - "proxyHeaders": { - "request": { - "Referer": "http://dood.to", - }, + "url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-doodstream_it", + "notWebReady": true, + "proxyHeaders": { + "request": { + "Referer": "http://dood.to", }, }, - "name": "WebStreamr 🇮🇹", - "title": "the-devils-bath-2024-sd-sub-ita + }, + "name": "WebStreamr 🇮🇹", + "title": "the-devils-bath-2024-sd-sub-ita 🔗 DoodStream", - "url": "https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000", + "url": "https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000", + }, +] +`; + +exports[`resolve returns sorted results 2`] = ` +[ + { + "behaviorHints": { + "bingeGroup": "webstreamr-dropload_de", + "videoSize": 1395864371, }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-external_de", + "name": "WebStreamr 🇩🇪 1080p", + "title": "des-teufels-bad-2024 +💾 1.3 GB 🔗 Dropload", + "url": "https://srv34.dropload.io/hls2/01/00197/lyo2h1snpe5c_h/master.m3u8?t=MZSqs3F338ocbM2eTs8X23s585dWcYuilGrNNcidNas&s=1749846846&e=14400&f=987607&srv=srv27&i=0.0&sp=0&ii=50.99.15.93&p1=srv27&p2=srv27", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-dropload_it", + "videoSize": 1181116006, + }, + "name": "WebStreamr 🇮🇹 720p", + "title": "the-devils-bath-2024-sd-sub-ita +💾 1.1 GB 🔗 Dropload", + "url": "https://srv34.dropload.io/hls2/01/00200/xsr90y2ltdyx_h/master.m3u8?t=rsJMeR8rtWZxFi_3DOdlt98U0rvAMincCe4-awDQzHE&s=1749846846&e=14400&f=1004116&srv=srv24&i=0.0&sp=0&ii=50.99.15.93&p1=srv24&p2=srv24", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-supervideo_it", + "videoSize": 1181116006, + }, + "name": "WebStreamr 🇮🇹 720p", + "title": "the-devils-bath-2024-sd-sub-ita +💾 1.1 GB 🔗 SuperVideo", + "url": "https://hfs311.serversicuro.cc/hls/,dnzpcjj427g4a3gyvbth53rxqgffbu4vol4o7gg7pskpzb6josghbkwhnisq,.urlset/master.m3u8", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-supervideo_de", + "videoSize": 1073741824, + }, + "name": "WebStreamr 🇩🇪 720p", + "title": "des-teufels-bad-2024 +💾 1 GB 🔗 SuperVideo", + "url": "https://hfs309.serversicuro.cc/hls/,dnzpfi3d27g4a3gyvbmh5klwtl65qh654hyjtgupd6p577fze64mz3bsmdyq,.urlset/master.m3u8", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-doodstream_de", + "notWebReady": true, + "proxyHeaders": { + "request": { + "Referer": "http://dood.to", + }, }, - "externalUrl": "https://mixdrop.ag/e/3nzwveprim63or6", - "name": "WebStreamr 🇩🇪 ⚠️ external", - "title": "🔗 mixdrop.ag", + "videoSize": 791702409, }, - { - "behaviorHints": { - "bingeGroup": "webstreamr-external_it", + "name": "WebStreamr 🇩🇪", + "title": "des-teufels-bad-2024 +💾 755.03 MB 🔗 DoodStream", + "url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-doodstream_it", + "notWebReady": true, + "proxyHeaders": { + "request": { + "Referer": "http://dood.to", + }, }, - "externalUrl": "https://mixdrop.ag/e/vk196d6xfzwwo1", - "name": "WebStreamr 🇮🇹 ⚠️ external", - "title": "🔗 mixdrop.ag", }, - ], - "ttl": 900000, -} + "name": "WebStreamr 🇮🇹", + "title": "the-devils-bath-2024-sd-sub-ita +🔗 DoodStream", + "url": "https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-external_de", + }, + "externalUrl": "https://mixdrop.ag/e/3nzwveprim63or6", + "name": "WebStreamr 🇩🇪 ⚠️ external", + "title": "🔗 mixdrop.ag", + }, + { + "behaviorHints": { + "bingeGroup": "webstreamr-external_it", + }, + "externalUrl": "https://mixdrop.ag/e/vk196d6xfzwwo1", + "name": "WebStreamr 🇮🇹 ⚠️ external", + "title": "🔗 mixdrop.ag", + }, +] `; exports[`resolve returns source errors as stream 1`] = ` diff --git a/src/utils/__snapshots__/manifest.test.ts.snap b/src/utils/__snapshots__/manifest.test.ts.snap index 6fd0fcb..20edf22 100644 --- a/src/utils/__snapshots__/manifest.test.ts.snap +++ b/src/utils/__snapshots__/manifest.test.ts.snap @@ -19,8 +19,8 @@ exports[`buildManifest has checked source with appropriate config 1`] = ` "type": "checkbox", }, { - "key": "excludeExternalUrls", - "title": "Exclude external URLs from results", + "key": "includeExternalUrls", + "title": "Include external URLs from results", "type": "checkbox", }, ] @@ -44,8 +44,8 @@ exports[`buildManifest has unchecked source without a config 1`] = ` "type": "checkbox", }, { - "key": "excludeExternalUrls", - "title": "Exclude external URLs from results", + "key": "includeExternalUrls", + "title": "Include external URLs from results", "type": "checkbox", }, ] diff --git a/src/utils/config.ts b/src/utils/config.ts index 394cbfe..0181cbe 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -4,4 +4,4 @@ export const getDefaultConfig = (): Config => { return { en: 'on' }; }; -export const showExternalUrls = (config: Config): boolean => !('excludeExternalUrls' in config); +export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config; diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index 4ebc6ce..28384dc 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -51,9 +51,9 @@ export const buildManifest = (sources: Source[], config: Config): ManifestWithCo } manifest.config.push({ - key: 'excludeExternalUrls', + key: 'includeExternalUrls', type: 'checkbox', - title: 'Exclude external URLs from results', + title: 'Include external URLs from results', }); return manifest;