Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore: remove duplicate URLs

Browse source
This commit is contained in:
WebStreamr 2026-03-27 19:40:27 +00:00
parent 89e57d48d9
commit b87bf62362
No known key found for this signature in database
3 changed files with 130 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

200
src/utils/StreamResolver.test.ts
View file

@ -128,7 +128,7 @@ describe('resolve', () => {
protected readonly extractInternal = async (): Promise<UrlResult[]> =>
[
{
url: new URL('https://example.com'),
url: new URL('https://example1.com'),
format: Format.unknown,
isExternal: true,
label: 'hoster.com',
@ -137,106 +137,11 @@ describe('resolve', () => {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_challenge, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.cloudflare_censor, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.media_flow_proxy_auth, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new BlockedError(new URL('https://example.com'), BlockedReason.unknown, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://working2.com'),
format: Format.unknown,
label: 'working1',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new TooManyRequestsError(new URL('https://example.com'), 10),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example.com'),
format: Format.unknown,
isExternal: true,
error: new TooManyTimeoutsError(new URL('https://example.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://working1.com'),
format: Format.unknown,
label: 'working2',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example2.com'),
format: Format.unknown,
isExternal: true,
error: new TypeError(),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example2.com'),
format: Format.unknown,
isExternal: true,
error: new TimeoutError(new URL('https://example2.com')),
error: new BlockedError(new URL('https://example2.com'), BlockedReason.cloudflare_challenge, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
@ -247,7 +152,7 @@ describe('resolve', () => {
url: new URL('https://example3.com'),
format: Format.unknown,
isExternal: true,
error: new QueueIsFullError(new URL('https://example3.com')),
error: new BlockedError(new URL('https://example3.com'), BlockedReason.cloudflare_censor, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
@ -258,7 +163,7 @@ describe('resolve', () => {
url: new URL('https://example4.com'),
format: Format.unknown,
isExternal: true,
error: new HttpError(new URL('https://example4.com'), 500, 'Internal Server Error', { 'x-foo': 'bar' }),
error: new BlockedError(new URL('https://example4.com'), BlockedReason.media_flow_proxy_auth, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
@ -269,7 +174,102 @@ describe('resolve', () => {
url: new URL('https://example5.com'),
format: Format.unknown,
isExternal: true,
error: new HttpError(new URL('https://example5.com'), 418, 'I\'m a tea pot', { 'x-foo': 'bar' }),
error: new BlockedError(new URL('https://example5.com'), BlockedReason.unknown, {}),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example6.com'),
format: Format.unknown,
label: 'working1',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example7.com'),
format: Format.unknown,
isExternal: true,
error: new TooManyRequestsError(new URL('https://example7.com'), 10),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example8.com'),
format: Format.unknown,
isExternal: true,
error: new TooManyTimeoutsError(new URL('https://example8.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example9.com'),
format: Format.unknown,
label: 'working2',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example10.com'),
format: Format.unknown,
isExternal: true,
error: new TypeError(),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example11.com'),
format: Format.unknown,
isExternal: true,
error: new TimeoutError(new URL('https://example11.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example12.com'),
format: Format.unknown,
isExternal: true,
error: new QueueIsFullError(new URL('https://example12.com')),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example13.com'),
format: Format.unknown,
isExternal: true,
error: new HttpError(new URL('https://example13.com'), 500, 'Internal Server Error', { 'x-foo': 'bar' }),
label: 'hoster.com',
ttl: this.ttl,
meta: {
countryCodes: [CountryCode.de],
},
},
{
url: new URL('https://example14.com'),
format: Format.unknown,
isExternal: true,
error: new HttpError(new URL('https://example14.com'), 418, 'I\'m a tea pot', { 'x-foo': 'bar' }),
label: 'hoster.com',
ttl: this.ttl,
meta: {

4
src/utils/StreamResolver.ts
View file

@ -144,6 +144,10 @@ export class StreamResolver {
streams.push(
...urlResults.filter(urlResult => (!urlResult.error || showErrors(ctx.config)) && !isResolutionExcluded(ctx.config, getClosestResolution(urlResult.meta?.height)))
.filter((urlResult, index, self) =>
// Remove duplicate URLs
index === self.findIndex(t => t.url.href === urlResult.url.href),
)
.map(urlResult => ({
...this.buildUrl(urlResult),
name: this.buildName(ctx, urlResult),

52
src/utils/__snapshots__/StreamResolver.test.ts.snap
View file

@ -10,7 +10,7 @@ exports[`resolve adds error info 1`] = `
},
"name": "WebStreamr 🇩🇪",
"title": "🔗 working1",
"url": "https://working2.com/",
"url": "https://example6.com/",
},
{
"behaviorHints": {
@ -19,14 +19,14 @@ exports[`resolve adds error info 1`] = `
},
"name": "WebStreamr 🇩🇪",
"title": "🔗 working2",
"url": "https://working1.com/",
"url": "https://example9.com/",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example1.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com",
},
@ -43,47 +43,47 @@ exports[`resolve adds error info 2`] = `
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example5.com/",
"externalUrl": "https://example14.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
❌ Request to example5.com failed with status 418 (I'm a tea pot). Request-id: test.",
❌ Request to example14.com failed with status 418 (I'm a tea pot). Request-id: test.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example4.com/",
"externalUrl": "https://example13.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
❌ Remote server example4.com has issues. We can't fix this, please try later again.",
❌ Remote server example13.com has issues. We can't fix this, please try later again.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example3.com/",
"externalUrl": "https://example12.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
⏳ Request queue for example3.com is full. Please try again later or consider self-hosting.",
⏳ Request queue for example12.com is full. Please try again later or consider self-hosting.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example2.com/",
"externalUrl": "https://example11.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
🐢 Request to example2.com timed out.",
🐢 Request to example11.com timed out.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example2.com/",
"externalUrl": "https://example10.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
❌ Request failed. Request-id: test.",
@ -93,37 +93,37 @@ exports[`resolve adds error info 2`] = `
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example8.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
🚦 Too many recent timeouts when requesting example.com. Please try again later.",
🚦 Too many recent timeouts when requesting example8.com. Please try again later.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example7.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
🚦 Request to example.com was rate-limited. Please try again later or consider self-hosting.",
🚦 Request to example7.com was rate-limited. Please try again later or consider self-hosting.",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example5.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
⚠️ Request to example.com was blocked. Reason: unknown",
⚠️ Request to example5.com was blocked. Reason: unknown",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example4.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
⚠️ MediaFlow Proxy authentication failed. Please set the correct password.",
@ -133,20 +133,20 @@ exports[`resolve adds error info 2`] = `
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example3.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
⚠️ Request to example.com was blocked. Reason: cloudflare_censor",
⚠️ Request to example3.com was blocked. Reason: cloudflare_censor",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example2.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com
⚠️ Request to example.com was blocked. Reason: cloudflare_challenge",
⚠️ Request to example2.com was blocked. Reason: cloudflare_challenge",
},
{
"behaviorHints": {
@ -155,7 +155,7 @@ exports[`resolve adds error info 2`] = `
},
"name": "WebStreamr 🇩🇪",
"title": "🔗 working1",
"url": "https://working2.com/",
"url": "https://example6.com/",
},
{
"behaviorHints": {
@ -164,14 +164,14 @@ exports[`resolve adds error info 2`] = `
},
"name": "WebStreamr 🇩🇪",
"title": "🔗 working2",
"url": "https://working1.com/",
"url": "https://example9.com/",
},
{
"behaviorHints": {
"bingeGroup": "webstreamr-undefined-undefined-de",
"notWebReady": true,
},
"externalUrl": "https://example.com/",
"externalUrl": "https://example1.com/",
"name": "WebStreamr 🇩🇪",
"title": "🔗 hoster.com",
},