From 731bd334703f0592f083d4264c382c28acf68eb1 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:41:28 +0000 Subject: [PATCH 1/5] chore(fetcher): log more infos about responses --- src/utils/Fetcher.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 4a96035..9731d44 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -235,7 +235,7 @@ export class Fetcher { let httpCacheItem = await this.cacheGet(cacheKey); const noCache = init?.noCache ?? false; if (httpCacheItem && !noCache) { - this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); + this.logger.info(`Cached fetch ${request.method} ${url}: ${httpCacheItem.status} (${httpCacheItem.statusText})`, ctx); return this.handleHttpCacheItem(ctx, httpCacheItem, url, init); } @@ -260,7 +260,7 @@ export class Fetcher { if (isRateLimitedRaw && isRateLimitedRaw.value && isRateLimitedRaw.expires) { const ttl = isRateLimitedRaw.expires - Date.now(); if (ttl <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) { - this.logger.info('Wait out rate limit', ctx); + this.logger.info(`Wait out rate limit for ${url}`, ctx); await this.sleep(ttl); @@ -292,6 +292,8 @@ export class Fetcher { response = await fetch(finalUrl, finalInit); } catch (error) { + this.logger.info(`Got error ${error} for ${url}`, ctx); + if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) { await this.increaseTimeoutsCount(url); throw new TimeoutError(); @@ -300,6 +302,8 @@ export class Fetcher { throw error; } + this.logger.info(`Got ${response.status} (${response.statusText}) for ${url}`, ctx); + await this.decreaseTimeoutsCount(url); if (response.status === 429) { From 4078f0c404a76f4c5a3c12ec74108d3d03840e34 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:51:12 +0000 Subject: [PATCH 2/5] feat: allow to disable cache --- src/extractor/ExtractorRegistry.ts | 4 +- src/source/Source.ts | 4 +- src/types.ts | 2 +- src/utils/Fetcher.ts | 5 +- src/utils/__snapshots__/manifest.test.ts.snap | 111 +++++++++++++++++- src/utils/config.ts | 2 + src/utils/manifest.test.ts | 16 ++- src/utils/manifest.ts | 7 ++ 8 files changed, 140 insertions(+), 11 deletions(-) diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index b65bc83..96a45ad 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite'; import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import winston from 'winston'; import { Context, Meta, UrlResult } from '../types'; -import { getCacheDir, isExtractorDisabled } from '../utils'; +import { getCacheDir, isExtractorDisabled, noCache } from '../utils'; import { Extractor } from './Extractor'; export class ExtractorRegistry { @@ -41,7 +41,7 @@ export class ExtractorRegistry { const storedDataRaw = await this.urlResultCache.getRaw(cacheKey); const expires = storedDataRaw?.expires; - if (storedDataRaw && expires) { + if (storedDataRaw && expires && !noCache(ctx.config)) { // Ignore the cache randomly after at least 2/3 of the TTL passed to start refreshing results slowly const refreshTimestamp = this.randomInteger(expires - extractor.ttl * (2 / 3), expires); const now = Date.now(); diff --git a/src/source/Source.ts b/src/source/Source.ts index 3f5feec..2ef1bd6 100644 --- a/src/source/Source.ts +++ b/src/source/Source.ts @@ -4,7 +4,7 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import { ContentType } from 'stremio-addon-sdk'; import { NotFoundError } from '../error'; import { Context, CountryCode, Meta } from '../types'; -import { getCacheDir, Id } from '../utils'; +import { getCacheDir, Id, noCache } from '../utils'; export interface SourceResult { url: URL; @@ -44,7 +44,7 @@ export abstract class Source { let sourceResults = (await Source.sourceResultCache.get(cacheKey)) ?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) })); - if (!sourceResults) { + if (!sourceResults || noCache(ctx.config)) { try { sourceResults = await this.handleInternal(ctx, type, id); } catch (error) { diff --git a/src/types.ts b/src/types.ts index 0857a68..c04729e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,7 +17,7 @@ export type CustomManifest = Manifest & { }; }; -export type Config = Partial & Record<`disableExtractor_${string}`, string>>; +export type Config = Partial & Record<`disableExtractor_${string}`, string>>; export enum CountryCode { multi = 'multi', diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 9731d44..eb5e9c0 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -7,6 +7,7 @@ import { fetch, Headers, RequestInit, Response } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; +import { noCache } from './config'; import { createDispatcher } from './dispatcher'; import { envGet } from './env'; @@ -233,8 +234,8 @@ export class Fetcher { const cacheKey = this.determineCacheKey(url, init); let httpCacheItem = await this.cacheGet(cacheKey); - const noCache = init?.noCache ?? false; - if (httpCacheItem && !noCache) { + const disableCache = init?.noCache ?? noCache(ctx.config); + if (httpCacheItem && !disableCache) { this.logger.info(`Cached fetch ${request.method} ${url}: ${httpCacheItem.status} (${httpCacheItem.statusText})`, ctx); return this.handleHttpCacheItem(ctx, httpCacheItem, url, init); } diff --git a/src/utils/__snapshots__/manifest.test.ts.snap b/src/utils/__snapshots__/manifest.test.ts.snap index 452b4fb..e1a58a7 100644 --- a/src/utils/__snapshots__/manifest.test.ts.snap +++ b/src/utils/__snapshots__/manifest.test.ts.snap @@ -72,6 +72,11 @@ exports[`buildManifest default manifest 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ], "description": "Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs. @@ -131,6 +136,11 @@ exports[`buildManifest disable extractors 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, { "default": "checked", "key": "disableExtractor_doodstream", @@ -145,10 +155,9 @@ exports[`buildManifest disable extractors 1`] = ` ] `; -exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` +exports[`buildManifest has checked includeExternalUrls 1`] = ` [ { - "default": "checked", "key": "showErrors", "title": "Show errors", "type": "checkbox", @@ -177,6 +186,89 @@ exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked noCache 1`] = ` +[ + { + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "default": "checked", + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked showErrors 1`] = ` +[ + { + "default": "checked", + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -227,6 +319,11 @@ exports[`buildManifest has checked source with appropriate config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -285,6 +382,11 @@ exports[`buildManifest has unchecked source without a config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -318,5 +420,10 @@ exports[`buildManifest showErrors and includeExternalUrls are unchecked by defau "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; diff --git a/src/utils/config.ts b/src/utils/config.ts index fd503db..2702554 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,6 +9,8 @@ export const showErrors = (config: Config): boolean => 'showErrors' in config; export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config; +export const noCache = (config: Config): boolean => 'noCache' in config; + export const hasMultiEnabled = (config: Config): boolean => 'multi' in config; export const disableExtractorConfigKey = (extractor: Extractor): string => `disableExtractor_${extractor.id}`; diff --git a/src/utils/manifest.test.ts b/src/utils/manifest.test.ts index bbe709d..9e71309 100644 --- a/src/utils/manifest.test.ts +++ b/src/utils/manifest.test.ts @@ -50,8 +50,20 @@ describe('buildManifest', () => { expect(manifest.config).toMatchSnapshot(); }); - test('has checked showErrors and includeExternalUrls', () => { - const manifest = buildManifest([], [], { showErrors: 'on', includeExternalUrls: 'on' }); + test('has checked showErrors', () => { + const manifest = buildManifest([], [], { showErrors: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked includeExternalUrls', () => { + const manifest = buildManifest([], [], { includeExternalUrls: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked noCache', () => { + const manifest = buildManifest([], [], { noCache: 'on' }); expect(manifest.config).toMatchSnapshot(); }); diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index e954fc5..a1c8d48 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -96,6 +96,13 @@ export const buildManifest = (sources: Source[], extractors: Extractor[], config default: config['proxyConfig'] ?? '', }); + manifest.config.push({ + key: 'noCache', + type: 'checkbox', + title: 'Disable cache', + ...('noCache' in config && { default: 'checked' }), + }); + extractors.forEach((extractor) => { if (extractor.id === 'external') { return; From 24219135874be6cdecdb95cabfed86ad60d6c451 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:59:21 +0000 Subject: [PATCH 3/5] chore: log error count in final result log --- src/utils/StreamResolver.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index b19fba3..1e4ac48 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -86,7 +86,8 @@ export class StreamResolver { return a.label.localeCompare(b.label); }); - this.logger.info(`Return ${urlResults.length} streams`, ctx); + const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, 0); + this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx); streams.push( ...urlResults.filter(urlResult => !urlResult.error || showErrors(ctx.config)) From b67e1b85d634be413e004625b13403acc445ce72 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:27:17 +0000 Subject: [PATCH 4/5] chore(fetcher): log proxy usage, allow to disable proxy via config --- src/utils/Fetcher.ts | 10 +++++----- src/utils/StreamResolver.ts | 8 ++++---- src/utils/dispatcher.ts | 22 +++++++++------------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index eb5e9c0..77be3a1 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -8,7 +8,7 @@ import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; import { noCache } from './config'; -import { createDispatcher } from './dispatcher'; +import { createProxyAgent, getProxyForUrl } from './dispatcher'; import { envGet } from './env'; interface HttpCacheItem { @@ -254,7 +254,9 @@ export class Fetcher { }; protected async fetchWithTimeout(ctx: Context, url: URL, init?: CustomRequestInit, tryCount = 0): Promise { - this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx); + const proxyUrl = getProxyForUrl(ctx, url); + + this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url} via proxy ${proxyUrl}`, ctx); const isRateLimitedRaw = await this.rateLimitedCache.getRaw(url.host); /* istanbul ignore if */ @@ -282,13 +284,11 @@ export class Fetcher { finalUrl.username = ''; finalUrl.password = ''; - const dispatcher = createDispatcher(ctx, url); - const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT), - ...(/* istanbul ignore next */ dispatcher && { dispatcher }), + ...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }), }; response = await fetch(finalUrl, finalInit); diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index 1e4ac48..5b78976 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -39,7 +39,7 @@ export class StreamResolver { const streams: Stream[] = []; - let sourceErrorOccurred = false; + let sourceErrorCount = 0; const urlResults: UrlResult[] = []; const sourcePromises = sources.map(async (source) => { if (!source.contentTypes.includes(type)) { @@ -55,7 +55,7 @@ export class StreamResolver { urlResults.push(...sourceUrlResults.flat()); } catch (error) { - sourceErrorOccurred = true; + sourceErrorCount++; if (showErrors(ctx.config)) { streams.push({ @@ -86,7 +86,7 @@ export class StreamResolver { return a.label.localeCompare(b.label); }); - const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, 0); + const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, sourceErrorCount); this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx); streams.push( @@ -107,7 +107,7 @@ export class StreamResolver { })), ); - const ttl = !sourceErrorOccurred ? this.determineTtl(urlResults) : undefined; + const ttl = sourceErrorCount === 0 ? this.determineTtl(urlResults) : undefined; return { streams, diff --git a/src/utils/dispatcher.ts b/src/utils/dispatcher.ts index e15b340..8e05c77 100644 --- a/src/utils/dispatcher.ts +++ b/src/utils/dispatcher.ts @@ -3,15 +3,7 @@ import { minimatch } from 'minimatch'; import { Dispatcher, ProxyAgent } from 'undici'; import { Context } from '../types'; -const createProxyAgent = (proxyUrl: URL): Dispatcher => { - if (proxyUrl.protocol === 'socks5:') { - return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); - } - - return new ProxyAgent({ uri: proxyUrl.href }); -}; - -const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { +export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => { const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG']; if (proxyConfig) { @@ -22,16 +14,20 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined = } if (hostPattern === '*' || minimatch(url.host, hostPattern)) { - return createProxyAgent(new URL(proxy)); + return proxy === 'false' ? undefined : new URL(proxy); } } } else if (process.env['ALL_PROXY']) { - return createProxyAgent(new URL(process.env['ALL_PROXY'])); + return new URL(process.env['ALL_PROXY']); } return undefined; }; -export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { - return createBasicDispatcher(ctx, url); +export const createProxyAgent = (proxyUrl: URL): Dispatcher => { + if (proxyUrl.protocol === 'socks5:') { + return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); + } + + return new ProxyAgent({ uri: proxyUrl.href }); }; From a79b0bdfdd3f7dd26026b71cdb8430c3bd07624c Mon Sep 17 00:00:00 2001 From: webstreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:28:43 +0200 Subject: [PATCH 5/5] chore(release): release v0.51.0 (#349) --- .release-please-manifest.json | 2 +- CHANGELOG.md | 19 +++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- src/utils/manifest.ts | 2 +- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b07164f..75d439a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.50.4" + ".": "0.51.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 7075a3e..59c54db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [0.51.0](https://github.com/webstreamr/webstreamr/compare/v0.50.4...v0.51.0) (2025-09-13) + + +### Miscellaneous Chores + +* **fetcher:** log more infos about responses ([731bd33](https://github.com/webstreamr/webstreamr/commit/731bd334703f0592f083d4264c382c28acf68eb1)) +* **fetcher:** log proxy usage, allow to disable proxy via config ([b67e1b8](https://github.com/webstreamr/webstreamr/commit/b67e1b85d634be413e004625b13403acc445ce72)) +* log error count in final result log ([2421913](https://github.com/webstreamr/webstreamr/commit/24219135874be6cdecdb95cabfed86ad60d6c451)) + + +### Features + +* allow to disable cache ([4078f0c](https://github.com/webstreamr/webstreamr/commit/4078f0c404a76f4c5a3c12ec74108d3d03840e34)) + + +### Reverts + +* Revert "fix(extractor): use SuperVideo embed URls which are not triggering CF challenges" ([4351ddc](https://github.com/webstreamr/webstreamr/commit/4351ddc00dc349bde60968dc1490359e4720f924)) + ## [0.50.4](https://github.com/webstreamr/webstreamr/compare/v0.50.3...v0.50.4) (2025-09-12) diff --git a/package-lock.json b/package-lock.json index ad1ea98..0f40aa7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "webstreamr", - "version": "0.50.4", + "version": "0.51.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "webstreamr", - "version": "0.50.4", + "version": "0.51.0", "license": "MIT", "dependencies": { "@keyv/sqlite": "^4.0.5", diff --git a/package.json b/package.json index c5cbe89..aaecc89 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "webstreamr", "description": "Provides HTTP URLs from streaming websites.", - "version": "0.50.4", + "version": "0.51.0", "type": "commonjs", "scripts": { "analyse": "tsc --noEmit --project ./tsconfig.dev.json", diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index a1c8d48..2b9e238 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -10,7 +10,7 @@ const typedEntries = (obj: T): [keyof T, T[keyof T]][] => (Obj export const buildManifest = (sources: Source[], extractors: Extractor[], config: Config): CustomManifest => { const manifest: CustomManifest = { id: envGetAppId(), - version: '0.50.4', // x-release-please-version + version: '0.51.0', // x-release-please-version name: envGetAppName(), description: 'Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs.', resources: [