diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b07164f..75d439a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.50.4" + ".": "0.51.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 7075a3e..59c54db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [0.51.0](https://github.com/webstreamr/webstreamr/compare/v0.50.4...v0.51.0) (2025-09-13) + + +### Miscellaneous Chores + +* **fetcher:** log more infos about responses ([731bd33](https://github.com/webstreamr/webstreamr/commit/731bd334703f0592f083d4264c382c28acf68eb1)) +* **fetcher:** log proxy usage, allow to disable proxy via config ([b67e1b8](https://github.com/webstreamr/webstreamr/commit/b67e1b85d634be413e004625b13403acc445ce72)) +* log error count in final result log ([2421913](https://github.com/webstreamr/webstreamr/commit/24219135874be6cdecdb95cabfed86ad60d6c451)) + + +### Features + +* allow to disable cache ([4078f0c](https://github.com/webstreamr/webstreamr/commit/4078f0c404a76f4c5a3c12ec74108d3d03840e34)) + + +### Reverts + +* Revert "fix(extractor): use SuperVideo embed URls which are not triggering CF challenges" ([4351ddc](https://github.com/webstreamr/webstreamr/commit/4351ddc00dc349bde60968dc1490359e4720f924)) + ## [0.50.4](https://github.com/webstreamr/webstreamr/compare/v0.50.3...v0.50.4) (2025-09-12) diff --git a/package-lock.json b/package-lock.json index 6f59579..e6a2171 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "webstreamr", - "version": "0.50.4", + "version": "0.51.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "webstreamr", - "version": "0.50.4", + "version": "0.51.0", "license": "MIT", "dependencies": { "@keyv/sqlite": "^4.0.5", diff --git a/package.json b/package.json index bd522cd..a9c6544 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "webstreamr", "description": "Provides HTTP URLs from streaming websites.", - "version": "0.50.4", + "version": "0.51.0", "type": "commonjs", "scripts": { "analyse": "tsc --noEmit --project ./tsconfig.dev.json", diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index b65bc83..96a45ad 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite'; import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import winston from 'winston'; import { Context, Meta, UrlResult } from '../types'; -import { getCacheDir, isExtractorDisabled } from '../utils'; +import { getCacheDir, isExtractorDisabled, noCache } from '../utils'; import { Extractor } from './Extractor'; export class ExtractorRegistry { @@ -41,7 +41,7 @@ export class ExtractorRegistry { const storedDataRaw = await this.urlResultCache.getRaw(cacheKey); const expires = storedDataRaw?.expires; - if (storedDataRaw && expires) { + if (storedDataRaw && expires && !noCache(ctx.config)) { // Ignore the cache randomly after at least 2/3 of the TTL passed to start refreshing results slowly const refreshTimestamp = this.randomInteger(expires - extractor.ttl * (2 / 3), expires); const now = Date.now(); diff --git a/src/source/Source.ts b/src/source/Source.ts index 3f5feec..2ef1bd6 100644 --- a/src/source/Source.ts +++ b/src/source/Source.ts @@ -4,7 +4,7 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import { ContentType } from 'stremio-addon-sdk'; import { NotFoundError } from '../error'; import { Context, CountryCode, Meta } from '../types'; -import { getCacheDir, Id } from '../utils'; +import { getCacheDir, Id, noCache } from '../utils'; export interface SourceResult { url: URL; @@ -44,7 +44,7 @@ export abstract class Source { let sourceResults = (await Source.sourceResultCache.get(cacheKey)) ?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) })); - if (!sourceResults) { + if (!sourceResults || noCache(ctx.config)) { try { sourceResults = await this.handleInternal(ctx, type, id); } catch (error) { diff --git a/src/types.ts b/src/types.ts index 0857a68..c04729e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,7 +17,7 @@ export type CustomManifest = Manifest & { }; }; -export type Config = Partial & Record<`disableExtractor_${string}`, string>>; +export type Config = Partial & Record<`disableExtractor_${string}`, string>>; export enum CountryCode { multi = 'multi', diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 4a96035..77be3a1 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -7,7 +7,8 @@ import { fetch, Headers, RequestInit, Response } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; -import { createDispatcher } from './dispatcher'; +import { noCache } from './config'; +import { createProxyAgent, getProxyForUrl } from './dispatcher'; import { envGet } from './env'; interface HttpCacheItem { @@ -233,9 +234,9 @@ export class Fetcher { const cacheKey = this.determineCacheKey(url, init); let httpCacheItem = await this.cacheGet(cacheKey); - const noCache = init?.noCache ?? false; - if (httpCacheItem && !noCache) { - this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); + const disableCache = init?.noCache ?? noCache(ctx.config); + if (httpCacheItem && !disableCache) { + this.logger.info(`Cached fetch ${request.method} ${url}: ${httpCacheItem.status} (${httpCacheItem.statusText})`, ctx); return this.handleHttpCacheItem(ctx, httpCacheItem, url, init); } @@ -253,14 +254,16 @@ export class Fetcher { }; protected async fetchWithTimeout(ctx: Context, url: URL, init?: CustomRequestInit, tryCount = 0): Promise { - this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx); + const proxyUrl = getProxyForUrl(ctx, url); + + this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url} via proxy ${proxyUrl}`, ctx); const isRateLimitedRaw = await this.rateLimitedCache.getRaw(url.host); /* istanbul ignore if */ if (isRateLimitedRaw && isRateLimitedRaw.value && isRateLimitedRaw.expires) { const ttl = isRateLimitedRaw.expires - Date.now(); if (ttl <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) { - this.logger.info('Wait out rate limit', ctx); + this.logger.info(`Wait out rate limit for ${url}`, ctx); await this.sleep(ttl); @@ -281,17 +284,17 @@ export class Fetcher { finalUrl.username = ''; finalUrl.password = ''; - const dispatcher = createDispatcher(ctx, url); - const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT), - ...(/* istanbul ignore next */ dispatcher && { dispatcher }), + ...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }), }; response = await fetch(finalUrl, finalInit); } catch (error) { + this.logger.info(`Got error ${error} for ${url}`, ctx); + if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) { await this.increaseTimeoutsCount(url); throw new TimeoutError(); @@ -300,6 +303,8 @@ export class Fetcher { throw error; } + this.logger.info(`Got ${response.status} (${response.statusText}) for ${url}`, ctx); + await this.decreaseTimeoutsCount(url); if (response.status === 429) { diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index b19fba3..5b78976 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -39,7 +39,7 @@ export class StreamResolver { const streams: Stream[] = []; - let sourceErrorOccurred = false; + let sourceErrorCount = 0; const urlResults: UrlResult[] = []; const sourcePromises = sources.map(async (source) => { if (!source.contentTypes.includes(type)) { @@ -55,7 +55,7 @@ export class StreamResolver { urlResults.push(...sourceUrlResults.flat()); } catch (error) { - sourceErrorOccurred = true; + sourceErrorCount++; if (showErrors(ctx.config)) { streams.push({ @@ -86,7 +86,8 @@ export class StreamResolver { return a.label.localeCompare(b.label); }); - this.logger.info(`Return ${urlResults.length} streams`, ctx); + const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, sourceErrorCount); + this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx); streams.push( ...urlResults.filter(urlResult => !urlResult.error || showErrors(ctx.config)) @@ -106,7 +107,7 @@ export class StreamResolver { })), ); - const ttl = !sourceErrorOccurred ? this.determineTtl(urlResults) : undefined; + const ttl = sourceErrorCount === 0 ? this.determineTtl(urlResults) : undefined; return { streams, diff --git a/src/utils/__snapshots__/manifest.test.ts.snap b/src/utils/__snapshots__/manifest.test.ts.snap index 452b4fb..e1a58a7 100644 --- a/src/utils/__snapshots__/manifest.test.ts.snap +++ b/src/utils/__snapshots__/manifest.test.ts.snap @@ -72,6 +72,11 @@ exports[`buildManifest default manifest 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ], "description": "Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs. @@ -131,6 +136,11 @@ exports[`buildManifest disable extractors 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, { "default": "checked", "key": "disableExtractor_doodstream", @@ -145,10 +155,9 @@ exports[`buildManifest disable extractors 1`] = ` ] `; -exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` +exports[`buildManifest has checked includeExternalUrls 1`] = ` [ { - "default": "checked", "key": "showErrors", "title": "Show errors", "type": "checkbox", @@ -177,6 +186,89 @@ exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked noCache 1`] = ` +[ + { + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "default": "checked", + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked showErrors 1`] = ` +[ + { + "default": "checked", + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -227,6 +319,11 @@ exports[`buildManifest has checked source with appropriate config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -285,6 +382,11 @@ exports[`buildManifest has unchecked source without a config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -318,5 +420,10 @@ exports[`buildManifest showErrors and includeExternalUrls are unchecked by defau "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; diff --git a/src/utils/config.ts b/src/utils/config.ts index fd503db..2702554 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,6 +9,8 @@ export const showErrors = (config: Config): boolean => 'showErrors' in config; export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config; +export const noCache = (config: Config): boolean => 'noCache' in config; + export const hasMultiEnabled = (config: Config): boolean => 'multi' in config; export const disableExtractorConfigKey = (extractor: Extractor): string => `disableExtractor_${extractor.id}`; diff --git a/src/utils/dispatcher.ts b/src/utils/dispatcher.ts index e15b340..8e05c77 100644 --- a/src/utils/dispatcher.ts +++ b/src/utils/dispatcher.ts @@ -3,15 +3,7 @@ import { minimatch } from 'minimatch'; import { Dispatcher, ProxyAgent } from 'undici'; import { Context } from '../types'; -const createProxyAgent = (proxyUrl: URL): Dispatcher => { - if (proxyUrl.protocol === 'socks5:') { - return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); - } - - return new ProxyAgent({ uri: proxyUrl.href }); -}; - -const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { +export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => { const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG']; if (proxyConfig) { @@ -22,16 +14,20 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined = } if (hostPattern === '*' || minimatch(url.host, hostPattern)) { - return createProxyAgent(new URL(proxy)); + return proxy === 'false' ? undefined : new URL(proxy); } } } else if (process.env['ALL_PROXY']) { - return createProxyAgent(new URL(process.env['ALL_PROXY'])); + return new URL(process.env['ALL_PROXY']); } return undefined; }; -export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { - return createBasicDispatcher(ctx, url); +export const createProxyAgent = (proxyUrl: URL): Dispatcher => { + if (proxyUrl.protocol === 'socks5:') { + return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); + } + + return new ProxyAgent({ uri: proxyUrl.href }); }; diff --git a/src/utils/manifest.test.ts b/src/utils/manifest.test.ts index bbe709d..9e71309 100644 --- a/src/utils/manifest.test.ts +++ b/src/utils/manifest.test.ts @@ -50,8 +50,20 @@ describe('buildManifest', () => { expect(manifest.config).toMatchSnapshot(); }); - test('has checked showErrors and includeExternalUrls', () => { - const manifest = buildManifest([], [], { showErrors: 'on', includeExternalUrls: 'on' }); + test('has checked showErrors', () => { + const manifest = buildManifest([], [], { showErrors: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked includeExternalUrls', () => { + const manifest = buildManifest([], [], { includeExternalUrls: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked noCache', () => { + const manifest = buildManifest([], [], { noCache: 'on' }); expect(manifest.config).toMatchSnapshot(); }); diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index e954fc5..2b9e238 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -10,7 +10,7 @@ const typedEntries = (obj: T): [keyof T, T[keyof T]][] => (Obj export const buildManifest = (sources: Source[], extractors: Extractor[], config: Config): CustomManifest => { const manifest: CustomManifest = { id: envGetAppId(), - version: '0.50.4', // x-release-please-version + version: '0.51.0', // x-release-please-version name: envGetAppName(), description: 'Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs.', resources: [ @@ -96,6 +96,13 @@ export const buildManifest = (sources: Source[], extractors: Extractor[], config default: config['proxyConfig'] ?? '', }); + manifest.config.push({ + key: 'noCache', + type: 'checkbox', + title: 'Disable cache', + ...('noCache' in config && { default: 'checked' }), + }); + extractors.forEach((extractor) => { if (extractor.id === 'external') { return;