Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

Merge branch 'main' into renovate/jsdom-27.x

Browse source
This commit is contained in:
webstreamr 2025-09-13 20:30:26 +02:00 committed by GitHub
commit 6e46f62878
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 189 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.release-please-manifest.json
View file

@ -1,3 +1,3 @@
{
".": "0.50.4"
".": "0.51.0"
}

19
CHANGELOG.md
View file

@ -1,5 +1,24 @@
# Changelog
## [0.51.0](https://github.com/webstreamr/webstreamr/compare/v0.50.4...v0.51.0) (2025-09-13)
### Miscellaneous Chores
* **fetcher:** log more infos about responses ([731bd33](https://github.com/webstreamr/webstreamr/commit/731bd334703f0592f083d4264c382c28acf68eb1))
* **fetcher:** log proxy usage, allow to disable proxy via config ([b67e1b8](https://github.com/webstreamr/webstreamr/commit/b67e1b85d634be413e004625b13403acc445ce72))
* log error count in final result log ([2421913](https://github.com/webstreamr/webstreamr/commit/24219135874be6cdecdb95cabfed86ad60d6c451))
### Features
* allow to disable cache ([4078f0c](https://github.com/webstreamr/webstreamr/commit/4078f0c404a76f4c5a3c12ec74108d3d03840e34))
### Reverts
* Revert "fix(extractor): use SuperVideo embed URls which are not triggering CF challenges" ([4351ddc](https://github.com/webstreamr/webstreamr/commit/4351ddc00dc349bde60968dc1490359e4720f924))
## [0.50.4](https://github.com/webstreamr/webstreamr/compare/v0.50.3...v0.50.4) (2025-09-12)

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "webstreamr",
"version": "0.50.4",
"version": "0.51.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "webstreamr",
"version": "0.50.4",
"version": "0.51.0",
"license": "MIT",
"dependencies": {
"@keyv/sqlite": "^4.0.5",

2
package.json
View file

@ -1,7 +1,7 @@
{
"name": "webstreamr",
"description": "Provides HTTP URLs from streaming websites.",
"version": "0.50.4",
"version": "0.51.0",
"type": "commonjs",
"scripts": {
"analyse": "tsc --noEmit --project ./tsconfig.dev.json",

4
src/extractor/ExtractorRegistry.ts
View file

@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite';
import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
import winston from 'winston';
import { Context, Meta, UrlResult } from '../types';
import { getCacheDir, isExtractorDisabled } from '../utils';
import { getCacheDir, isExtractorDisabled, noCache } from '../utils';
import { Extractor } from './Extractor';
export class ExtractorRegistry {
@ -41,7 +41,7 @@ export class ExtractorRegistry {
const storedDataRaw = await this.urlResultCache.getRaw<UrlResult[]>(cacheKey);
const expires = storedDataRaw?.expires;
if (storedDataRaw && expires) {
if (storedDataRaw && expires && !noCache(ctx.config)) {
// Ignore the cache randomly after at least 2/3 of the TTL passed to start refreshing results slowly
const refreshTimestamp = this.randomInteger(expires - extractor.ttl * (2 / 3), expires);
const now = Date.now();

4
src/source/Source.ts
View file

@ -4,7 +4,7 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable';
import { ContentType } from 'stremio-addon-sdk';
import { NotFoundError } from '../error';
import { Context, CountryCode, Meta } from '../types';
import { getCacheDir, Id } from '../utils';
import { getCacheDir, Id, noCache } from '../utils';
export interface SourceResult {
url: URL;
@ -44,7 +44,7 @@ export abstract class Source {
let sourceResults = (await Source.sourceResultCache.get<SourceResult[]>(cacheKey))
?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) }));
if (!sourceResults) {
if (!sourceResults || noCache(ctx.config)) {
try {
sourceResults = await this.handleInternal(ctx, type, id);
} catch (error) {

2
src/types.ts
View file

@ -17,7 +17,7 @@ export type CustomManifest = Manifest & {
};
};
export type Config = Partial<Record<CountryCode | 'showErrors' | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword' | 'proxyConfig', string> & Record<`disableExtractor_${string}`, string>>;
export type Config = Partial<Record<CountryCode | 'showErrors' | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword' | 'proxyConfig' | 'noCache', string> & Record<`disableExtractor_${string}`, string>>;
export enum CountryCode {
multi = 'multi',

23
src/utils/Fetcher.ts
View file

@ -7,7 +7,8 @@ import { fetch, Headers, RequestInit, Response } from 'undici';
import winston from 'winston';
import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error';
import { BlockedReason, Context } from '../types';
import { createDispatcher } from './dispatcher';
import { noCache } from './config';
import { createProxyAgent, getProxyForUrl } from './dispatcher';
import { envGet } from './env';
interface HttpCacheItem {
@ -233,9 +234,9 @@ export class Fetcher {
const cacheKey = this.determineCacheKey(url, init);
let httpCacheItem = await this.cacheGet(cacheKey);
const noCache = init?.noCache ?? false;
if (httpCacheItem && !noCache) {
this.logger.info(`Cached fetch ${request.method} ${url}`, ctx);
const disableCache = init?.noCache ?? noCache(ctx.config);
if (httpCacheItem && !disableCache) {
this.logger.info(`Cached fetch ${request.method} ${url}: ${httpCacheItem.status} (${httpCacheItem.statusText})`, ctx);
return this.handleHttpCacheItem(ctx, httpCacheItem, url, init);
}
@ -253,14 +254,16 @@ export class Fetcher {
};
protected async fetchWithTimeout(ctx: Context, url: URL, init?: CustomRequestInit, tryCount = 0): Promise<Response> {
this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx);
const proxyUrl = getProxyForUrl(ctx, url);
this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url} via proxy ${proxyUrl}`, ctx);
const isRateLimitedRaw = await this.rateLimitedCache.getRaw<true>(url.host);
/* istanbul ignore if */
if (isRateLimitedRaw && isRateLimitedRaw.value && isRateLimitedRaw.expires) {
const ttl = isRateLimitedRaw.expires - Date.now();
if (ttl <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) {
this.logger.info('Wait out rate limit', ctx);
this.logger.info(`Wait out rate limit for ${url}`, ctx);
await this.sleep(ttl);
@ -281,17 +284,17 @@ export class Fetcher {
finalUrl.username = '';
finalUrl.password = '';
const dispatcher = createDispatcher(ctx, url);
const finalInit = {
...init,
keepalive: true,
signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT),
...(/* istanbul ignore next */ dispatcher && { dispatcher }),
...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }),
};
response = await fetch(finalUrl, finalInit);
} catch (error) {
this.logger.info(`Got error ${error} for ${url}`, ctx);
if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) {
await this.increaseTimeoutsCount(url);
throw new TimeoutError();
@ -300,6 +303,8 @@ export class Fetcher {
throw error;
}
this.logger.info(`Got ${response.status} (${response.statusText}) for ${url}`, ctx);
await this.decreaseTimeoutsCount(url);
if (response.status === 429) {

9
src/utils/StreamResolver.ts
View file

@ -39,7 +39,7 @@ export class StreamResolver {
const streams: Stream[] = [];
let sourceErrorOccurred = false;
let sourceErrorCount = 0;
const urlResults: UrlResult[] = [];
const sourcePromises = sources.map(async (source) => {
if (!source.contentTypes.includes(type)) {
@ -55,7 +55,7 @@ export class StreamResolver {
urlResults.push(...sourceUrlResults.flat());
} catch (error) {
sourceErrorOccurred = true;
sourceErrorCount++;
if (showErrors(ctx.config)) {
streams.push({
@ -86,7 +86,8 @@ export class StreamResolver {
return a.label.localeCompare(b.label);
});
this.logger.info(`Return ${urlResults.length} streams`, ctx);
const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, sourceErrorCount);
this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx);
streams.push(
...urlResults.filter(urlResult => !urlResult.error || showErrors(ctx.config))
@ -106,7 +107,7 @@ export class StreamResolver {
})),
);
const ttl = !sourceErrorOccurred ? this.determineTtl(urlResults) : undefined;
const ttl = sourceErrorCount === 0 ? this.determineTtl(urlResults) : undefined;
return {
streams,

111
src/utils/__snapshots__/manifest.test.ts.snap
View file

@ -72,6 +72,11 @@ exports[`buildManifest default manifest 1`] = `
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
],
"description": "Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs.
@ -131,6 +136,11 @@ exports[`buildManifest disable extractors 1`] = `
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
{
"default": "checked",
"key": "disableExtractor_doodstream",
@ -145,10 +155,9 @@ exports[`buildManifest disable extractors 1`] = `
]
`;
exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = `
exports[`buildManifest has checked includeExternalUrls 1`] = `
[
{
"default": "checked",
"key": "showErrors",
"title": "Show errors",
"type": "checkbox",
@ -177,6 +186,89 @@ exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = `
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;
exports[`buildManifest has checked noCache 1`] = `
[
{
"key": "showErrors",
"title": "Show errors",
"type": "checkbox",
},
{
"key": "includeExternalUrls",
"title": "Include external URLs in results",
"type": "checkbox",
},
{
"default": "",
"key": "mediaFlowProxyUrl",
"title": "MediaFlow Proxy URL",
"type": "text",
},
{
"default": "",
"key": "mediaFlowProxyPassword",
"title": "MediaFlow Proxy Password",
"type": "password",
},
{
"default": "",
"key": "proxyConfig",
"title": "Proxy Config",
"type": "text",
},
{
"default": "checked",
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;
exports[`buildManifest has checked showErrors 1`] = `
[
{
"default": "checked",
"key": "showErrors",
"title": "Show errors",
"type": "checkbox",
},
{
"key": "includeExternalUrls",
"title": "Include external URLs in results",
"type": "checkbox",
},
{
"default": "",
"key": "mediaFlowProxyUrl",
"title": "MediaFlow Proxy URL",
"type": "text",
},
{
"default": "",
"key": "mediaFlowProxyPassword",
"title": "MediaFlow Proxy Password",
"type": "password",
},
{
"default": "",
"key": "proxyConfig",
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;
@ -227,6 +319,11 @@ exports[`buildManifest has checked source with appropriate config 1`] = `
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;
@ -285,6 +382,11 @@ exports[`buildManifest has unchecked source without a config 1`] = `
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;
@ -318,5 +420,10 @@ exports[`buildManifest showErrors and includeExternalUrls are unchecked by defau
"title": "Proxy Config",
"type": "text",
},
{
"key": "noCache",
"title": "Disable cache",
"type": "checkbox",
},
]
`;

2
src/utils/config.ts
View file

@ -9,6 +9,8 @@ export const showErrors = (config: Config): boolean => 'showErrors' in config;
export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config;
export const noCache = (config: Config): boolean => 'noCache' in config;
export const hasMultiEnabled = (config: Config): boolean => 'multi' in config;
export const disableExtractorConfigKey = (extractor: Extractor): string => `disableExtractor_${extractor.id}`;

22
src/utils/dispatcher.ts
View file

@ -3,15 +3,7 @@ import { minimatch } from 'minimatch';
import { Dispatcher, ProxyAgent } from 'undici';
import { Context } from '../types';
const createProxyAgent = (proxyUrl: URL): Dispatcher => {
if (proxyUrl.protocol === 'socks5:') {
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
}
return new ProxyAgent({ uri: proxyUrl.href });
};
const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => {
export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => {
const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG'];
if (proxyConfig) {
@ -22,16 +14,20 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined =
}
if (hostPattern === '*' || minimatch(url.host, hostPattern)) {
return createProxyAgent(new URL(proxy));
return proxy === 'false' ? undefined : new URL(proxy);
}
}
} else if (process.env['ALL_PROXY']) {
return createProxyAgent(new URL(process.env['ALL_PROXY']));
return new URL(process.env['ALL_PROXY']);
}
return undefined;
};
export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => {
return createBasicDispatcher(ctx, url);
export const createProxyAgent = (proxyUrl: URL): Dispatcher => {
if (proxyUrl.protocol === 'socks5:') {
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) });
}
return new ProxyAgent({ uri: proxyUrl.href });
};

16
src/utils/manifest.test.ts
View file

@ -50,8 +50,20 @@ describe('buildManifest', () => {
expect(manifest.config).toMatchSnapshot();
});
test('has checked showErrors and includeExternalUrls', () => {
const manifest = buildManifest([], [], { showErrors: 'on', includeExternalUrls: 'on' });
test('has checked showErrors', () => {
const manifest = buildManifest([], [], { showErrors: 'on' });
expect(manifest.config).toMatchSnapshot();
});
test('has checked includeExternalUrls', () => {
const manifest = buildManifest([], [], { includeExternalUrls: 'on' });
expect(manifest.config).toMatchSnapshot();
});
test('has checked noCache', () => {
const manifest = buildManifest([], [], { noCache: 'on' });
expect(manifest.config).toMatchSnapshot();
});

9
src/utils/manifest.ts
View file

@ -10,7 +10,7 @@ const typedEntries = <T extends object>(obj: T): [keyof T, T[keyof T]][] => (Obj
export const buildManifest = (sources: Source[], extractors: Extractor[], config: Config): CustomManifest => {
const manifest: CustomManifest = {
id: envGetAppId(),
version: '0.50.4', // x-release-please-version
version: '0.51.0', // x-release-please-version
name: envGetAppName(),
description: 'Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs.',
resources: [
@ -96,6 +96,13 @@ export const buildManifest = (sources: Source[], extractors: Extractor[], config
default: config['proxyConfig'] ?? '',
});
manifest.config.push({
key: 'noCache',
type: 'checkbox',
title: 'Disable cache',
...('noCache' in config && { default: 'checked' }),
});
extractors.forEach((extractor) => {
if (extractor.id === 'external') {
return;