feat: add support for proxy per host using minimatch
This commit is contained in:
WebStreamr
parent
2f370d1b9c
commit
b513a8926b
7 changed files with 224 additions and 21 deletions
153
package-lock.json
generated
153
package-lock.json
generated
|
|
@ -19,6 +19,7 @@
|
|||
"fetch-socks": "^1.3.2",
|
||||
"http-cache-semantics": "^4.2.0",
|
||||
"jsdom": "^26.1.0",
|
||||
"minimatch": "^10.0.3",
|
||||
"randomstring": "^1.3.1",
|
||||
"rot13-cipher": "^1.0.0",
|
||||
"slugify": "^1.6.6",
|
||||
|
|
@ -842,6 +843,19 @@
|
|||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/config-array/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/config-helpers": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.1.tgz",
|
||||
|
|
@ -909,6 +923,19 @@
|
|||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/eslintrc/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/js": {
|
||||
"version": "9.35.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.35.0.tgz",
|
||||
|
|
@ -1005,6 +1032,27 @@
|
|||
"url": "https://github.com/sponsors/nzakas"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/balanced-match": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
|
||||
"integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/brace-expansion": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz",
|
||||
"integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@isaacs/balanced-match": "^4.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/cliui": {
|
||||
"version": "8.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
|
||||
|
|
@ -3341,6 +3389,19 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/cacache/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"license": "ISC",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/cacache/node_modules/minipass": {
|
||||
"version": "3.3.6",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
|
||||
|
|
@ -4703,6 +4764,19 @@
|
|||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-plugin-import/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-scope": {
|
||||
"version": "8.4.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz",
|
||||
|
|
@ -4766,6 +4840,19 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint/node_modules/p-locate": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
|
||||
|
|
@ -7786,16 +7873,18 @@
|
|||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"devOptional": true,
|
||||
"version": "10.0.3",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz",
|
||||
"integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
"@isaacs/brace-expansion": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
"node": "20 || >=22"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/minimist": {
|
||||
|
|
@ -8164,6 +8253,19 @@
|
|||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"license": "ISC",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp/node_modules/semver": {
|
||||
"version": "7.7.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
|
||||
|
|
@ -8230,6 +8332,19 @@
|
|||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/nodemon/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/nodemon/node_modules/semver": {
|
||||
"version": "7.7.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
|
||||
|
|
@ -9280,6 +9395,19 @@
|
|||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/rimraf/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"license": "ISC",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/rot13-cipher": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/rot13-cipher/-/rot13-cipher-1.0.0.tgz",
|
||||
|
|
@ -10344,6 +10472,19 @@
|
|||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/test-exclude/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/text-hex": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@
|
|||
"fetch-socks": "^1.3.2",
|
||||
"http-cache-semantics": "^4.2.0",
|
||||
"jsdom": "^26.1.0",
|
||||
"minimatch": "^10.0.3",
|
||||
"randomstring": "^1.3.1",
|
||||
"rot13-cipher": "^1.0.0",
|
||||
"slugify": "^1.6.6",
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ export type CustomManifest = Manifest & {
|
|||
};
|
||||
};
|
||||
|
||||
export type Config = Partial<Record<CountryCode | 'showErrors' | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword', string> & Record<`disableExtractor_${string}`, string>>;
|
||||
export type Config = Partial<Record<CountryCode | 'showErrors' | 'includeExternalUrls' | 'mediaFlowProxyUrl' | 'mediaFlowProxyPassword' | 'proxyConfig', string> & Record<`disableExtractor_${string}`, string>>;
|
||||
|
||||
export enum CountryCode {
|
||||
multi = 'multi',
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ export type CustomRequestInit = RequestInit & {
|
|||
|
||||
export class Fetcher {
|
||||
private readonly MIN_CACHE_TTL = 900000; // 15m
|
||||
private readonly DEFAULT_TIMEOUT = 5000;
|
||||
private readonly DEFAULT_TIMEOUT = 10000;
|
||||
private readonly DEFAULT_QUEUE_LIMIT = 5;
|
||||
private readonly DEFAULT_QUEUE_TIMEOUT = 5000;
|
||||
private readonly DEFAULT_TIMEOUTS_COUNT_THROW = 30;
|
||||
|
|
@ -283,7 +283,7 @@ export class Fetcher {
|
|||
finalUrl.username = '';
|
||||
finalUrl.password = '';
|
||||
|
||||
const finalInit = { ...init, keepalive: true, signal: controller.signal, dispatcher: createDispatcher() };
|
||||
const finalInit = { ...init, keepalive: true, signal: controller.signal, dispatcher: createDispatcher(ctx, url) };
|
||||
|
||||
response = await fetch(finalUrl, finalInit);
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -66,6 +66,12 @@ exports[`buildManifest default manifest 1`] = `
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
],
|
||||
"description": "Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs.
|
||||
|
||||
|
|
@ -119,6 +125,12 @@ exports[`buildManifest disable extractors 1`] = `
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
{
|
||||
"default": "checked",
|
||||
"key": "disableExtractor_doodstream",
|
||||
|
|
@ -159,6 +171,12 @@ exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = `
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
|
|
@ -203,6 +221,12 @@ exports[`buildManifest has checked source with appropriate config 1`] = `
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
|
|
@ -255,6 +279,12 @@ exports[`buildManifest has unchecked source without a config 1`] = `
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
|
|
@ -282,5 +312,11 @@ exports[`buildManifest showErrors and includeExternalUrls are unchecked by defau
|
|||
"title": "MediaFlow Proxy Password",
|
||||
"type": "password",
|
||||
},
|
||||
{
|
||||
"default": "",
|
||||
"key": "proxyConfig",
|
||||
"title": "Proxy Config",
|
||||
"type": "text",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -1,21 +1,39 @@
|
|||
import { socksDispatcher } from 'fetch-socks';
|
||||
import { minimatch } from 'minimatch';
|
||||
import { Agent, Dispatcher, interceptors, ProxyAgent } from 'undici';
|
||||
import { Context } from '../types';
|
||||
|
||||
const createBasicDispatcher = (): Dispatcher => {
|
||||
if (process.env['ALL_PROXY']) {
|
||||
const proxyUrl = new URL(process.env['ALL_PROXY']);
|
||||
if (proxyUrl.protocol === 'socks5:') {
|
||||
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }, { allowH2: true });
|
||||
} else {
|
||||
return new ProxyAgent({ uri: proxyUrl.href, allowH2: true });
|
||||
}
|
||||
} else {
|
||||
return new Agent({ allowH2: true });
|
||||
const createProxyAgent = (proxyUrl: URL): Dispatcher => {
|
||||
if (proxyUrl.protocol === 'socks5:') {
|
||||
return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }, { allowH2: true });
|
||||
}
|
||||
|
||||
return new ProxyAgent({ uri: proxyUrl.href, allowH2: true });
|
||||
};
|
||||
|
||||
export const createDispatcher = (): Dispatcher => {
|
||||
return createBasicDispatcher().compose(
|
||||
const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher => {
|
||||
const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG'];
|
||||
|
||||
if (proxyConfig) {
|
||||
for (const rule of proxyConfig.split(',')) {
|
||||
const [hostPattern, proxy] = rule.split(/:(.+)/);
|
||||
if (!hostPattern || !proxy) {
|
||||
throw new Error(`Proxy rule "${rule}" is invalid.`);
|
||||
}
|
||||
|
||||
if (hostPattern === '*' || minimatch(url.host, hostPattern)) {
|
||||
return createProxyAgent(new URL(proxy));
|
||||
}
|
||||
}
|
||||
} else if (process.env['ALL_PROXY']) {
|
||||
return createProxyAgent(new URL(process.env['ALL_PROXY']));
|
||||
}
|
||||
|
||||
return new Agent({ allowH2: true });
|
||||
};
|
||||
|
||||
export const createDispatcher = (ctx: Context, url: URL): Dispatcher => {
|
||||
return createBasicDispatcher(ctx, url).compose(
|
||||
interceptors.dns(),
|
||||
interceptors.retry({ maxRetries: 3 }),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -89,6 +89,13 @@ export const buildManifest = (sources: Source[], extractors: Extractor[], config
|
|||
default: config['mediaFlowProxyPassword'] ?? '',
|
||||
});
|
||||
|
||||
manifest.config.push({
|
||||
key: 'proxyConfig',
|
||||
type: 'text',
|
||||
title: 'Proxy Config',
|
||||
default: config['proxyConfig'] ?? '',
|
||||
});
|
||||
|
||||
extractors.forEach((extractor) => {
|
||||
if (extractor.id === 'external') {
|
||||
return;
|
||||
|
|
|
|||
Loading…
Reference in a new issue