From 8acbd37df30eb61ea6c5df186b6e8752cbf7b836 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sun, 14 Sep 2025 17:38:04 +0000 Subject: [PATCH] revert: Revert recent reverts again This reverts commit 78db17995752ba286d69782b9014d61ef51f2740. --- package-lock.json | 267 ++++++++++++------ package.json | 2 +- src/extractor/ExtractorRegistry.ts | 4 +- src/source/Source.ts | 4 +- src/types.ts | 2 +- src/utils/Fetcher.ts | 23 +- src/utils/StreamResolver.ts | 9 +- src/utils/__snapshots__/manifest.test.ts.snap | 111 +++++++- src/utils/config.ts | 2 + src/utils/dispatcher.ts | 22 +- src/utils/manifest.test.ts | 16 +- src/utils/manifest.ts | 7 + 12 files changed, 350 insertions(+), 119 deletions(-) diff --git a/package-lock.json b/package-lock.json index 244f67a..a0ff0ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "express": "^5.1.0", "fetch-socks": "^1.3.2", "http-cache-semantics": "^4.2.0", - "jsdom": "^26.1.0", + "jsdom": "^27.0.0", "minimatch": "^10.0.3", "randomstring": "^1.3.1", "rot13-cipher": "^1.0.0", @@ -55,23 +55,44 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", - "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-4.0.4.tgz", + "integrity": "sha512-cKjSKvWGmAziQWbCouOsFwb14mp1betm8Y7Fn+yglDMUUu3r9DCbJ9iJbeFDenLMqFbIMC0pQP8K+B8LAxX3OQ==", "license": "MIT", "dependencies": { - "@csstools/css-calc": "^2.1.3", - "@csstools/css-color-parser": "^3.0.9", - "@csstools/css-parser-algorithms": "^3.0.4", - "@csstools/css-tokenizer": "^3.0.3", - "lru-cache": "^10.4.3" + "@csstools/css-calc": "^2.1.4", + "@csstools/css-color-parser": "^3.0.10", + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4", + "lru-cache": "^11.1.0" } }, "node_modules/@asamuzakjp/css-color/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "license": "ISC" + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.1.tgz", + "integrity": "sha512-r8LA6i4LP4EeWOhqBaZZjDWwehd1xUJPCJd9Sv300H0ZmcUER4+JPh7bqqZeqs1o5pgtgvXm+d9UGrB5zZGDiQ==", + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.5.4.tgz", + "integrity": "sha512-RNSNk1dnB8lAn+xdjlRoM4CzdVrHlmXZtSXAWs2jyl4PiBRWqTZr9ML5M710qgd9RPTBsVG6P0SLy7dwy0Foig==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.1.0", + "is-potential-custom-element-name": "^1.0.1" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "license": "MIT" }, "node_modules/@babel/code-frame": { "version": "7.27.1", @@ -721,6 +742,28 @@ "@csstools/css-tokenizer": "^3.0.4" } }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.0.14.tgz", + "integrity": "sha512-zSlIxa20WvMojjpCSy8WrNpcZ61RqfTfX3XTaOeVlGJrt/8HF3YbzgFZa01yTbT4GWQLwfTcC3EB8i3XnB647Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, "node_modules/@csstools/css-tokenizer": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", @@ -3137,6 +3180,15 @@ ], "license": "MIT" }, + "node_modules/bidi-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", + "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", + "license": "MIT", + "dependencies": { + "require-from-string": "^2.0.2" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3946,6 +3998,19 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/css-tree": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.1.0.tgz", + "integrity": "sha512-0eW44TGN5SQXU1mWSkKwFstI/22X2bG1nYzZTYMAWjylYURhse752YgbE4Cx46AC+bAvI+/dYTPRk1LqSUnu6w==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.12.2", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, "node_modules/css-what": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", @@ -3959,29 +4024,30 @@ } }, "node_modules/cssstyle": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", - "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-5.3.0.tgz", + "integrity": "sha512-RveJPnk3m7aarYQ2bJ6iw+Urh55S6FzUiqtBq+TihnTDP4cI8y/TYDqGOyqgnG1J1a6BxJXZsV9JFSTulm9Z7g==", "license": "MIT", "dependencies": { - "@asamuzakjp/css-color": "^3.2.0", - "rrweb-cssom": "^0.8.0" + "@asamuzakjp/css-color": "^4.0.3", + "@csstools/css-syntax-patches-for-csstree": "^1.0.14", + "css-tree": "^3.1.0" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/data-urls": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", - "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-6.0.0.tgz", + "integrity": "sha512-BnBS08aLUM+DKamupXs3w2tJJoqU+AkaE/+6vQxi/G/DPmIZFJJp9Dkb1kM03AZx8ADehDUZgsNxju3mPXZYIA==", "license": "MIT", "dependencies": { "whatwg-mimetype": "^4.0.0", - "whatwg-url": "^14.0.0" + "whatwg-url": "^15.0.0" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/data-view-buffer": { @@ -7333,34 +7399,34 @@ } }, "node_modules/jsdom": { - "version": "26.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", - "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-27.0.0.tgz", + "integrity": "sha512-lIHeR1qlIRrIN5VMccd8tI2Sgw6ieYXSVktcSHaNe3Z5nE/tcPQYQWOq00wxMvYOsz+73eAkNenVvmPC6bba9A==", "license": "MIT", "dependencies": { - "cssstyle": "^4.2.1", - "data-urls": "^5.0.0", + "@asamuzakjp/dom-selector": "^6.5.4", + "cssstyle": "^5.3.0", + "data-urls": "^6.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.16", - "parse5": "^7.2.1", + "parse5": "^7.3.0", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", - "tough-cookie": "^5.1.1", + "tough-cookie": "^6.0.0", "w3c-xmlserializer": "^5.0.0", - "webidl-conversions": "^7.0.0", + "webidl-conversions": "^8.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", - "whatwg-url": "^14.1.1", - "ws": "^8.18.0", + "whatwg-url": "^15.0.0", + "ws": "^8.18.2", "xml-name-validator": "^5.0.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "peerDependencies": { "canvas": "^3.0.0" @@ -7371,36 +7437,6 @@ } } }, - "node_modules/jsdom/node_modules/tldts": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", - "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", - "license": "MIT", - "dependencies": { - "tldts-core": "^6.1.86" - }, - "bin": { - "tldts": "bin/cli.js" - } - }, - "node_modules/jsdom/node_modules/tldts-core": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", - "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", - "license": "MIT" - }, - "node_modules/jsdom/node_modules/tough-cookie": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", - "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", - "license": "BSD-3-Clause", - "dependencies": { - "tldts": "^6.1.32" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -7723,6 +7759,12 @@ "node": ">= 0.4" } }, + "node_modules/mdn-data": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.12.2.tgz", + "integrity": "sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==", + "license": "CC0-1.0" + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -8090,6 +8132,25 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/napi-build-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", @@ -8398,12 +8459,6 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, - "node_modules/nwsapi": { - "version": "2.2.22", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.22.tgz", - "integrity": "sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ==", - "license": "MIT" - }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -8825,7 +8880,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -8874,6 +8928,35 @@ "node": ">= 0.4" } }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -9240,6 +9323,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", @@ -9898,6 +9990,15 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", @@ -11112,12 +11213,12 @@ } }, "node_modules/webidl-conversions": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", - "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.0.tgz", + "integrity": "sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA==", "license": "BSD-2-Clause", "engines": { - "node": ">=12" + "node": ">=20" } }, "node_modules/whatwg-encoding": { @@ -11142,16 +11243,16 @@ } }, "node_modules/whatwg-url": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", - "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-15.0.0.tgz", + "integrity": "sha512-+0q+Pc6oUhtbbeUfuZd4heMNOLDJDdagYxv756mCf9vnLF+NTj4zvv5UyYNkHJpc3CJIesMVoEIOdhi7L9RObA==", "license": "MIT", "dependencies": { - "tr46": "^5.1.0", - "webidl-conversions": "^7.0.0" + "tr46": "^5.1.1", + "webidl-conversions": "^8.0.0" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/which": { diff --git a/package.json b/package.json index 61909d4..929f061 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "express": "^5.1.0", "fetch-socks": "^1.3.2", "http-cache-semantics": "^4.2.0", - "jsdom": "^26.1.0", + "jsdom": "^27.0.0", "minimatch": "^10.0.3", "randomstring": "^1.3.1", "rot13-cipher": "^1.0.0", diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index b65bc83..96a45ad 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -3,7 +3,7 @@ import KeyvSqlite from '@keyv/sqlite'; import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import winston from 'winston'; import { Context, Meta, UrlResult } from '../types'; -import { getCacheDir, isExtractorDisabled } from '../utils'; +import { getCacheDir, isExtractorDisabled, noCache } from '../utils'; import { Extractor } from './Extractor'; export class ExtractorRegistry { @@ -41,7 +41,7 @@ export class ExtractorRegistry { const storedDataRaw = await this.urlResultCache.getRaw(cacheKey); const expires = storedDataRaw?.expires; - if (storedDataRaw && expires) { + if (storedDataRaw && expires && !noCache(ctx.config)) { // Ignore the cache randomly after at least 2/3 of the TTL passed to start refreshing results slowly const refreshTimestamp = this.randomInteger(expires - extractor.ttl * (2 / 3), expires); const now = Date.now(); diff --git a/src/source/Source.ts b/src/source/Source.ts index 3f5feec..2ef1bd6 100644 --- a/src/source/Source.ts +++ b/src/source/Source.ts @@ -4,7 +4,7 @@ import { Cacheable, CacheableMemory, Keyv } from 'cacheable'; import { ContentType } from 'stremio-addon-sdk'; import { NotFoundError } from '../error'; import { Context, CountryCode, Meta } from '../types'; -import { getCacheDir, Id } from '../utils'; +import { getCacheDir, Id, noCache } from '../utils'; export interface SourceResult { url: URL; @@ -44,7 +44,7 @@ export abstract class Source { let sourceResults = (await Source.sourceResultCache.get(cacheKey)) ?.map(sourceResult => ({ ...sourceResult, url: new URL(sourceResult.url) })); - if (!sourceResults) { + if (!sourceResults || noCache(ctx.config)) { try { sourceResults = await this.handleInternal(ctx, type, id); } catch (error) { diff --git a/src/types.ts b/src/types.ts index 0857a68..c04729e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,7 +17,7 @@ export type CustomManifest = Manifest & { }; }; -export type Config = Partial & Record<`disableExtractor_${string}`, string>>; +export type Config = Partial & Record<`disableExtractor_${string}`, string>>; export enum CountryCode { multi = 'multi', diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 4a96035..77be3a1 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -7,7 +7,8 @@ import { fetch, Headers, RequestInit, Response } from 'undici'; import winston from 'winston'; import { BlockedError, HttpError, NotFoundError, QueueIsFullError, TimeoutError, TooManyRequestsError, TooManyTimeoutsError } from '../error'; import { BlockedReason, Context } from '../types'; -import { createDispatcher } from './dispatcher'; +import { noCache } from './config'; +import { createProxyAgent, getProxyForUrl } from './dispatcher'; import { envGet } from './env'; interface HttpCacheItem { @@ -233,9 +234,9 @@ export class Fetcher { const cacheKey = this.determineCacheKey(url, init); let httpCacheItem = await this.cacheGet(cacheKey); - const noCache = init?.noCache ?? false; - if (httpCacheItem && !noCache) { - this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); + const disableCache = init?.noCache ?? noCache(ctx.config); + if (httpCacheItem && !disableCache) { + this.logger.info(`Cached fetch ${request.method} ${url}: ${httpCacheItem.status} (${httpCacheItem.statusText})`, ctx); return this.handleHttpCacheItem(ctx, httpCacheItem, url, init); } @@ -253,14 +254,16 @@ export class Fetcher { }; protected async fetchWithTimeout(ctx: Context, url: URL, init?: CustomRequestInit, tryCount = 0): Promise { - this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx); + const proxyUrl = getProxyForUrl(ctx, url); + + this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url} via proxy ${proxyUrl}`, ctx); const isRateLimitedRaw = await this.rateLimitedCache.getRaw(url.host); /* istanbul ignore if */ if (isRateLimitedRaw && isRateLimitedRaw.value && isRateLimitedRaw.expires) { const ttl = isRateLimitedRaw.expires - Date.now(); if (ttl <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) { - this.logger.info('Wait out rate limit', ctx); + this.logger.info(`Wait out rate limit for ${url}`, ctx); await this.sleep(ttl); @@ -281,17 +284,17 @@ export class Fetcher { finalUrl.username = ''; finalUrl.password = ''; - const dispatcher = createDispatcher(ctx, url); - const finalInit = { ...init, keepalive: true, signal: AbortSignal.timeout(init?.timeout ?? this.DEFAULT_TIMEOUT), - ...(/* istanbul ignore next */ dispatcher && { dispatcher }), + ...(/* istanbul ignore next */ proxyUrl && { dispatcher: createProxyAgent(proxyUrl) }), }; response = await fetch(finalUrl, finalInit); } catch (error) { + this.logger.info(`Got error ${error} for ${url}`, ctx); + if (error instanceof DOMException && ['AbortError', 'TimeoutError'].includes(error.name)) { await this.increaseTimeoutsCount(url); throw new TimeoutError(); @@ -300,6 +303,8 @@ export class Fetcher { throw error; } + this.logger.info(`Got ${response.status} (${response.statusText}) for ${url}`, ctx); + await this.decreaseTimeoutsCount(url); if (response.status === 429) { diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index b19fba3..5b78976 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -39,7 +39,7 @@ export class StreamResolver { const streams: Stream[] = []; - let sourceErrorOccurred = false; + let sourceErrorCount = 0; const urlResults: UrlResult[] = []; const sourcePromises = sources.map(async (source) => { if (!source.contentTypes.includes(type)) { @@ -55,7 +55,7 @@ export class StreamResolver { urlResults.push(...sourceUrlResults.flat()); } catch (error) { - sourceErrorOccurred = true; + sourceErrorCount++; if (showErrors(ctx.config)) { streams.push({ @@ -86,7 +86,8 @@ export class StreamResolver { return a.label.localeCompare(b.label); }); - this.logger.info(`Return ${urlResults.length} streams`, ctx); + const errorCount = urlResults.reduce((count, urlResult) => urlResult.error ? count + 1 : count, sourceErrorCount); + this.logger.info(`Got ${urlResults.length} url results, including ${errorCount} errors`, ctx); streams.push( ...urlResults.filter(urlResult => !urlResult.error || showErrors(ctx.config)) @@ -106,7 +107,7 @@ export class StreamResolver { })), ); - const ttl = !sourceErrorOccurred ? this.determineTtl(urlResults) : undefined; + const ttl = sourceErrorCount === 0 ? this.determineTtl(urlResults) : undefined; return { streams, diff --git a/src/utils/__snapshots__/manifest.test.ts.snap b/src/utils/__snapshots__/manifest.test.ts.snap index 452b4fb..e1a58a7 100644 --- a/src/utils/__snapshots__/manifest.test.ts.snap +++ b/src/utils/__snapshots__/manifest.test.ts.snap @@ -72,6 +72,11 @@ exports[`buildManifest default manifest 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ], "description": "Provides HTTP URLs from streaming websites. Configure add-on for additional languages. Add MediaFlow proxy for more URLs. @@ -131,6 +136,11 @@ exports[`buildManifest disable extractors 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, { "default": "checked", "key": "disableExtractor_doodstream", @@ -145,10 +155,9 @@ exports[`buildManifest disable extractors 1`] = ` ] `; -exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` +exports[`buildManifest has checked includeExternalUrls 1`] = ` [ { - "default": "checked", "key": "showErrors", "title": "Show errors", "type": "checkbox", @@ -177,6 +186,89 @@ exports[`buildManifest has checked showErrors and includeExternalUrls 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked noCache 1`] = ` +[ + { + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "default": "checked", + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, +] +`; + +exports[`buildManifest has checked showErrors 1`] = ` +[ + { + "default": "checked", + "key": "showErrors", + "title": "Show errors", + "type": "checkbox", + }, + { + "key": "includeExternalUrls", + "title": "Include external URLs in results", + "type": "checkbox", + }, + { + "default": "", + "key": "mediaFlowProxyUrl", + "title": "MediaFlow Proxy URL", + "type": "text", + }, + { + "default": "", + "key": "mediaFlowProxyPassword", + "title": "MediaFlow Proxy Password", + "type": "password", + }, + { + "default": "", + "key": "proxyConfig", + "title": "Proxy Config", + "type": "text", + }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -227,6 +319,11 @@ exports[`buildManifest has checked source with appropriate config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -285,6 +382,11 @@ exports[`buildManifest has unchecked source without a config 1`] = ` "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; @@ -318,5 +420,10 @@ exports[`buildManifest showErrors and includeExternalUrls are unchecked by defau "title": "Proxy Config", "type": "text", }, + { + "key": "noCache", + "title": "Disable cache", + "type": "checkbox", + }, ] `; diff --git a/src/utils/config.ts b/src/utils/config.ts index fd503db..2702554 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,6 +9,8 @@ export const showErrors = (config: Config): boolean => 'showErrors' in config; export const showExternalUrls = (config: Config): boolean => 'includeExternalUrls' in config; +export const noCache = (config: Config): boolean => 'noCache' in config; + export const hasMultiEnabled = (config: Config): boolean => 'multi' in config; export const disableExtractorConfigKey = (extractor: Extractor): string => `disableExtractor_${extractor.id}`; diff --git a/src/utils/dispatcher.ts b/src/utils/dispatcher.ts index e15b340..8e05c77 100644 --- a/src/utils/dispatcher.ts +++ b/src/utils/dispatcher.ts @@ -3,15 +3,7 @@ import { minimatch } from 'minimatch'; import { Dispatcher, ProxyAgent } from 'undici'; import { Context } from '../types'; -const createProxyAgent = (proxyUrl: URL): Dispatcher => { - if (proxyUrl.protocol === 'socks5:') { - return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); - } - - return new ProxyAgent({ uri: proxyUrl.href }); -}; - -const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { +export const getProxyForUrl = (ctx: Context, url: URL): URL | undefined => { const proxyConfig = ctx.config['proxyConfig'] || process.env['PROXY_CONFIG']; if (proxyConfig) { @@ -22,16 +14,20 @@ const createBasicDispatcher = (ctx: Context, url: URL): Dispatcher | undefined = } if (hostPattern === '*' || minimatch(url.host, hostPattern)) { - return createProxyAgent(new URL(proxy)); + return proxy === 'false' ? undefined : new URL(proxy); } } } else if (process.env['ALL_PROXY']) { - return createProxyAgent(new URL(process.env['ALL_PROXY'])); + return new URL(process.env['ALL_PROXY']); } return undefined; }; -export const createDispatcher = (ctx: Context, url: URL): Dispatcher | undefined => { - return createBasicDispatcher(ctx, url); +export const createProxyAgent = (proxyUrl: URL): Dispatcher => { + if (proxyUrl.protocol === 'socks5:') { + return socksDispatcher({ type: 5, host: proxyUrl.hostname, port: parseInt(proxyUrl.port) }); + } + + return new ProxyAgent({ uri: proxyUrl.href }); }; diff --git a/src/utils/manifest.test.ts b/src/utils/manifest.test.ts index bbe709d..9e71309 100644 --- a/src/utils/manifest.test.ts +++ b/src/utils/manifest.test.ts @@ -50,8 +50,20 @@ describe('buildManifest', () => { expect(manifest.config).toMatchSnapshot(); }); - test('has checked showErrors and includeExternalUrls', () => { - const manifest = buildManifest([], [], { showErrors: 'on', includeExternalUrls: 'on' }); + test('has checked showErrors', () => { + const manifest = buildManifest([], [], { showErrors: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked includeExternalUrls', () => { + const manifest = buildManifest([], [], { includeExternalUrls: 'on' }); + + expect(manifest.config).toMatchSnapshot(); + }); + + test('has checked noCache', () => { + const manifest = buildManifest([], [], { noCache: 'on' }); expect(manifest.config).toMatchSnapshot(); }); diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts index 57d3baf..9ddc705 100644 --- a/src/utils/manifest.ts +++ b/src/utils/manifest.ts @@ -96,6 +96,13 @@ export const buildManifest = (sources: Source[], extractors: Extractor[], config default: config['proxyConfig'] ?? '', }); + manifest.config.push({ + key: 'noCache', + type: 'checkbox', + title: 'Disable cache', + ...('noCache' in config && { default: 'checked' }), + }); + extractors.forEach((extractor) => { if (extractor.id === 'external') { return;