From fbc1aa994c091a39a97c48d417e32afbe63f78d9 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sat, 31 May 2025 21:13:52 +0000 Subject: [PATCH] fix: switch back to fetch and do not generate user agents seems to be causing less cloudflare challenges --- package-lock.json | 241 ++++++------------ package.json | 8 +- src/index.ts | 10 +- src/utils/Fetcher.test.ts | 104 ++++---- src/utils/Fetcher.ts | 119 +++++---- ...8.htmlhttps:ds2play.comefzfvfq3ngig0.error | 2 +- .../head-https:ds2play.comefzfvfq3ngig0.error | 2 +- .../Fetcher/head-https:some-url.test.error | 2 +- ...-https:streamtape.comeBjp2vjrdBxsK82.error | 2 +- ...-https:streamtape.comegjA1OQ4klyHxgJ.error | 2 +- ...iesid61945andsa1andepi1andidTypetmdb.error | 1 - src/utils/__mocks__/Fetcher.ts | 61 ++--- src/utils/tmdb.ts | 2 +- 13 files changed, 240 insertions(+), 316 deletions(-) delete mode 100644 src/utils/__fixtures__/Fetcher/https:frembed.clubapiseriesid61945andsa1andepi1andidTypetmdb.error diff --git a/package-lock.json b/package-lock.json index 03d0db0..77efca2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,14 @@ "license": "MIT", "dependencies": { "@isaacs/ttlcache": "^1.4.1", - "axios-cache-interceptor": "^1.8.0", - "axios-retry": "^4.5.0", "bytes": "^3.1.2", "cheerio": "^1.0.0", "country-emoji": "^1.5.6", "express": "^5.1.0", + "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", "unpacker": "^1.0.1", - "user-agents": "^1.1.536", "uuid": "^11.1.0", "winston": "^3.17.0" }, @@ -28,15 +26,15 @@ "@stylistic/eslint-plugin": "^4.2.0", "@types/bytes": "^3.1.5", "@types/express": "^5.0.1", + "@types/http-cache-semantics": "^4.0.4", "@types/jest": "^29.5.14", "@types/make-fetch-happen": "^10.0.4", "@types/node": "^22.15.3", "@types/randomstring": "^1.3.0", "@types/stremio-addon-sdk": "^1.6.11", - "@types/user-agents": "^1.0.4", - "axios-mock-adapter": "^2.1.0", "babel-jest": "^29.7.0", "eslint": "^9.26.0", + "fetch-mock": "^12.5.2", "jest": "^29.7.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", @@ -1509,6 +1507,13 @@ "@types/send": "*" } }, + "node_modules/@types/glob-to-regexp": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz", + "integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", @@ -1519,6 +1524,13 @@ "@types/node": "*" } }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/http-errors": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", @@ -1692,13 +1704,6 @@ "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "license": "MIT" }, - "node_modules/@types/user-agents": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@types/user-agents/-/user-agents-1.0.4.tgz", - "integrity": "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -2119,65 +2124,8 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz", - "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", - "license": "MIT", - "peer": true, - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/axios-cache-interceptor": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/axios-cache-interceptor/-/axios-cache-interceptor-1.8.0.tgz", - "integrity": "sha512-cTNnPGJyQkxnWp0EWvE3NRvgURU5cWw/Qx3dIhXyHSM4Ip0c7EEe0I3an0Jwa549m1CAOg57ibj27YRNLmQCcg==", - "license": "MIT", - "dependencies": { - "cache-parser": "1.2.5", - "fast-defer": "1.1.8", - "object-code": "1.3.3" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/arthurfiorette/axios-cache-interceptor?sponsor=1" - }, - "peerDependencies": { - "axios": "^1" - } - }, - "node_modules/axios-mock-adapter": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.1.0.tgz", - "integrity": "sha512-AZUe4OjECGCNNssH8SOdtneiQELsqTsat3SQQCWLPjN436/H+L9AjWfV7bF+Zg/YL9cgbhrz5671hoh+Tbn98w==", "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "is-buffer": "^2.0.5" - }, - "peerDependencies": { - "axios": ">= 0.17.0" - } - }, - "node_modules/axios-retry": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.5.0.tgz", - "integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==", - "license": "Apache-2.0", - "dependencies": { - "is-retry-allowed": "^2.2.0" - }, - "peerDependencies": { - "axios": "0.x || 1.x" - } + "license": "MIT" }, "node_modules/babel-jest": { "version": "29.7.0", @@ -2419,12 +2367,6 @@ "node": ">= 0.8" } }, - "node_modules/cache-parser": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/cache-parser/-/cache-parser-1.2.5.tgz", - "integrity": "sha512-Md/4VhAHByQ9frQ15WD6LrMNiVw9AEl/J7vWIXw+sxT6fSOpbtt6LHTp76vy8+bOESPBO94117Hm2bIjlI7XjA==", - "license": "MIT" - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -2726,6 +2668,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -2918,6 +2861,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -2932,6 +2876,16 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -3164,6 +3118,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -3554,12 +3509,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-defer": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/fast-defer/-/fast-defer-1.1.8.tgz", - "integrity": "sha512-lEJeOH5VL5R09j6AA0D4Uvq7AgsHw0dAImQQ+F3iSyHZuAxyQfWobsagGpTcOPvJr3urmKRHrs+Gs9hV+/Qm/Q==", - "license": "MIT" - }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -3630,6 +3579,22 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "license": "MIT" }, + "node_modules/fetch-mock": { + "version": "12.5.2", + "resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.2.tgz", + "integrity": "sha512-b5KGDFmdmado2MPQjZl6ix3dAG3iwCitb0XQwN72y2s9VnWZ3ObaGNy+bkpm1390foiLDybdJ7yjRGKD36kATw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/glob-to-regexp": "^0.4.4", + "dequal": "^2.0.3", + "glob-to-regexp": "^0.4.1", + "regexparam": "^3.0.0" + }, + "engines": { + "node": ">=18.11.0" + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -3737,31 +3702,11 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", "license": "MIT" }, - "node_modules/follow-redirects": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", - "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "peer": true, - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/form-data": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -3777,6 +3722,7 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -3786,6 +3732,7 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -3958,6 +3905,13 @@ "node": ">=10.13.0" } }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/glob/node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4044,6 +3998,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -4093,6 +4048,12 @@ "entities": "^4.5.0" } }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -4252,30 +4213,6 @@ "node": ">=8" } }, - "node_modules/is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", @@ -4351,18 +4288,6 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, - "node_modules/is-retry-allowed": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", - "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -5250,12 +5175,6 @@ "node": ">=8" } }, - "node_modules/lodash.clonedeep": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", - "license": "MIT" - }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -5624,12 +5543,6 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, - "node_modules/object-code": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/object-code/-/object-code-1.3.3.tgz", - "integrity": "sha512-/Ds4Xd5xzrtUOJ+xJQ57iAy0BZsZltOHssnDgcZ8DOhgh41q1YJCnTPnWdWSLkNGNnxYzhYChjc5dgC9mEERCA==", - "license": "MIT" - }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -6005,13 +5918,6 @@ "node": ">= 0.10" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT", - "peer": true - }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", @@ -6177,6 +6083,16 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/regexparam": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz", + "integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -7120,15 +7036,6 @@ "punycode": "^2.1.0" } }, - "node_modules/user-agents": { - "version": "1.1.550", - "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.550.tgz", - "integrity": "sha512-lt8m9L4qfBityhTJjr36G2ZRlVtDpQyM8H7s5l2avLWkCeSLdJm0yzCyn1ZEP/fu2GFt10LJGu0hdLKEYT6Dew==", - "license": "BSD-2-Clause", - "dependencies": { - "lodash.clonedeep": "^4.5.0" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 92be385..6f55d74 100644 --- a/package.json +++ b/package.json @@ -32,16 +32,14 @@ }, "dependencies": { "@isaacs/ttlcache": "^1.4.1", - "axios-cache-interceptor": "^1.8.0", - "axios-retry": "^4.5.0", "bytes": "^3.1.2", "cheerio": "^1.0.0", "country-emoji": "^1.5.6", "express": "^5.1.0", + "http-cache-semantics": "^4.2.0", "randomstring": "^1.3.1", "slugify": "^1.6.6", "unpacker": "^1.0.1", - "user-agents": "^1.1.536", "uuid": "^11.1.0", "winston": "^3.17.0" }, @@ -50,15 +48,15 @@ "@stylistic/eslint-plugin": "^4.2.0", "@types/bytes": "^3.1.5", "@types/express": "^5.0.1", + "@types/http-cache-semantics": "^4.0.4", "@types/jest": "^29.5.14", "@types/make-fetch-happen": "^10.0.4", "@types/node": "^22.15.3", "@types/randomstring": "^1.3.0", "@types/stremio-addon-sdk": "^1.6.11", - "@types/user-agents": "^1.0.4", - "axios-mock-adapter": "^2.1.0", "babel-jest": "^29.7.0", "eslint": "^9.26.0", + "fetch-mock": "^12.5.2", "jest": "^29.7.0", "nodemon": "^3.1.10", "ts-jest": "^29.3.2", diff --git a/src/index.ts b/src/index.ts index 1de3510..8b0d5fb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,5 @@ import express, { NextFunction, Request, Response } from 'express'; import { v4 as uuidv4 } from 'uuid'; -import axios from 'axios'; -import axiosRetry from 'axios-retry'; -import { setupCache } from 'axios-cache-interceptor'; import winston from 'winston'; import { CineHDPlus, @@ -30,12 +27,7 @@ const logger = winston.createLogger({ ], }); -setupCache(axios); -axiosRetry(axios, { - retries: 1, -}); - -const fetcher = new Fetcher(axios, logger); +const fetcher = new Fetcher(logger); const extractorRegistry = new ExtractorRegistry(logger, fetcher); diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 93fef98..89f9f43 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -1,20 +1,21 @@ import winston from 'winston'; -import axios, { AxiosError } from 'axios'; -import AxiosMockAdapter from 'axios-mock-adapter'; +import fetchMock from 'fetch-mock'; import { Fetcher } from './Fetcher'; import { Context } from '../types'; import { CloudflareChallengeError, NotFoundError } from '../error'; +fetchMock.mockGlobal(); -const axiosMock = new AxiosMockAdapter(axios); - -const fetcher = new Fetcher(axios, winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); +const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); describe('fetch', () => { const ctx: Context = { id: 'id', ip: '127.0.0.1', config: { de: 'on' } }; + afterEach(() => { + fetchMock.clearHistory(); + }); + test('text passes successful response through setting headers', async () => { - axiosMock.onGet().reply(200, 'some text'); - const axiosSpy = jest.spyOn(axios, 'get'); + fetchMock.get('https://some-url.test/', 'some text'); const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/')); const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } }); @@ -22,71 +23,80 @@ describe('fetch', () => { expect(responseText1).toBe('some text'); expect(responseText2).toStrictEqual(responseText1); - expect(axiosSpy).toHaveBeenCalledWith( - 'https://some-url.test/', - { - headers: { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en', - 'Forwarded': 'for=127.0.0.1', - 'Origin': 'https://some-url.test', - 'Priority': 'u=0', - 'Referer': 'https://some-url.test', - 'User-Agent': expect.not.stringMatching(/jest/), - 'X-Forwarded-For': '127.0.0.1', - 'X-Forwarded-Proto': 'https', - 'X-Real-IP': '127.0.0.1', - }, - responseType: 'text', - timeout: 5000, + expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({ + headers: { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + 'Forwarded': 'for=127.0.0.1', + 'Origin': 'https://some-url.test', + 'Priority': 'u=0', + 'Referer': 'https://some-url.test', + 'X-Forwarded-For': '127.0.0.1', + 'X-Forwarded-Proto': 'https', + 'X-Real-IP': '127.0.0.1', }, - ); + }); }); test('textPost ', async () => { - axiosMock.onPost().reply(200, 'some text'); + fetchMock.post('https://some-post-url.test/', 'some text'); - expect(await fetcher.textPost(ctx, new URL('https://some-url.test/'), { foo: 'bar' })).toBe('some text'); + expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), { foo: 'bar' })).toBe('some text'); }); test('head', async () => { - axiosMock.onHead().reply(200, undefined, { 'X-Fake-Response': 'foo' }); + fetchMock.head('https://some-head-url.test/', { status: 200, headers: { 'X-Fake-Response': 'foo' } }); - expect(await fetcher.head(ctx, new URL('https://some-url.test/'))).toMatchObject({ 'X-Fake-Response': 'foo' }); + expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' }); }); test('uses context referer', async () => { - axiosMock.onGet().reply(200, 'some text'); - const axiosSpy = jest.spyOn(axios, 'get'); + fetchMock.get('https://some-referer-url.test/', 'some text'); - await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-url.test/')); + await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-referer-url.test/')); - expect(axiosSpy).toHaveBeenCalledWith( - 'https://some-url.test/', - expect.objectContaining({ - headers: expect.objectContaining({ - Origin: 'https://example.com', - Referer: 'https://example.com/foo/bar', - }), - }), - ); + expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({ + headers: { + Origin: 'https://example.com', + Referer: 'https://example.com/foo/bar', + }, + }); }); test('converts 404 to custom NotFoundError', async () => { - axiosMock.onGet().reply(404); + fetchMock.get('https://some-404-url.test/', 404); - await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(NotFoundError); + await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError); }); test('converts Cloudflare challenge block to custom CloudflareChallengeError', async () => { - axiosMock.onGet().reply(403, undefined, { 'cf-mitigated': 'challenge' }); + fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); - await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError); + await expect(fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError); }); test('passes through other errors', async () => { - axiosMock.onGet().networkError(); + fetchMock.get('https://some-error-url.test/', 500); - await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(AxiosError); + await expect(fetcher.text(ctx, new URL('https://some-error-url.test/'))).rejects.toBeInstanceOf(Error); + }); + + test('passes through exceptions', async () => { + fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') }); + + await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError); + }); + + test('times out after 10 seconds', async () => { + jest.useFakeTimers(); + fetchMock.get('https://some-timeout-url.test/', 200, { delay: 15000 }); + + const promise = fetcher.text(ctx, new URL('https://some-timeout-url.test/')); + + jest.advanceTimersByTime(13000); + + await expect(promise).rejects.toBeInstanceOf(DOMException); + + jest.useRealTimers(); }); }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index c70ab15..b1a964e 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,47 +1,39 @@ -import { AxiosInstance, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders, isAxiosError } from 'axios'; +import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; -import UserAgent from 'user-agents'; import winston from 'winston'; import { Context } from '../types'; import { CloudflareChallengeError, NotFoundError } from '../error'; +import { clearTimeout } from 'node:timers'; + +interface HttpCacheItem { policy: CachePolicy; status: number; statusText: string; body: string } export class Fetcher { - private readonly axios: AxiosInstance; private readonly logger: winston.Logger; - private readonly ipUserAgentCache: TTLCache; + private readonly httpCache: TTLCache; - constructor(axios: AxiosInstance, logger: winston.Logger) { - this.axios = axios; + constructor(logger: winston.Logger) { this.logger = logger; - this.ipUserAgentCache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h + this.httpCache = new TTLCache(); } - readonly text = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { - this.logger.info(`Fetch ${url}`, ctx); - - return (await this.errorWrapper(() => this.axios.get(url.href, this.getConfig(ctx, url, config)))).data; + readonly text = async (ctx: Context, url: URL, init?: RequestInit): Promise => { + return (await this.cachedFetch(ctx, url, init)).body; }; - readonly textPost = async (ctx: Context, url: URL, data: unknown, config?: AxiosRequestConfig): Promise => { - this.logger.info(`Fetch post ${url}`, ctx); - - return (await this.errorWrapper(() => this.axios.post(url.href, data, this.getConfig(ctx, url, config)))).data; + readonly textPost = async (ctx: Context, url: URL, data: unknown, init?: RequestInit): Promise => { + return (await this.cachedFetch(ctx, url, { ...init, method: 'POST', body: JSON.stringify(data) })).body; }; - readonly head = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { - this.logger.info(`Fetch head ${url}`, ctx); - - return (await this.errorWrapper(() => this.axios.head(url.href, this.getConfig(ctx, url, config)))).headers; + readonly head = async (ctx: Context, url: URL, init?: RequestInit): Promise => { + return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders(); }; - private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => { + private readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => { const origin = ctx.referer?.origin ?? url.origin; const referer = ctx.referer?.href ?? url.origin; return { - responseType: 'text', - timeout: 5000, - ...config, + ...init, headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en', @@ -49,43 +41,80 @@ export class Fetcher { 'Origin': `${origin}`, 'Priority': 'u=0', 'Referer': `${referer}`, - 'User-Agent': this.createUserAgentForIp(ctx.ip), 'X-Forwarded-For': ctx.ip, 'X-Forwarded-Proto': url.protocol.slice(0, -1), 'X-Real-IP': ctx.ip, - ...config?.headers, + ...init?.headers, }, }; }; - private readonly createUserAgentForIp = (ip: string): string => { - let userAgent = this.ipUserAgentCache.get(ip); - if (userAgent) { - return userAgent; + private readonly handleHttpCacheItem = (httpCacheItem: HttpCacheItem): HttpCacheItem => { + if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) { + return httpCacheItem; } - userAgent = (new UserAgent()).toString(); + if (httpCacheItem.status === 404) { + throw new NotFoundError(); + } - this.ipUserAgentCache.set(ip, userAgent); + if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') { + throw new CloudflareChallengeError(); + } - return userAgent; + throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}`); }; - private readonly errorWrapper = async (callable: () => T): Promise => { - try { - return await callable(); - } catch (error) { - if (isAxiosError(error)) { - if (error.response?.status === 404) { - throw new NotFoundError(); - } + private readonly headersToObject = (headers: Headers): Record => { + const obj: Record = {}; - if (error.response?.headers['cf-mitigated'] === 'challenge') { - throw new CloudflareChallengeError(); - } - } + headers.forEach((value, name) => { + obj[name] = value; + }); - throw error; + return obj; + }; + + private readonly cachedFetch = async (ctx: Context, url: URL, init?: RequestInit): Promise => { + const newInit = this.getInit(ctx, url, init); + + const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} }; + + let httpCacheItem: HttpCacheItem | undefined = this.httpCache.get(url.href); + if (httpCacheItem) { + this.logger.info(`Cached fetch ${request.method} ${url}`, ctx); + return this.handleHttpCacheItem(httpCacheItem); } + + this.logger.info(`Fetch ${request.method} ${url}`, ctx); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 10000); + + let response; + try { + response = await fetch(url, { ...newInit, signal: controller.signal }); + } finally { + clearTimeout(timer); + } + + const body = await response.text(); + + const policy = new CachePolicy(request, { status: response.status, headers: this.headersToObject(response.headers) }, { shared: false }); + + httpCacheItem = { policy, status: response.status, statusText: response.statusText, body }; + + let ttl; + if (response.ok) { + ttl = Math.max(policy.timeToLive(), 900000); // 15m at least + } else { + ttl = policy.timeToLive(); + } + + if (ttl > 0) { + this.httpCache.set(url.href, httpCacheItem, { ttl }); + } + + return this.handleHttpCacheItem(httpCacheItem); }; } diff --git a/src/utils/__fixtures__/Fetcher/head-https:ahvsh.come83izf7qzwpaehttps:netu.frembed.fune0DFgfkcXOsDPhttps:likessb.come7yjgl1x56n08.htmlhttps:ds2play.comefzfvfq3ngig0.error b/src/utils/__fixtures__/Fetcher/head-https:ahvsh.come83izf7qzwpaehttps:netu.frembed.fune0DFgfkcXOsDPhttps:likessb.come7yjgl1x56n08.htmlhttps:ds2play.comefzfvfq3ngig0.error index f13a70b..6ee2b83 100644 --- a/src/utils/__fixtures__/Fetcher/head-https:ahvsh.come83izf7qzwpaehttps:netu.frembed.fune0DFgfkcXOsDPhttps:likessb.come7yjgl1x56n08.htmlhttps:ds2play.comefzfvfq3ngig0.error +++ b/src/utils/__fixtures__/Fetcher/head-https:ahvsh.come83izf7qzwpaehttps:netu.frembed.fune0DFgfkcXOsDPhttps:likessb.come7yjgl1x56n08.htmlhttps:ds2play.comefzfvfq3ngig0.error @@ -1 +1 @@ -{"message":"socket hang up","code":"ECONNRESET"} \ No newline at end of file +TypeError: fetch failed \ No newline at end of file diff --git a/src/utils/__fixtures__/Fetcher/head-https:ds2play.comefzfvfq3ngig0.error b/src/utils/__fixtures__/Fetcher/head-https:ds2play.comefzfvfq3ngig0.error index df87c8f..6ee2b83 100644 --- a/src/utils/__fixtures__/Fetcher/head-https:ds2play.comefzfvfq3ngig0.error +++ b/src/utils/__fixtures__/Fetcher/head-https:ds2play.comefzfvfq3ngig0.error @@ -1 +1 @@ -{"message":"unable to verify the first certificate","code":"UNABLE_TO_VERIFY_LEAF_SIGNATURE"} \ No newline at end of file +TypeError: fetch failed \ No newline at end of file diff --git a/src/utils/__fixtures__/Fetcher/head-https:some-url.test.error b/src/utils/__fixtures__/Fetcher/head-https:some-url.test.error index 70557bb..6ee2b83 100644 --- a/src/utils/__fixtures__/Fetcher/head-https:some-url.test.error +++ b/src/utils/__fixtures__/Fetcher/head-https:some-url.test.error @@ -1 +1 @@ -{"message":"getaddrinfo ENOTFOUND some-url.test","code":"ENOTFOUND"} \ No newline at end of file +TypeError: fetch failed \ No newline at end of file diff --git a/src/utils/__fixtures__/Fetcher/head-https:streamtape.comeBjp2vjrdBxsK82.error b/src/utils/__fixtures__/Fetcher/head-https:streamtape.comeBjp2vjrdBxsK82.error index a7841b2..c31051e 100644 --- a/src/utils/__fixtures__/Fetcher/head-https:streamtape.comeBjp2vjrdBxsK82.error +++ b/src/utils/__fixtures__/Fetcher/head-https:streamtape.comeBjp2vjrdBxsK82.error @@ -1 +1 @@ -{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}} \ No newline at end of file +Fetcher error: 404: Not Found \ No newline at end of file diff --git a/src/utils/__fixtures__/Fetcher/head-https:streamtape.comegjA1OQ4klyHxgJ.error b/src/utils/__fixtures__/Fetcher/head-https:streamtape.comegjA1OQ4klyHxgJ.error index a7841b2..c31051e 100644 --- a/src/utils/__fixtures__/Fetcher/head-https:streamtape.comegjA1OQ4klyHxgJ.error +++ b/src/utils/__fixtures__/Fetcher/head-https:streamtape.comegjA1OQ4klyHxgJ.error @@ -1 +1 @@ -{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}} \ No newline at end of file +Fetcher error: 404: Not Found \ No newline at end of file diff --git a/src/utils/__fixtures__/Fetcher/https:frembed.clubapiseriesid61945andsa1andepi1andidTypetmdb.error b/src/utils/__fixtures__/Fetcher/https:frembed.clubapiseriesid61945andsa1andepi1andidTypetmdb.error deleted file mode 100644 index d73315c..0000000 --- a/src/utils/__fixtures__/Fetcher/https:frembed.clubapiseriesid61945andsa1andepi1andidTypetmdb.error +++ /dev/null @@ -1 +0,0 @@ -{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":"{\"error\":\"Series not found\"}"}} \ No newline at end of file diff --git a/src/utils/__mocks__/Fetcher.ts b/src/utils/__mocks__/Fetcher.ts index dd9203a..a181404 100644 --- a/src/utils/__mocks__/Fetcher.ts +++ b/src/utils/__mocks__/Fetcher.ts @@ -1,74 +1,63 @@ import fs from 'node:fs'; -import Axios, { AxiosError, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios'; import slugify from 'slugify'; import { Context } from '../../types'; export class Fetcher { - readonly text = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { + readonly text = async (_ctx: Context, url: URL, init?: RequestInit): Promise => { const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(url.href)}`; - return this.fixtureWrapper(path, async () => (await Axios.create().get(url.href, this.getConfig(config))).data); + return this.fetchy(path, url, init); }; - readonly textPost = async (_ctx: Context, url: URL, data: unknown, config?: AxiosRequestConfig): Promise => { + readonly textPost = async (_ctx: Context, url: URL, data: unknown, init?: RequestInit): Promise => { const path = `${__dirname}/../__fixtures__/Fetcher/post-${slugify(url.href)}-${slugify(JSON.stringify(data))}`; - return this.fixtureWrapper(path, async () => (await Axios.create().post(url.href, data, this.getConfig(config))).data); + return this.fetchy(path, url, { ...init, method: 'POST', body: JSON.stringify(data) }); }; - readonly head = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { + readonly head = async (_ctx: Context, url: URL, init?: RequestInit): Promise => { const path = `${__dirname}/../__fixtures__/Fetcher/head-${slugify(url.href)}`; - return JSON.parse(await this.fixtureWrapper(path, async () => JSON.stringify((await Axios.create().head(url.href, this.getConfig(config))).headers))); + return JSON.parse(await this.fetchy(path, url, { ...init, method: 'HEAD' })); }; - private readonly fixtureWrapper = async (path: string, callable: () => Promise): Promise => { + private readonly fetchy = async (path: string, url: URL, init?: RequestInit): Promise => { const errorPath = `${path}.error`; if (fs.existsSync(errorPath)) { - const fixtureError = JSON.parse(fs.readFileSync(errorPath).toString()); - throw new AxiosError(fixtureError.message, fixtureError.code, undefined, undefined, fixtureError.response); + throw new Error(fs.readFileSync(errorPath).toString()); } else if (fs.existsSync(path)) { return fs.readFileSync(path).toString(); } else { let response; - try { if (process.env['TEST_UPDATE_FIXTURES']) { - response = await callable(); + response = await fetch(url, init); } else { console.error(`No fixture found at "${path}".`); process.exit(1); } } catch (error) { - if (error instanceof AxiosError) { - const fixtureError = JSON.stringify({ - message: error.message, - code: error.code, - ...(error.response && { - response: { - status: error.response.status, - statusText: error.response.statusText, - data: error.response.data, - }, - }), - }); - fs.writeFileSync(errorPath, fixtureError); - } - + fs.writeFileSync(errorPath, `${error}`); throw error; } - fs.writeFileSync(path, response); + if (!response.ok) { + const message = `Fetcher error: ${response.status}: ${response.statusText}`; + fs.writeFileSync(errorPath, message); + throw new Error(message); + } - return response; + let result; + if (init?.method === 'HEAD') { + result = JSON.stringify(response.headers); + } else { + result = await response.text(); + } + + fs.writeFileSync(path, result); + + return result; } }; - - private readonly getConfig = (config?: AxiosRequestConfig): AxiosRequestConfig => { - return { - responseType: 'text', - ...config, - }; - }; } diff --git a/src/utils/tmdb.ts b/src/utils/tmdb.ts index d6e298e..44399b1 100644 --- a/src/utils/tmdb.ts +++ b/src/utils/tmdb.ts @@ -6,7 +6,7 @@ export interface TmdbId { id: number; series: number | undefined; episode: numbe export const getTmdbIdFromImdbId = async (ctx: Context, fetcher: Fetcher, imdbId: ImdbId): Promise => { const url = new URL(`https://api.themoviedb.org/3/find/${imdbId.id}?external_source=imdb_id`); - const config = { headers: { Authorization: 'Bearer ' + process.env['TMDB_ACCESS_TOKEN'] } }; + const config = { 'headers': { Authorization: 'Bearer ' + process.env['TMDB_ACCESS_TOKEN'] }, 'Content-Type': 'application/json' }; const response = JSON.parse(await fetcher.text(ctx, url, config)); const id = (imdbId.series ? response.tv_results[0] : response.movie_results[0])?.id;