fix: switch back to fetch and do not generate user agents
seems to be causing less cloudflare challenges
This commit is contained in:
WebStreamr
parent
2f9a349d31
commit
fbc1aa994c
13 changed files with 240 additions and 316 deletions
241
package-lock.json
generated
241
package-lock.json
generated
|
|
@ -10,16 +10,14 @@
|
|||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@isaacs/ttlcache": "^1.4.1",
|
||||
"axios-cache-interceptor": "^1.8.0",
|
||||
"axios-retry": "^4.5.0",
|
||||
"bytes": "^3.1.2",
|
||||
"cheerio": "^1.0.0",
|
||||
"country-emoji": "^1.5.6",
|
||||
"express": "^5.1.0",
|
||||
"http-cache-semantics": "^4.2.0",
|
||||
"randomstring": "^1.3.1",
|
||||
"slugify": "^1.6.6",
|
||||
"unpacker": "^1.0.1",
|
||||
"user-agents": "^1.1.536",
|
||||
"uuid": "^11.1.0",
|
||||
"winston": "^3.17.0"
|
||||
},
|
||||
|
|
@ -28,15 +26,15 @@
|
|||
"@stylistic/eslint-plugin": "^4.2.0",
|
||||
"@types/bytes": "^3.1.5",
|
||||
"@types/express": "^5.0.1",
|
||||
"@types/http-cache-semantics": "^4.0.4",
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/make-fetch-happen": "^10.0.4",
|
||||
"@types/node": "^22.15.3",
|
||||
"@types/randomstring": "^1.3.0",
|
||||
"@types/stremio-addon-sdk": "^1.6.11",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"axios-mock-adapter": "^2.1.0",
|
||||
"babel-jest": "^29.7.0",
|
||||
"eslint": "^9.26.0",
|
||||
"fetch-mock": "^12.5.2",
|
||||
"jest": "^29.7.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
@ -1509,6 +1507,13 @@
|
|||
"@types/send": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/glob-to-regexp": {
|
||||
"version": "0.4.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz",
|
||||
"integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/graceful-fs": {
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz",
|
||||
|
|
@ -1519,6 +1524,13 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/http-cache-semantics": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
|
||||
"integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/http-errors": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz",
|
||||
|
|
@ -1692,13 +1704,6 @@
|
|||
"integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/user-agents": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/user-agents/-/user-agents-1.0.4.tgz",
|
||||
"integrity": "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/yargs": {
|
||||
"version": "17.0.33",
|
||||
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
|
||||
|
|
@ -2119,65 +2124,8 @@
|
|||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz",
|
||||
"integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.6",
|
||||
"form-data": "^4.0.0",
|
||||
"proxy-from-env": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-cache-interceptor": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/axios-cache-interceptor/-/axios-cache-interceptor-1.8.0.tgz",
|
||||
"integrity": "sha512-cTNnPGJyQkxnWp0EWvE3NRvgURU5cWw/Qx3dIhXyHSM4Ip0c7EEe0I3an0Jwa549m1CAOg57ibj27YRNLmQCcg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cache-parser": "1.2.5",
|
||||
"fast-defer": "1.1.8",
|
||||
"object-code": "1.3.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/arthurfiorette/axios-cache-interceptor?sponsor=1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"axios": "^1"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-mock-adapter": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.1.0.tgz",
|
||||
"integrity": "sha512-AZUe4OjECGCNNssH8SOdtneiQELsqTsat3SQQCWLPjN436/H+L9AjWfV7bF+Zg/YL9cgbhrz5671hoh+Tbn98w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"is-buffer": "^2.0.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"axios": ">= 0.17.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-retry": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.5.0.tgz",
|
||||
"integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"is-retry-allowed": "^2.2.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"axios": "0.x || 1.x"
|
||||
}
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/babel-jest": {
|
||||
"version": "29.7.0",
|
||||
|
|
@ -2419,12 +2367,6 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/cache-parser": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/cache-parser/-/cache-parser-1.2.5.tgz",
|
||||
"integrity": "sha512-Md/4VhAHByQ9frQ15WD6LrMNiVw9AEl/J7vWIXw+sxT6fSOpbtt6LHTp76vy8+bOESPBO94117Hm2bIjlI7XjA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/call-bind-apply-helpers": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
|
||||
|
|
@ -2726,6 +2668,7 @@
|
|||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
|
|
@ -2918,6 +2861,7 @@
|
|||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
|
|
@ -2932,6 +2876,16 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-newline": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
|
||||
|
|
@ -3164,6 +3118,7 @@
|
|||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
|
|
@ -3554,12 +3509,6 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-defer": {
|
||||
"version": "1.1.8",
|
||||
"resolved": "https://registry.npmjs.org/fast-defer/-/fast-defer-1.1.8.tgz",
|
||||
"integrity": "sha512-lEJeOH5VL5R09j6AA0D4Uvq7AgsHw0dAImQQ+F3iSyHZuAxyQfWobsagGpTcOPvJr3urmKRHrs+Gs9hV+/Qm/Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-glob": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||
|
|
@ -3630,6 +3579,22 @@
|
|||
"integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fetch-mock": {
|
||||
"version": "12.5.2",
|
||||
"resolved": "https://registry.npmjs.org/fetch-mock/-/fetch-mock-12.5.2.tgz",
|
||||
"integrity": "sha512-b5KGDFmdmado2MPQjZl6ix3dAG3iwCitb0XQwN72y2s9VnWZ3ObaGNy+bkpm1390foiLDybdJ7yjRGKD36kATw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/glob-to-regexp": "^0.4.4",
|
||||
"dequal": "^2.0.3",
|
||||
"glob-to-regexp": "^0.4.1",
|
||||
"regexparam": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/file-entry-cache": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
|
||||
|
|
@ -3737,31 +3702,11 @@
|
|||
"integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.9",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
|
||||
"integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/RubenVerborgh"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"debug": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz",
|
||||
"integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
|
|
@ -3777,6 +3722,7 @@
|
|||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
|
|
@ -3786,6 +3732,7 @@
|
|||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
|
|
@ -3958,6 +3905,13 @@
|
|||
"node": ">=10.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-to-regexp": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
|
||||
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/glob/node_modules/brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
|
|
@ -4044,6 +3998,7 @@
|
|||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"has-symbols": "^1.0.3"
|
||||
|
|
@ -4093,6 +4048,12 @@
|
|||
"entities": "^4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/http-cache-semantics": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz",
|
||||
"integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/http-errors": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
|
||||
|
|
@ -4252,30 +4213,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/is-buffer": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
|
||||
"integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/is-core-module": {
|
||||
"version": "2.16.1",
|
||||
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",
|
||||
|
|
@ -4351,18 +4288,6 @@
|
|||
"integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-retry-allowed": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
|
||||
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-stream": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
|
||||
|
|
@ -5250,12 +5175,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.clonedeep": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
|
||||
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.memoize": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
|
||||
|
|
@ -5624,12 +5543,6 @@
|
|||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/object-code": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/object-code/-/object-code-1.3.3.tgz",
|
||||
"integrity": "sha512-/Ds4Xd5xzrtUOJ+xJQ57iAy0BZsZltOHssnDgcZ8DOhgh41q1YJCnTPnWdWSLkNGNnxYzhYChjc5dgC9mEERCA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/object-inspect": {
|
||||
"version": "1.13.4",
|
||||
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
|
||||
|
|
@ -6005,13 +5918,6 @@
|
|||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/pstree.remy": {
|
||||
"version": "1.1.8",
|
||||
"resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
|
||||
|
|
@ -6177,6 +6083,16 @@
|
|||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/regexparam": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
||||
"integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
|
|
@ -7120,15 +7036,6 @@
|
|||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/user-agents": {
|
||||
"version": "1.1.550",
|
||||
"resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.550.tgz",
|
||||
"integrity": "sha512-lt8m9L4qfBityhTJjr36G2ZRlVtDpQyM8H7s5l2avLWkCeSLdJm0yzCyn1ZEP/fu2GFt10LJGu0hdLKEYT6Dew==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"lodash.clonedeep": "^4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
|
|
|
|||
|
|
@ -32,16 +32,14 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@isaacs/ttlcache": "^1.4.1",
|
||||
"axios-cache-interceptor": "^1.8.0",
|
||||
"axios-retry": "^4.5.0",
|
||||
"bytes": "^3.1.2",
|
||||
"cheerio": "^1.0.0",
|
||||
"country-emoji": "^1.5.6",
|
||||
"express": "^5.1.0",
|
||||
"http-cache-semantics": "^4.2.0",
|
||||
"randomstring": "^1.3.1",
|
||||
"slugify": "^1.6.6",
|
||||
"unpacker": "^1.0.1",
|
||||
"user-agents": "^1.1.536",
|
||||
"uuid": "^11.1.0",
|
||||
"winston": "^3.17.0"
|
||||
},
|
||||
|
|
@ -50,15 +48,15 @@
|
|||
"@stylistic/eslint-plugin": "^4.2.0",
|
||||
"@types/bytes": "^3.1.5",
|
||||
"@types/express": "^5.0.1",
|
||||
"@types/http-cache-semantics": "^4.0.4",
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/make-fetch-happen": "^10.0.4",
|
||||
"@types/node": "^22.15.3",
|
||||
"@types/randomstring": "^1.3.0",
|
||||
"@types/stremio-addon-sdk": "^1.6.11",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"axios-mock-adapter": "^2.1.0",
|
||||
"babel-jest": "^29.7.0",
|
||||
"eslint": "^9.26.0",
|
||||
"fetch-mock": "^12.5.2",
|
||||
"jest": "^29.7.0",
|
||||
"nodemon": "^3.1.10",
|
||||
"ts-jest": "^29.3.2",
|
||||
|
|
|
|||
10
src/index.ts
10
src/index.ts
|
|
@ -1,8 +1,5 @@
|
|||
import express, { NextFunction, Request, Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import axios from 'axios';
|
||||
import axiosRetry from 'axios-retry';
|
||||
import { setupCache } from 'axios-cache-interceptor';
|
||||
import winston from 'winston';
|
||||
import {
|
||||
CineHDPlus,
|
||||
|
|
@ -30,12 +27,7 @@ const logger = winston.createLogger({
|
|||
],
|
||||
});
|
||||
|
||||
setupCache(axios);
|
||||
axiosRetry(axios, {
|
||||
retries: 1,
|
||||
});
|
||||
|
||||
const fetcher = new Fetcher(axios, logger);
|
||||
const fetcher = new Fetcher(logger);
|
||||
|
||||
const extractorRegistry = new ExtractorRegistry(logger, fetcher);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
import winston from 'winston';
|
||||
import axios, { AxiosError } from 'axios';
|
||||
import AxiosMockAdapter from 'axios-mock-adapter';
|
||||
import fetchMock from 'fetch-mock';
|
||||
import { Fetcher } from './Fetcher';
|
||||
import { Context } from '../types';
|
||||
import { CloudflareChallengeError, NotFoundError } from '../error';
|
||||
fetchMock.mockGlobal();
|
||||
|
||||
const axiosMock = new AxiosMockAdapter(axios);
|
||||
|
||||
const fetcher = new Fetcher(axios, winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
|
||||
|
||||
describe('fetch', () => {
|
||||
const ctx: Context = { id: 'id', ip: '127.0.0.1', config: { de: 'on' } };
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.clearHistory();
|
||||
});
|
||||
|
||||
test('text passes successful response through setting headers', async () => {
|
||||
axiosMock.onGet().reply(200, 'some text');
|
||||
const axiosSpy = jest.spyOn(axios, 'get');
|
||||
fetchMock.get('https://some-url.test/', 'some text');
|
||||
|
||||
const responseText1 = await fetcher.text(ctx, new URL('https://some-url.test/'));
|
||||
const responseText2 = await fetcher.text(ctx, new URL('https://some-url.test/'), { headers: { 'User-Agent': 'jest' } });
|
||||
|
|
@ -22,71 +23,80 @@ describe('fetch', () => {
|
|||
expect(responseText1).toBe('some text');
|
||||
expect(responseText2).toStrictEqual(responseText1);
|
||||
|
||||
expect(axiosSpy).toHaveBeenCalledWith(
|
||||
'https://some-url.test/',
|
||||
{
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Forwarded': 'for=127.0.0.1',
|
||||
'Origin': 'https://some-url.test',
|
||||
'Priority': 'u=0',
|
||||
'Referer': 'https://some-url.test',
|
||||
'User-Agent': expect.not.stringMatching(/jest/),
|
||||
'X-Forwarded-For': '127.0.0.1',
|
||||
'X-Forwarded-Proto': 'https',
|
||||
'X-Real-IP': '127.0.0.1',
|
||||
},
|
||||
responseType: 'text',
|
||||
timeout: 5000,
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
'Forwarded': 'for=127.0.0.1',
|
||||
'Origin': 'https://some-url.test',
|
||||
'Priority': 'u=0',
|
||||
'Referer': 'https://some-url.test',
|
||||
'X-Forwarded-For': '127.0.0.1',
|
||||
'X-Forwarded-Proto': 'https',
|
||||
'X-Real-IP': '127.0.0.1',
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('textPost ', async () => {
|
||||
axiosMock.onPost().reply(200, 'some text');
|
||||
fetchMock.post('https://some-post-url.test/', 'some text');
|
||||
|
||||
expect(await fetcher.textPost(ctx, new URL('https://some-url.test/'), { foo: 'bar' })).toBe('some text');
|
||||
expect(await fetcher.textPost(ctx, new URL('https://some-post-url.test/'), { foo: 'bar' })).toBe('some text');
|
||||
});
|
||||
|
||||
test('head', async () => {
|
||||
axiosMock.onHead().reply(200, undefined, { 'X-Fake-Response': 'foo' });
|
||||
fetchMock.head('https://some-head-url.test/', { status: 200, headers: { 'X-Fake-Response': 'foo' } });
|
||||
|
||||
expect(await fetcher.head(ctx, new URL('https://some-url.test/'))).toMatchObject({ 'X-Fake-Response': 'foo' });
|
||||
expect(await fetcher.head(ctx, new URL('https://some-head-url.test/'))).toMatchObject({ 'x-fake-response': 'foo' });
|
||||
});
|
||||
|
||||
test('uses context referer', async () => {
|
||||
axiosMock.onGet().reply(200, 'some text');
|
||||
const axiosSpy = jest.spyOn(axios, 'get');
|
||||
fetchMock.get('https://some-referer-url.test/', 'some text');
|
||||
|
||||
await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-url.test/'));
|
||||
await fetcher.text({ ...ctx, referer: new URL('https://example.com/foo/bar') }, new URL('https://some-referer-url.test/'));
|
||||
|
||||
expect(axiosSpy).toHaveBeenCalledWith(
|
||||
'https://some-url.test/',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Origin: 'https://example.com',
|
||||
Referer: 'https://example.com/foo/bar',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock.callHistory.callLogs[0]?.args[1]).toMatchObject({
|
||||
headers: {
|
||||
Origin: 'https://example.com',
|
||||
Referer: 'https://example.com/foo/bar',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('converts 404 to custom NotFoundError', async () => {
|
||||
axiosMock.onGet().reply(404);
|
||||
fetchMock.get('https://some-404-url.test/', 404);
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
|
||||
await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
|
||||
});
|
||||
|
||||
test('converts Cloudflare challenge block to custom CloudflareChallengeError', async () => {
|
||||
axiosMock.onGet().reply(403, undefined, { 'cf-mitigated': 'challenge' });
|
||||
fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError);
|
||||
await expect(fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError);
|
||||
});
|
||||
|
||||
test('passes through other errors', async () => {
|
||||
axiosMock.onGet().networkError();
|
||||
fetchMock.get('https://some-error-url.test/', 500);
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(AxiosError);
|
||||
await expect(fetcher.text(ctx, new URL('https://some-error-url.test/'))).rejects.toBeInstanceOf(Error);
|
||||
});
|
||||
|
||||
test('passes through exceptions', async () => {
|
||||
fetchMock.get('https://some-exception-url.test/', { throws: new TypeError('Failed to fetch') });
|
||||
|
||||
await expect(fetcher.text(ctx, new URL('https://some-exception-url.test/'))).rejects.toBeInstanceOf(TypeError);
|
||||
});
|
||||
|
||||
test('times out after 10 seconds', async () => {
|
||||
jest.useFakeTimers();
|
||||
fetchMock.get('https://some-timeout-url.test/', 200, { delay: 15000 });
|
||||
|
||||
const promise = fetcher.text(ctx, new URL('https://some-timeout-url.test/'));
|
||||
|
||||
jest.advanceTimersByTime(13000);
|
||||
|
||||
await expect(promise).rejects.toBeInstanceOf(DOMException);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,47 +1,39 @@
|
|||
import { AxiosInstance, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders, isAxiosError } from 'axios';
|
||||
import CachePolicy from 'http-cache-semantics';
|
||||
import TTLCache from '@isaacs/ttlcache';
|
||||
import UserAgent from 'user-agents';
|
||||
import winston from 'winston';
|
||||
import { Context } from '../types';
|
||||
import { CloudflareChallengeError, NotFoundError } from '../error';
|
||||
import { clearTimeout } from 'node:timers';
|
||||
|
||||
interface HttpCacheItem { policy: CachePolicy; status: number; statusText: string; body: string }
|
||||
|
||||
export class Fetcher {
|
||||
private readonly axios: AxiosInstance;
|
||||
private readonly logger: winston.Logger;
|
||||
private readonly ipUserAgentCache: TTLCache<string, string>;
|
||||
private readonly httpCache: TTLCache<string, HttpCacheItem>;
|
||||
|
||||
constructor(axios: AxiosInstance, logger: winston.Logger) {
|
||||
this.axios = axios;
|
||||
constructor(logger: winston.Logger) {
|
||||
this.logger = logger;
|
||||
this.ipUserAgentCache = new TTLCache({ max: 1024, ttl: 86400000 }); // 24h
|
||||
this.httpCache = new TTLCache();
|
||||
}
|
||||
|
||||
readonly text = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<string> => {
|
||||
this.logger.info(`Fetch ${url}`, ctx);
|
||||
|
||||
return (await this.errorWrapper(() => this.axios.get(url.href, this.getConfig(ctx, url, config)))).data;
|
||||
readonly text = async (ctx: Context, url: URL, init?: RequestInit): Promise<string> => {
|
||||
return (await this.cachedFetch(ctx, url, init)).body;
|
||||
};
|
||||
|
||||
readonly textPost = async (ctx: Context, url: URL, data: unknown, config?: AxiosRequestConfig): Promise<string> => {
|
||||
this.logger.info(`Fetch post ${url}`, ctx);
|
||||
|
||||
return (await this.errorWrapper(() => this.axios.post(url.href, data, this.getConfig(ctx, url, config)))).data;
|
||||
readonly textPost = async (ctx: Context, url: URL, data: unknown, init?: RequestInit): Promise<string> => {
|
||||
return (await this.cachedFetch(ctx, url, { ...init, method: 'POST', body: JSON.stringify(data) })).body;
|
||||
};
|
||||
|
||||
readonly head = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<RawAxiosResponseHeaders | AxiosResponseHeaders> => {
|
||||
this.logger.info(`Fetch head ${url}`, ctx);
|
||||
|
||||
return (await this.errorWrapper(() => this.axios.head(url.href, this.getConfig(ctx, url, config)))).headers;
|
||||
readonly head = async (ctx: Context, url: URL, init?: RequestInit): Promise<CachePolicy.Headers> => {
|
||||
return (await this.cachedFetch(ctx, url, { ...init, method: 'HEAD' })).policy.responseHeaders();
|
||||
};
|
||||
|
||||
private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => {
|
||||
private readonly getInit = (ctx: Context, url: URL, init?: RequestInit): RequestInit => {
|
||||
const origin = ctx.referer?.origin ?? url.origin;
|
||||
const referer = ctx.referer?.href ?? url.origin;
|
||||
|
||||
return {
|
||||
responseType: 'text',
|
||||
timeout: 5000,
|
||||
...config,
|
||||
...init,
|
||||
headers: {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en',
|
||||
|
|
@ -49,43 +41,80 @@ export class Fetcher {
|
|||
'Origin': `${origin}`,
|
||||
'Priority': 'u=0',
|
||||
'Referer': `${referer}`,
|
||||
'User-Agent': this.createUserAgentForIp(ctx.ip),
|
||||
'X-Forwarded-For': ctx.ip,
|
||||
'X-Forwarded-Proto': url.protocol.slice(0, -1),
|
||||
'X-Real-IP': ctx.ip,
|
||||
...config?.headers,
|
||||
...init?.headers,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
private readonly createUserAgentForIp = (ip: string): string => {
|
||||
let userAgent = this.ipUserAgentCache.get(ip);
|
||||
if (userAgent) {
|
||||
return userAgent;
|
||||
private readonly handleHttpCacheItem = (httpCacheItem: HttpCacheItem): HttpCacheItem => {
|
||||
if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) {
|
||||
return httpCacheItem;
|
||||
}
|
||||
|
||||
userAgent = (new UserAgent()).toString();
|
||||
if (httpCacheItem.status === 404) {
|
||||
throw new NotFoundError();
|
||||
}
|
||||
|
||||
this.ipUserAgentCache.set(ip, userAgent);
|
||||
if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') {
|
||||
throw new CloudflareChallengeError();
|
||||
}
|
||||
|
||||
return userAgent;
|
||||
throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}`);
|
||||
};
|
||||
|
||||
private readonly errorWrapper = async <T>(callable: () => T): Promise<T> => {
|
||||
try {
|
||||
return await callable();
|
||||
} catch (error) {
|
||||
if (isAxiosError(error)) {
|
||||
if (error.response?.status === 404) {
|
||||
throw new NotFoundError();
|
||||
}
|
||||
private readonly headersToObject = (headers: Headers): Record<string, string> => {
|
||||
const obj: Record<string, string> = {};
|
||||
|
||||
if (error.response?.headers['cf-mitigated'] === 'challenge') {
|
||||
throw new CloudflareChallengeError();
|
||||
}
|
||||
}
|
||||
headers.forEach((value, name) => {
|
||||
obj[name] = value;
|
||||
});
|
||||
|
||||
throw error;
|
||||
return obj;
|
||||
};
|
||||
|
||||
private readonly cachedFetch = async (ctx: Context, url: URL, init?: RequestInit): Promise<HttpCacheItem> => {
|
||||
const newInit = this.getInit(ctx, url, init);
|
||||
|
||||
const request: CachePolicy.Request = { url: url.href, method: newInit.method ?? 'GET', headers: {} };
|
||||
|
||||
let httpCacheItem: HttpCacheItem | undefined = this.httpCache.get(url.href);
|
||||
if (httpCacheItem) {
|
||||
this.logger.info(`Cached fetch ${request.method} ${url}`, ctx);
|
||||
return this.handleHttpCacheItem(httpCacheItem);
|
||||
}
|
||||
|
||||
this.logger.info(`Fetch ${request.method} ${url}`, ctx);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 10000);
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, { ...newInit, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
const body = await response.text();
|
||||
|
||||
const policy = new CachePolicy(request, { status: response.status, headers: this.headersToObject(response.headers) }, { shared: false });
|
||||
|
||||
httpCacheItem = { policy, status: response.status, statusText: response.statusText, body };
|
||||
|
||||
let ttl;
|
||||
if (response.ok) {
|
||||
ttl = Math.max(policy.timeToLive(), 900000); // 15m at least
|
||||
} else {
|
||||
ttl = policy.timeToLive();
|
||||
}
|
||||
|
||||
if (ttl > 0) {
|
||||
this.httpCache.set(url.href, httpCacheItem, { ttl });
|
||||
}
|
||||
|
||||
return this.handleHttpCacheItem(httpCacheItem);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"message":"socket hang up","code":"ECONNRESET"}
|
||||
TypeError: fetch failed
|
||||
|
|
@ -1 +1 @@
|
|||
{"message":"unable to verify the first certificate","code":"UNABLE_TO_VERIFY_LEAF_SIGNATURE"}
|
||||
TypeError: fetch failed
|
||||
|
|
@ -1 +1 @@
|
|||
{"message":"getaddrinfo ENOTFOUND some-url.test","code":"ENOTFOUND"}
|
||||
TypeError: fetch failed
|
||||
|
|
@ -1 +1 @@
|
|||
{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}}
|
||||
Fetcher error: 404: Not Found
|
||||
|
|
@ -1 +1 @@
|
|||
{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":""}}
|
||||
Fetcher error: 404: Not Found
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"message":"Request failed with status code 404","code":"ERR_BAD_REQUEST","response":{"status":404,"statusText":"Not Found","data":"{\"error\":\"Series not found\"}"}}
|
||||
|
|
@ -1,74 +1,63 @@
|
|||
import fs from 'node:fs';
|
||||
import Axios, { AxiosError, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios';
|
||||
import slugify from 'slugify';
|
||||
import { Context } from '../../types';
|
||||
|
||||
export class Fetcher {
|
||||
readonly text = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<string> => {
|
||||
readonly text = async (_ctx: Context, url: URL, init?: RequestInit): Promise<string> => {
|
||||
const path = `${__dirname}/../__fixtures__/Fetcher/${slugify(url.href)}`;
|
||||
|
||||
return this.fixtureWrapper(path, async () => (await Axios.create().get(url.href, this.getConfig(config))).data);
|
||||
return this.fetchy(path, url, init);
|
||||
};
|
||||
|
||||
readonly textPost = async (_ctx: Context, url: URL, data: unknown, config?: AxiosRequestConfig): Promise<string> => {
|
||||
readonly textPost = async (_ctx: Context, url: URL, data: unknown, init?: RequestInit): Promise<string> => {
|
||||
const path = `${__dirname}/../__fixtures__/Fetcher/post-${slugify(url.href)}-${slugify(JSON.stringify(data))}`;
|
||||
|
||||
return this.fixtureWrapper(path, async () => (await Axios.create().post(url.href, data, this.getConfig(config))).data);
|
||||
return this.fetchy(path, url, { ...init, method: 'POST', body: JSON.stringify(data) });
|
||||
};
|
||||
|
||||
readonly head = async (_ctx: Context, url: URL, config?: AxiosRequestConfig): Promise<RawAxiosResponseHeaders | AxiosResponseHeaders> => {
|
||||
readonly head = async (_ctx: Context, url: URL, init?: RequestInit): Promise<unknown> => {
|
||||
const path = `${__dirname}/../__fixtures__/Fetcher/head-${slugify(url.href)}`;
|
||||
|
||||
return JSON.parse(await this.fixtureWrapper(path, async () => JSON.stringify((await Axios.create().head(url.href, this.getConfig(config))).headers)));
|
||||
return JSON.parse(await this.fetchy(path, url, { ...init, method: 'HEAD' }));
|
||||
};
|
||||
|
||||
private readonly fixtureWrapper = async (path: string, callable: () => Promise<string>): Promise<string> => {
|
||||
private readonly fetchy = async (path: string, url: URL, init?: RequestInit): Promise<string> => {
|
||||
const errorPath = `${path}.error`;
|
||||
|
||||
if (fs.existsSync(errorPath)) {
|
||||
const fixtureError = JSON.parse(fs.readFileSync(errorPath).toString());
|
||||
throw new AxiosError(fixtureError.message, fixtureError.code, undefined, undefined, fixtureError.response);
|
||||
throw new Error(fs.readFileSync(errorPath).toString());
|
||||
} else if (fs.existsSync(path)) {
|
||||
return fs.readFileSync(path).toString();
|
||||
} else {
|
||||
let response;
|
||||
|
||||
try {
|
||||
if (process.env['TEST_UPDATE_FIXTURES']) {
|
||||
response = await callable();
|
||||
response = await fetch(url, init);
|
||||
} else {
|
||||
console.error(`No fixture found at "${path}".`);
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof AxiosError) {
|
||||
const fixtureError = JSON.stringify({
|
||||
message: error.message,
|
||||
code: error.code,
|
||||
...(error.response && {
|
||||
response: {
|
||||
status: error.response.status,
|
||||
statusText: error.response.statusText,
|
||||
data: error.response.data,
|
||||
},
|
||||
}),
|
||||
});
|
||||
fs.writeFileSync(errorPath, fixtureError);
|
||||
}
|
||||
|
||||
fs.writeFileSync(errorPath, `${error}`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
fs.writeFileSync(path, response);
|
||||
if (!response.ok) {
|
||||
const message = `Fetcher error: ${response.status}: ${response.statusText}`;
|
||||
fs.writeFileSync(errorPath, message);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
return response;
|
||||
let result;
|
||||
if (init?.method === 'HEAD') {
|
||||
result = JSON.stringify(response.headers);
|
||||
} else {
|
||||
result = await response.text();
|
||||
}
|
||||
|
||||
fs.writeFileSync(path, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
private readonly getConfig = (config?: AxiosRequestConfig): AxiosRequestConfig => {
|
||||
return {
|
||||
responseType: 'text',
|
||||
...config,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ export interface TmdbId { id: number; series: number | undefined; episode: numbe
|
|||
|
||||
export const getTmdbIdFromImdbId = async (ctx: Context, fetcher: Fetcher, imdbId: ImdbId): Promise<TmdbId> => {
|
||||
const url = new URL(`https://api.themoviedb.org/3/find/${imdbId.id}?external_source=imdb_id`);
|
||||
const config = { headers: { Authorization: 'Bearer ' + process.env['TMDB_ACCESS_TOKEN'] } };
|
||||
const config = { 'headers': { Authorization: 'Bearer ' + process.env['TMDB_ACCESS_TOKEN'] }, 'Content-Type': 'application/json' };
|
||||
const response = JSON.parse(await fetcher.text(ctx, url, config));
|
||||
|
||||
const id = (imdbId.series ? response.tv_results[0] : response.movie_results[0])?.id;
|
||||
|
|
|
|||
Loading…
Reference in a new issue