diff --git a/package-lock.json b/package-lock.json index 9778c13..8dfc0f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,9 +11,11 @@ "dependencies": { "cheerio": "^1.0.0", "express": "^5.1.0", + "lru-cache": "^11.1.0", "make-fetch-happen": "^14.0.3", "slugify": "^1.6.6", - "unpacker": "^1.0.1" + "unpacker": "^1.0.1", + "user-agents": "^1.1.536" }, "devDependencies": { "@eslint/js": "^9.26.0", @@ -23,6 +25,7 @@ "@types/make-fetch-happen": "^10.0.4", "@types/node": "^22.15.3", "@types/stremio-addon-sdk": "^1.6.11", + "@types/user-agents": "^1.0.4", "babel-jest": "^29.7.0", "eslint": "^9.26.0", "jest": "^29.7.0", @@ -140,6 +143,16 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, "node_modules/@babel/helper-module-imports": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", @@ -1791,6 +1804,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/user-agents": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/user-agents/-/user-agents-1.0.4.tgz", + "integrity": "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -3792,6 +3812,16 @@ "node": ">=16" } }, + "node_modules/flat-cache/node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", @@ -5299,16 +5329,6 @@ "node": ">=6" } }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "json-buffer": "3.0.1" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -5363,6 +5383,12 @@ "node": ">=8" } }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -5378,13 +5404,12 @@ "license": "MIT" }, "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", + "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" + "engines": { + "node": "20 || >=22" } }, "node_modules/make-dir": { @@ -7519,6 +7544,15 @@ "punycode": "^2.1.0" } }, + "node_modules/user-agents": { + "version": "1.1.536", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.536.tgz", + "integrity": "sha512-N2a8i2Dp8OI4a7xFaE0720cqiicgYl4JQLmbcF7ThUMtj3hhQMzdzPi1zadPpp3e4jZBFOLEyASzplAvcIuQcA==", + "license": "BSD-2-Clause", + "dependencies": { + "lodash.clonedeep": "^4.5.0" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index 778dabf..a24f06f 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,11 @@ "dependencies": { "cheerio": "^1.0.0", "express": "^5.1.0", + "lru-cache": "^11.1.0", "make-fetch-happen": "^14.0.3", "slugify": "^1.6.6", - "unpacker": "^1.0.1" + "unpacker": "^1.0.1", + "user-agents": "^1.1.536" }, "devDependencies": { "@eslint/js": "^9.26.0", @@ -44,6 +46,7 @@ "@types/make-fetch-happen": "^10.0.4", "@types/node": "^22.15.3", "@types/stremio-addon-sdk": "^1.6.11", + "@types/user-agents": "^1.0.4", "babel-jest": "^29.7.0", "eslint": "^9.26.0", "jest": "^29.7.0", diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 0bc1cd5..a0001a9 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -35,7 +35,7 @@ describe('fetch', () => { expect(responseText).toBe('some text'); expect(mockedFetch).toHaveBeenCalledWith( 'https://some-url.test', - { headers: { 'User-Agent': 'jest', 'X-Forwarded-For': '127.0.0.1' } }, + { headers: { 'User-Agent': expect.not.stringMatching(/jest/), 'X-Forwarded-For': '127.0.0.1' } }, ); }); }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 02761cd..c39df23 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,12 +1,17 @@ import { FetchInterface, FetchOptions } from 'make-fetch-happen'; +import UserAgent from 'user-agents'; +import { LRUCache } from 'lru-cache'; import { logInfo } from './log'; import { Context } from '../types'; export class Fetcher { private readonly fetch: FetchInterface; + private readonly cache: LRUCache; + constructor(fetch: FetchInterface) { this.fetch = fetch; + this.cache = new LRUCache({ max: 128 }); } readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise => { @@ -18,6 +23,7 @@ export class Fetcher { ...opts, headers: { ...opts?.headers, + 'User-Agent': this.createUserAgentForIp(ctx.ip), 'X-Forwarded-For': ctx.ip, }, }, @@ -29,4 +35,17 @@ export class Fetcher { return await response.text(); }; + + private readonly createUserAgentForIp = (ip: string): string => { + let userAgent = this.cache.get(ip); + if (userAgent) { + return userAgent; + } + + userAgent = (new UserAgent()).toString(); + + this.cache.set(ip, userAgent); + + return userAgent; + }; }