Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

feat: use randomized user agent for fetch

Browse source
This commit is contained in:
WebStreamr 2025-05-11 20:05:31 +00:00
parent efef039cde
commit b114ac11b7
No known key found for this signature in database
4 changed files with 75 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
package-lock.json generated
View file

@ -11,9 +11,11 @@
"dependencies": {
"cheerio": "^1.0.0",
"express": "^5.1.0",
"lru-cache": "^11.1.0",
"make-fetch-happen": "^14.0.3",
"slugify": "^1.6.6",
"unpacker": "^1.0.1"
"unpacker": "^1.0.1",
"user-agents": "^1.1.536"
},
"devDependencies": {
"@eslint/js": "^9.26.0",
@ -23,6 +25,7 @@
"@types/make-fetch-happen": "^10.0.4",
"@types/node": "^22.15.3",
"@types/stremio-addon-sdk": "^1.6.11",
"@types/user-agents": "^1.0.4",
"babel-jest": "^29.7.0",
"eslint": "^9.26.0",
"jest": "^29.7.0",
@ -140,6 +143,16 @@
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dev": true,
"license": "ISC",
"dependencies": {
"yallist": "^3.0.2"
}
},
"node_modules/@babel/helper-module-imports": {
"version": "7.27.1",
"resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz",
@ -1791,6 +1804,13 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/user-agents": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@types/user-agents/-/user-agents-1.0.4.tgz",
"integrity": "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/yargs": {
"version": "17.0.33",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
@ -3792,6 +3812,16 @@
"node": ">=16"
}
},
"node_modules/flat-cache/node_modules/keyv": {
"version": "4.5.4",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
"integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
"dev": true,
"license": "MIT",
"dependencies": {
"json-buffer": "3.0.1"
}
},
"node_modules/flatted": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
@ -5299,16 +5329,6 @@
"node": ">=6"
}
},
"node_modules/keyv": {
"version": "4.5.4",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
"integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
"dev": true,
"license": "MIT",
"dependencies": {
"json-buffer": "3.0.1"
}
},
"node_modules/kleur": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@ -5363,6 +5383,12 @@
"node": ">=8"
}
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==",
"license": "MIT"
},
"node_modules/lodash.memoize": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@ -5378,13 +5404,12 @@
"license": "MIT"
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dev": true,
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz",
"integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==",
"license": "ISC",
"dependencies": {
"yallist": "^3.0.2"
"engines": {
"node": "20 || >=22"
}
},
"node_modules/make-dir": {
@ -7519,6 +7544,15 @@
"punycode": "^2.1.0"
}
},
"node_modules/user-agents": {
"version": "1.1.536",
"resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.536.tgz",
"integrity": "sha512-N2a8i2Dp8OI4a7xFaE0720cqiicgYl4JQLmbcF7ThUMtj3hhQMzdzPi1zadPpp3e4jZBFOLEyASzplAvcIuQcA==",
"license": "BSD-2-Clause",
"dependencies": {
"lodash.clonedeep": "^4.5.0"
}
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",

5
package.json
View file

@ -32,9 +32,11 @@
"dependencies": {
"cheerio": "^1.0.0",
"express": "^5.1.0",
"lru-cache": "^11.1.0",
"make-fetch-happen": "^14.0.3",
"slugify": "^1.6.6",
"unpacker": "^1.0.1"
"unpacker": "^1.0.1",
"user-agents": "^1.1.536"
},
"devDependencies": {
"@eslint/js": "^9.26.0",
@ -44,6 +46,7 @@
"@types/make-fetch-happen": "^10.0.4",
"@types/node": "^22.15.3",
"@types/stremio-addon-sdk": "^1.6.11",
"@types/user-agents": "^1.0.4",
"babel-jest": "^29.7.0",
"eslint": "^9.26.0",
"jest": "^29.7.0",

2
src/utils/Fetcher.test.ts
View file

@ -35,7 +35,7 @@ describe('fetch', () => {
expect(responseText).toBe('some text');
expect(mockedFetch).toHaveBeenCalledWith(
'https://some-url.test',
{ headers: { 'User-Agent': 'jest', 'X-Forwarded-For': '127.0.0.1' } },
{ headers: { 'User-Agent': expect.not.stringMatching(/jest/), 'X-Forwarded-For': '127.0.0.1' } },
);
});
});

19
src/utils/Fetcher.ts
View file

@ -1,12 +1,17 @@
import { FetchInterface, FetchOptions } from 'make-fetch-happen';
import UserAgent from 'user-agents';
import { LRUCache } from 'lru-cache';
import { logInfo } from './log';
import { Context } from '../types';
export class Fetcher {
private readonly fetch: FetchInterface;
private readonly cache: LRUCache<string, string>;
constructor(fetch: FetchInterface) {
this.fetch = fetch;
this.cache = new LRUCache({ max: 128 });
}
readonly text = async (ctx: Context, uriOrRequest: string | Request, opts?: FetchOptions): Promise<string> => {
@ -18,6 +23,7 @@ export class Fetcher {
...opts,
headers: {
...opts?.headers,
'User-Agent': this.createUserAgentForIp(ctx.ip),
'X-Forwarded-For': ctx.ip,
},
},
@ -29,4 +35,17 @@ export class Fetcher {
return await response.text();
};
private readonly createUserAgentForIp = (ip: string): string => {
let userAgent = this.cache.get(ip);
if (userAgent) {
return userAgent;
}
userAgent = (new UserAgent()).toString();
this.cache.set(ip, userAgent);
return userAgent;
};
}