chore(extractor): do not make any requests for external URLs

This commit is contained in:
WebStreamr 2025-10-24 14:09:12 +00:00
parent 56e72b73fe
commit 77164dfaa7
No known key found for this signature in database
2 changed files with 16 additions and 15 deletions

View file

@ -1,4 +1,3 @@
import { BlockedError } from '../error';
import { Context, Format, Meta, UrlResult } from '../types';
import { showExternalUrls } from '../utils';
import { Extractor } from './Extractor';
@ -14,19 +13,7 @@ export class ExternalUrl extends Extractor {
return showExternalUrls(ctx.config) && null !== url.host.match(/.*/);
}
protected async extractInternal(ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> {
const headers = { Referer: meta.referer ?? url.href };
try {
// Make sure the URL is accessible, but avoid causing noise and delays doing this
await this.fetcher.head(ctx, url, { noFlareSolverr: true, timeout: 1000, headers });
} catch (error) {
/* istanbul ignore if */
if (!(error instanceof BlockedError)) {
return [];
}
}
protected async extractInternal(_ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> {
return [
{
url: url,

View file

@ -1,6 +1,20 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
exports[`ExternalUrl 404 - not found 1`] = `[]`;
exports[`ExternalUrl 404 - not found 1`] = `
[
{
"format": "unknown",
"isExternal": true,
"label": "streamtape.com",
"meta": {
"countryCodes": [],
},
"sourceId": "external_",
"ttl": 21600000,
"url": "https://streamtape.com/e/gjA1OQ4klyHxgJ",
},
]
`;
exports[`ExternalUrl netu.fanstream.us 1`] = `
[