chore(extractor): do not make any requests for external URLs
This commit is contained in:
parent
56e72b73fe
commit
77164dfaa7
2 changed files with 16 additions and 15 deletions
|
|
@ -1,4 +1,3 @@
|
|||
import { BlockedError } from '../error';
|
||||
import { Context, Format, Meta, UrlResult } from '../types';
|
||||
import { showExternalUrls } from '../utils';
|
||||
import { Extractor } from './Extractor';
|
||||
|
|
@ -14,19 +13,7 @@ export class ExternalUrl extends Extractor {
|
|||
return showExternalUrls(ctx.config) && null !== url.host.match(/.*/);
|
||||
}
|
||||
|
||||
protected async extractInternal(ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> {
|
||||
const headers = { Referer: meta.referer ?? url.href };
|
||||
|
||||
try {
|
||||
// Make sure the URL is accessible, but avoid causing noise and delays doing this
|
||||
await this.fetcher.head(ctx, url, { noFlareSolverr: true, timeout: 1000, headers });
|
||||
} catch (error) {
|
||||
/* istanbul ignore if */
|
||||
if (!(error instanceof BlockedError)) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
protected async extractInternal(_ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> {
|
||||
return [
|
||||
{
|
||||
url: url,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,20 @@
|
|||
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
|
||||
|
||||
exports[`ExternalUrl 404 - not found 1`] = `[]`;
|
||||
exports[`ExternalUrl 404 - not found 1`] = `
|
||||
[
|
||||
{
|
||||
"format": "unknown",
|
||||
"isExternal": true,
|
||||
"label": "streamtape.com",
|
||||
"meta": {
|
||||
"countryCodes": [],
|
||||
},
|
||||
"sourceId": "external_",
|
||||
"ttl": 21600000,
|
||||
"url": "https://streamtape.com/e/gjA1OQ4klyHxgJ",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`ExternalUrl netu.fanstream.us 1`] = `
|
||||
[
|
||||
|
|
|
|||
Loading…
Reference in a new issue