From 1ca4ea4abe03065dcd93382533d9cf5cb13dacdf Mon Sep 17 00:00:00 2001
From: WebStreamr <210764791+webstreamr@users.noreply.github.com>
Date: Mon, 9 Jun 2025 11:36:17 +0000
Subject: [PATCH] chore(extractor): don't use FlareSolverr for external URLs

---
 src/extractor/ExternalUrl.ts |  2 +-
 src/utils/Fetcher.ts         | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/extractor/ExternalUrl.ts b/src/extractor/ExternalUrl.ts
index b8f9d81..6fa1411 100644
--- a/src/extractor/ExternalUrl.ts
+++ b/src/extractor/ExternalUrl.ts
@@ -19,7 +19,7 @@ export class ExternalUrl implements Extractor {
 
   readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
     // We only want to make sure that the URL is accessible
-    await this.fetcher.head(ctx, url);
+    await this.fetcher.head(ctx, url, { noFlareSolverr: true });
 
     return [
       {
diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts
index 4dfa457..80ab825 100644
--- a/src/utils/Fetcher.ts
+++ b/src/utils/Fetcher.ts
@@ -41,6 +41,7 @@ interface FlareSolverrResult {
 }
 
 type CustomRequestInit = RequestInit & {
+  noFlareSolverr?: boolean;
   queueLimit?: number;
   queueErrorLimit?: number;
   timeout?: number;
@@ -95,7 +96,7 @@ export class Fetcher {
     };
   };
 
-  private readonly handleHttpCacheItem = async (ctx: Context, httpCacheItem: HttpCacheItem, url: URL): Promise<HttpCacheItem> => {
+  private readonly handleHttpCacheItem = async (ctx: Context, httpCacheItem: HttpCacheItem, url: URL, init?: CustomRequestInit): Promise<HttpCacheItem> => {
     if (httpCacheItem.status && httpCacheItem.status >= 200 && httpCacheItem.status <= 299) {
       return httpCacheItem;
     }
@@ -107,8 +108,9 @@ export class Fetcher {
     const responseHeaders = httpCacheItem.policy.responseHeaders();
 
     if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') {
+      const useFlareSolverr = init?.noFlareSolverr ?? true;
       const flareSolverrEndpoint = envGet('FLARESOLVERR_ENDPOINT');
-      if (!flareSolverrEndpoint) {
+      if (!useFlareSolverr || !flareSolverrEndpoint) {
         throw new BlockedError('cloudflare_challenge', httpCacheItem.policy.responseHeaders());
       }
 
@@ -183,7 +185,7 @@ export class Fetcher {
     let httpCacheItem: HttpCacheItem | undefined = this.httpCache.get(url.href);
     if (httpCacheItem) {
       this.logger.info(`Cached fetch ${request.method} ${url}`, ctx);
-      return this.handleHttpCacheItem(ctx, httpCacheItem, url);
+      return this.handleHttpCacheItem(ctx, httpCacheItem, url, init);
     }
 
     const response = await this.queuedFetch(ctx, url, newInit);
@@ -198,7 +200,7 @@ export class Fetcher {
       this.httpCache.set(url.href, httpCacheItem, { ttl });
     }
 
-    return this.handleHttpCacheItem(ctx, httpCacheItem, url);
+    return this.handleHttpCacheItem(ctx, httpCacheItem, url, init);
   };
 
   private readonly fetchWithTimeout = async (ctx: Context, url: URL, init?: CustomRequestInit): Promise<Response> => {