chore: generalize forbidden/blocked handling

src/error/BlockedError.ts

@@ -0,0 +1,11 @@
+import { BlockedReason } from '../types';
+
+export class BlockedError extends Error {
+  public readonly reason: BlockedReason;
+
+  constructor(reason: BlockedReason) {
+    super();
+
+    this.reason = reason;
+  }
+}

src/error/CloudflareChallengeError.ts

@@ -1 +0,0 @@
-export class CloudflareChallengeError extends Error {}

src/error/index.ts

@@ -1,2 +1,2 @@
-export * from './CloudflareChallengeError';
+export * from './BlockedError';
 export * from './NotFoundError';

src/extractor/ExtractorRegistry.ts

@@ -7,7 +7,7 @@ import { DoodStream } from './DoodStream';
 import { Dropload } from './Dropload';
 import { SuperVideo } from './SuperVideo';
 import { ExternalUrl } from './ExternalUrl';
-import { CloudflareChallengeError, NotFoundError } from '../error';
+import { BlockedError, NotFoundError } from '../error';
 
 export class ExtractorRegistry {
   private readonly logger: winston.Logger;
@@ -48,13 +48,13 @@ export class ExtractorRegistry {
       }
 
       /* istanbul ignore next */
-      if (error instanceof CloudflareChallengeError && !('excludeExternalUrls' in ctx.config)) {
-        this.logger.warn(`${extractor.id}: Request was blocked by Cloudflare challenge.`, ctx);
+      if (error instanceof BlockedError && !('excludeExternalUrls' in ctx.config)) {
+        this.logger.warn(`${extractor.id}: Request was blocked. Reason: ${error.reason}`, ctx);
 
         return {
           url: url,
           isExternal: true,
-          cloudflareChallenge: true,
+          blocked: error.reason,
           label: url.host,
           sourceId: '',
           meta,

src/types.ts

@@ -13,6 +13,8 @@ export type Config = Partial<Record<CountryCode | 'excludeExternalUrls', string>
 
 export type CountryCode = 'de' | 'en' | 'es' | 'fr' | 'it' | 'mx';
 
+export type BlockedReason = 'cloudflare_challenge' | 'unknown';
+
 export interface Meta {
   bytes?: number;
   countryCode: CountryCode;
@@ -23,7 +25,7 @@ export interface Meta {
 export interface UrlResult {
   url: URL;
   isExternal?: boolean;
-  cloudflareChallenge?: boolean;
+  blocked?: BlockedReason;
   label: string;
   sourceId?: string;
   meta: Meta;

src/utils/Fetcher.test.ts

@@ -2,7 +2,7 @@ import winston from 'winston';
 import fetchMock from 'fetch-mock';
 import { Fetcher } from './Fetcher';
 import { Context } from '../types';
-import { CloudflareChallengeError, NotFoundError } from '../error';
+import { BlockedError, NotFoundError } from '../error';
 fetchMock.mockGlobal();
 
 const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }));
@@ -68,10 +68,28 @@ describe('fetch', () => {
     await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError);
   });
 
-  test('converts Cloudflare challenge block to custom CloudflareChallengeError', async () => {
+  test('converts Cloudflare challenge block to custom BlockedError', async () => {
     fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } });
 
-    await expect(fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError);
+    try {
+      await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'));
+      fail();
+    } catch (error) {
+      expect(error).toBeInstanceOf(BlockedError);
+      expect(error).toMatchObject({ reason: 'cloudflare_challenge' });
+    }
+  });
+
+  test('converts generic forbidden to custom BlockedError', async () => {
+    fetchMock.get('https://some-forbidden-url.test/', { status: 403 });
+
+    try {
+      await fetcher.text(ctx, new URL('https://some-forbidden-url.test/'));
+      fail();
+    } catch (error) {
+      expect(error).toBeInstanceOf(BlockedError);
+      expect(error).toMatchObject({ reason: 'unknown' });
+    }
   });
 
   test('passes through other errors with detailed infos', async () => {

src/utils/Fetcher.ts

@@ -2,7 +2,7 @@ import CachePolicy from 'http-cache-semantics';
 import TTLCache from '@isaacs/ttlcache';
 import winston from 'winston';
 import { Context } from '../types';
-import { CloudflareChallengeError, NotFoundError } from '../error';
+import { BlockedError, NotFoundError } from '../error';
 import { clearTimeout } from 'node:timers';
 
 interface HttpCacheItem { policy: CachePolicy; status: number; statusText: string; body: string }
@@ -56,7 +56,11 @@ export class Fetcher {
     const responseHeaders = httpCacheItem.policy.responseHeaders();
 
     if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') {
-      throw new CloudflareChallengeError();
+      throw new BlockedError('cloudflare_challenge');
+    }
+
+    if (httpCacheItem.status === 403) {
+      throw new BlockedError('unknown');
     }
 
     throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}, response headers: ${JSON.stringify(responseHeaders)}`);

src/utils/StreamResolver.test.ts

@@ -71,7 +71,7 @@ describe('resolve', () => {
         return [{
           url: new URL('https://example.com'),
           isExternal: true,
-          cloudflareChallenge: true,
+          blocked: 'cloudflare_challenge',
           label: 'example.com',
           sourceId: '',
           meta: {

src/utils/StreamResolver.ts

@@ -91,8 +91,8 @@ export class StreamResolver {
           titleLines.push(urlResult.meta.title);
         }
         titleLines.push(titleSecondLineEntries.join(' '));
-        if (urlResult.cloudflareChallenge) {
-          titleLines.push('Request was blocked by Cloudflare.');
+        if (urlResult.blocked) {
+          titleLines.push('Request was blocked.');
         }
 
         const title = titleLines.join('\n');

src/utils/__snapshots__/StreamResolver.test.ts.snap

@@ -7,7 +7,7 @@ exports[`resolve adds Cloudflare blocked info 1`] = `
     "externalUrl": "https://example.com/",
     "name": "WebStreamr 🇩🇪 external",
     "title": "⚙️ example.com
-Request was blocked by Cloudflare.",
+Request was blocked.",
   },
 ]
 `;