From 2eb09fc65a1f0141abd1f9760c39cb454de00496 Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Sun, 1 Jun 2025 17:07:50 +0000 Subject: [PATCH] chore: generalize forbidden/blocked handling --- src/error/BlockedError.ts | 11 +++++++++ src/error/CloudflareChallengeError.ts | 1 - src/error/index.ts | 2 +- src/extractor/ExtractorRegistry.ts | 8 +++---- src/types.ts | 4 +++- src/utils/Fetcher.test.ts | 24 ++++++++++++++++--- src/utils/Fetcher.ts | 8 +++++-- src/utils/StreamResolver.test.ts | 2 +- src/utils/StreamResolver.ts | 4 ++-- .../__snapshots__/StreamResolver.test.ts.snap | 2 +- 10 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 src/error/BlockedError.ts delete mode 100644 src/error/CloudflareChallengeError.ts diff --git a/src/error/BlockedError.ts b/src/error/BlockedError.ts new file mode 100644 index 0000000..efb0eed --- /dev/null +++ b/src/error/BlockedError.ts @@ -0,0 +1,11 @@ +import { BlockedReason } from '../types'; + +export class BlockedError extends Error { + public readonly reason: BlockedReason; + + constructor(reason: BlockedReason) { + super(); + + this.reason = reason; + } +} diff --git a/src/error/CloudflareChallengeError.ts b/src/error/CloudflareChallengeError.ts deleted file mode 100644 index 57cad82..0000000 --- a/src/error/CloudflareChallengeError.ts +++ /dev/null @@ -1 +0,0 @@ -export class CloudflareChallengeError extends Error {} diff --git a/src/error/index.ts b/src/error/index.ts index 55ce3c7..e6cccbd 100644 --- a/src/error/index.ts +++ b/src/error/index.ts @@ -1,2 +1,2 @@ -export * from './CloudflareChallengeError'; +export * from './BlockedError'; export * from './NotFoundError'; diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index ab3bc13..c72909b 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -7,7 +7,7 @@ import { DoodStream } from './DoodStream'; import { Dropload } from './Dropload'; import { SuperVideo } from './SuperVideo'; import { ExternalUrl } from './ExternalUrl'; -import { CloudflareChallengeError, NotFoundError } from '../error'; +import { BlockedError, NotFoundError } from '../error'; export class ExtractorRegistry { private readonly logger: winston.Logger; @@ -48,13 +48,13 @@ export class ExtractorRegistry { } /* istanbul ignore next */ - if (error instanceof CloudflareChallengeError && !('excludeExternalUrls' in ctx.config)) { - this.logger.warn(`${extractor.id}: Request was blocked by Cloudflare challenge.`, ctx); + if (error instanceof BlockedError && !('excludeExternalUrls' in ctx.config)) { + this.logger.warn(`${extractor.id}: Request was blocked. Reason: ${error.reason}`, ctx); return { url: url, isExternal: true, - cloudflareChallenge: true, + blocked: error.reason, label: url.host, sourceId: '', meta, diff --git a/src/types.ts b/src/types.ts index 1dd3683..e29581f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,6 +13,8 @@ export type Config = Partial export type CountryCode = 'de' | 'en' | 'es' | 'fr' | 'it' | 'mx'; +export type BlockedReason = 'cloudflare_challenge' | 'unknown'; + export interface Meta { bytes?: number; countryCode: CountryCode; @@ -23,7 +25,7 @@ export interface Meta { export interface UrlResult { url: URL; isExternal?: boolean; - cloudflareChallenge?: boolean; + blocked?: BlockedReason; label: string; sourceId?: string; meta: Meta; diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index efcb254..15d401c 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -2,7 +2,7 @@ import winston from 'winston'; import fetchMock from 'fetch-mock'; import { Fetcher } from './Fetcher'; import { Context } from '../types'; -import { CloudflareChallengeError, NotFoundError } from '../error'; +import { BlockedError, NotFoundError } from '../error'; fetchMock.mockGlobal(); const fetcher = new Fetcher(winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] })); @@ -68,10 +68,28 @@ describe('fetch', () => { await expect(fetcher.text(ctx, new URL('https://some-404-url.test/'))).rejects.toBeInstanceOf(NotFoundError); }); - test('converts Cloudflare challenge block to custom CloudflareChallengeError', async () => { + test('converts Cloudflare challenge block to custom BlockedError', async () => { fetchMock.get('https://some-cloudflare-url.test/', { status: 403, headers: { 'cf-mitigated': 'challenge' } }); - await expect(fetcher.text(ctx, new URL('https://some-cloudflare-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError); + try { + await fetcher.text(ctx, new URL('https://some-cloudflare-url.test/')); + fail(); + } catch (error) { + expect(error).toBeInstanceOf(BlockedError); + expect(error).toMatchObject({ reason: 'cloudflare_challenge' }); + } + }); + + test('converts generic forbidden to custom BlockedError', async () => { + fetchMock.get('https://some-forbidden-url.test/', { status: 403 }); + + try { + await fetcher.text(ctx, new URL('https://some-forbidden-url.test/')); + fail(); + } catch (error) { + expect(error).toBeInstanceOf(BlockedError); + expect(error).toMatchObject({ reason: 'unknown' }); + } }); test('passes through other errors with detailed infos', async () => { diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 6eb49a0..2fca445 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -2,7 +2,7 @@ import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; import { Context } from '../types'; -import { CloudflareChallengeError, NotFoundError } from '../error'; +import { BlockedError, NotFoundError } from '../error'; import { clearTimeout } from 'node:timers'; interface HttpCacheItem { policy: CachePolicy; status: number; statusText: string; body: string } @@ -56,7 +56,11 @@ export class Fetcher { const responseHeaders = httpCacheItem.policy.responseHeaders(); if (httpCacheItem.policy.responseHeaders()['cf-mitigated'] === 'challenge') { - throw new CloudflareChallengeError(); + throw new BlockedError('cloudflare_challenge'); + } + + if (httpCacheItem.status === 403) { + throw new BlockedError('unknown'); } throw new Error(`Fetcher error: ${httpCacheItem.status}: ${httpCacheItem.statusText}, response headers: ${JSON.stringify(responseHeaders)}`); diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index a6a1980..3db011f 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -71,7 +71,7 @@ describe('resolve', () => { return [{ url: new URL('https://example.com'), isExternal: true, - cloudflareChallenge: true, + blocked: 'cloudflare_challenge', label: 'example.com', sourceId: '', meta: { diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index c5ed94d..2dc30d9 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -91,8 +91,8 @@ export class StreamResolver { titleLines.push(urlResult.meta.title); } titleLines.push(titleSecondLineEntries.join(' ')); - if (urlResult.cloudflareChallenge) { - titleLines.push('Request was blocked by Cloudflare.'); + if (urlResult.blocked) { + titleLines.push('Request was blocked.'); } const title = titleLines.join('\n'); diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index b86ad19..2753b44 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -7,7 +7,7 @@ exports[`resolve adds Cloudflare blocked info 1`] = ` "externalUrl": "https://example.com/", "name": "WebStreamr 🇩🇪 external", "title": "⚙️ example.com -Request was blocked by Cloudflare.", +Request was blocked.", }, ] `;