From 0638ce0a8c4791c781da49735d19ea864a5df13a Mon Sep 17 00:00:00 2001 From: WebStreamr <210764791+webstreamr@users.noreply.github.com> Date: Thu, 29 May 2025 17:17:17 +0000 Subject: [PATCH] fix: detect and report Cloudflare challenge blocks --- src/error/CloudflareChallengeError.ts | 1 + src/error/index.ts | 1 + src/extractor/ExtractorRegistry.ts | 16 +++++++++- src/types.ts | 3 +- src/utils/Fetcher.test.ts | 8 ++++- src/utils/Fetcher.ts | 13 ++++++-- src/utils/StreamResolver.test.ts | 30 ++++++++++++++++++- src/utils/StreamResolver.ts | 14 +++++++-- .../__snapshots__/StreamResolver.test.ts.snap | 18 ++++++++--- 9 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 src/error/CloudflareChallengeError.ts diff --git a/src/error/CloudflareChallengeError.ts b/src/error/CloudflareChallengeError.ts new file mode 100644 index 0000000..57cad82 --- /dev/null +++ b/src/error/CloudflareChallengeError.ts @@ -0,0 +1 @@ +export class CloudflareChallengeError extends Error {} diff --git a/src/error/index.ts b/src/error/index.ts index cb2a6ad..55ce3c7 100644 --- a/src/error/index.ts +++ b/src/error/index.ts @@ -1 +1,2 @@ +export * from './CloudflareChallengeError'; export * from './NotFoundError'; diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index bd5062e..91853d3 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -7,7 +7,7 @@ import { DoodStream } from './DoodStream'; import { Dropload } from './Dropload'; import { SuperVideo } from './SuperVideo'; import { ExternalUrl } from './ExternalUrl'; -import { NotFoundError } from '../error'; +import { CloudflareChallengeError, NotFoundError } from '../error'; export class ExtractorRegistry { private readonly logger: winston.Logger; @@ -43,6 +43,20 @@ export class ExtractorRegistry { return undefined; } + /* istanbul ignore next */ + if (error instanceof CloudflareChallengeError) { + this.logger.warn(`${extractor.id}: Request was blocked by Cloudflare challenge.`, ctx); + + return { + url: url, + isExternal: true, + cloudflareChallenge: true, + label: url.host, + sourceId: '', + meta, + }; + } + this.logger.warn(`${extractor.id} error: ${error}`, ctx); return undefined; } diff --git a/src/types.ts b/src/types.ts index eb30521..a905d3b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,8 +22,9 @@ export interface Meta { export interface UrlResult { url: URL; isExternal?: boolean; + cloudflareChallenge?: boolean; label: string; - sourceId: string; + sourceId?: string; meta: Meta; requestHeaders?: Record; } diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 6779031..a51fa81 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -3,7 +3,7 @@ import axios, { AxiosError } from 'axios'; import AxiosMockAdapter from 'axios-mock-adapter'; import { Fetcher } from './Fetcher'; import { Context } from '../types'; -import { NotFoundError } from '../error'; +import { CloudflareChallengeError, NotFoundError } from '../error'; const axiosMock = new AxiosMockAdapter(axios); @@ -125,6 +125,12 @@ describe('fetch', () => { await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(NotFoundError); }); + test('converts Cloudflare challenge block to custom CloudflareChallengeError', async () => { + axiosMock.onGet().reply(403, undefined, { 'cf-mitigated': 'challenge' }); + + await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(CloudflareChallengeError); + }); + test('passes through other errors', async () => { axiosMock.onGet().networkError(); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 1671e1d..106bcba 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -3,7 +3,7 @@ import TTLCache from '@isaacs/ttlcache'; import UserAgent from 'user-agents'; import winston from 'winston'; import { Context } from '../types'; -import { NotFoundError } from '../error'; +import { CloudflareChallengeError, NotFoundError } from '../error'; export class Fetcher { private readonly axios: AxiosInstance; @@ -69,9 +69,16 @@ export class Fetcher { try { return await callable(); } catch (error) { - if (isAxiosError(error) && error.response?.status === 404) { - throw new NotFoundError(); + if (isAxiosError(error)) { + if (error.response?.status === 404) { + throw new NotFoundError(); + } + + if (error.response?.headers['cf-mitigated'] === 'challenge') { + throw new CloudflareChallengeError(); + } } + throw error; } }; diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 7b09a93..a6a1980 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -3,7 +3,7 @@ import { ExtractorRegistry } from '../extractor'; import { StreamResolver } from './StreamResolver'; import { Handler, MeineCloud, MostraGuarda } from '../handler'; import { Fetcher } from './Fetcher'; -import { Context } from '../types'; +import { Context, CountryCode, UrlResult } from '../types'; import { NotFoundError } from '../error'; jest.mock('../utils/Fetcher'); @@ -57,6 +57,34 @@ describe('resolve', () => { expect(streams).toMatchSnapshot(); }); + test('adds Cloudflare blocked info', async () => { + class MockHandler implements Handler { + readonly id = 'mockhandler'; + + readonly label = 'MockHandler'; + + readonly contentTypes = ['movie']; + + readonly countryCodes: CountryCode[] = ['de']; + + readonly handle = async (): Promise<(UrlResult | undefined)[]> => { + return [{ + url: new URL('https://example.com'), + isExternal: true, + cloudflareChallenge: true, + label: 'example.com', + sourceId: '', + meta: { + countryCode: 'de', + }, + }]; + }; + } + + const streams = await streamResolver.resolve(ctx, [new MockHandler()], 'movie', 'tt11655566'); + expect(streams).toMatchSnapshot(); + }); + test('ignores not found errors', async () => { const mockHandler: Handler = { id: 'mockhandler', diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index 74cb34c..c5ed94d 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -85,14 +85,24 @@ export class StreamResolver { titleSecondLineEntries.push(`💾 ${bytes.format(urlResult.meta.bytes, { unitSeparator: ' ' })}`); } titleSecondLineEntries.push(`⚙️ ${urlResult.label}`); - const title = [urlResult.meta.title ?? '', titleSecondLineEntries.join(' ')].join('\n'); + + const titleLines = []; + if (urlResult.meta.title) { + titleLines.push(urlResult.meta.title); + } + titleLines.push(titleSecondLineEntries.join(' ')); + if (urlResult.cloudflareChallenge) { + titleLines.push('Request was blocked by Cloudflare.'); + } + + const title = titleLines.join('\n'); return { [urlResult.isExternal ? 'externalUrl' : 'url']: urlResult.url.href, name, title, behaviorHints: { - bingeGroup: `webstreamr-${urlResult.sourceId}`, + ...(urlResult.sourceId && { bingeGroup: `webstreamr-${urlResult.sourceId}` }), ...(urlResult.requestHeaders !== undefined && { notWebReady: true, proxyHeaders: { request: urlResult.requestHeaders }, diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index 53ea361..b86ad19 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -1,5 +1,17 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`resolve adds Cloudflare blocked info 1`] = ` +[ + { + "behaviorHints": {}, + "externalUrl": "https://example.com/", + "name": "WebStreamr 🇩🇪 external", + "title": "⚙️ example.com +Request was blocked by Cloudflare.", + }, +] +`; + exports[`resolve returns sorted results 1`] = ` [ { @@ -78,8 +90,7 @@ exports[`resolve returns sorted results 1`] = ` }, "externalUrl": "https://mixdrop.ag/e/3nzwveprim63or6", "name": "WebStreamr 🇩🇪 external", - "title": " -⚙️ mixdrop.ag", + "title": "⚙️ mixdrop.ag", }, { "behaviorHints": { @@ -87,8 +98,7 @@ exports[`resolve returns sorted results 1`] = ` }, "externalUrl": "https://mixdrop.ag/e/vk196d6xfzwwo1", "name": "WebStreamr 🇮🇹 external", - "title": " -⚙️ mixdrop.ag", + "title": "⚙️ mixdrop.ag", }, ] `;