diff --git a/src/extractor/ExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts index e1c3818..0bd187d 100644 --- a/src/extractor/ExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -12,10 +12,10 @@ const extractorRegistry = new ExtractorRegistry(logger, fetcher); describe('ExtractorRegistry', () => { const ctx: Context = { id: 'id', ip: '127.0.0.1', config: { de: 'on' } }; - test('returns undefined when no extractor can be found', async () => { + test('returns error result from extractor', async () => { const urlResult = await extractorRegistry.handle(ctx, new URL('https://some-url.test'), { countryCode: 'en' }); - expect(urlResult).toBeUndefined(); + expect(urlResult).toMatchSnapshot(); }); test('return external URLs by default', async () => { diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index d83c92c..b9291a7 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -7,7 +7,7 @@ import { DoodStream } from './DoodStream'; import { Dropload } from './Dropload'; import { SuperVideo } from './SuperVideo'; import { ExternalUrl } from './ExternalUrl'; -import { BlockedError, NotFoundError } from '../error'; +import { NotFoundError } from '../error'; export class ExtractorRegistry { private readonly logger: winston.Logger; @@ -47,24 +47,14 @@ export class ExtractorRegistry { return undefined; } - /* istanbul ignore next */ - if (error instanceof BlockedError && !('excludeExternalUrls' in ctx.config)) { - this.logger.warn(`${extractor.id}: Request was blocked. Reason: ${error.reason}`, ctx); - - return { - url: url, - isExternal: true, - blocked: error.reason, - label: url.host, - sourceId: '', - meta, - }; - } - - const cause = (error as Error & { cause?: unknown }).cause; - - this.logger.warn(`${extractor.id} error: ${error}, cause: ${cause}`, ctx); - return undefined; + return { + url, + isExternal: true, + error, + label: url.host, + sourceId: `${extractor.id}`, + meta, + }; } this.urlResultCache.set(url.href, urlResult, { ttl: extractor.ttl }); diff --git a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap index 3e656f6..e4cda16 100644 --- a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap +++ b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap @@ -11,3 +11,16 @@ exports[`ExtractorRegistry return external URLs by default 1`] = ` "url": "https://mixdrop.ag/e/3nzwveprim63or6", } `; + +exports[`ExtractorRegistry returns error result from extractor 1`] = ` +{ + "error": [Error: TypeError: fetch failed], + "isExternal": true, + "label": "some-url.test", + "meta": { + "countryCode": "en", + }, + "sourceId": "external", + "url": "https://some-url.test/", +} +`; diff --git a/src/handler/__snapshots__/Frembed.test.ts.snap b/src/handler/__snapshots__/Frembed.test.ts.snap index 3b8ddf1..5d40009 100644 --- a/src/handler/__snapshots__/Frembed.test.ts.snap +++ b/src/handler/__snapshots__/Frembed.test.ts.snap @@ -2,6 +2,17 @@ exports[`Frembed handle imdb black mirror s4e2 1`] = ` [ + { + "error": [Error: TypeError: fetch failed], + "isExternal": true, + "label": "ahvsh.com", + "meta": { + "countryCode": "fr", + "title": "Black Mirror 4x2", + }, + "sourceId": "external", + "url": "https://ahvsh.com/e/83izf7qzwpae,https://netu.frembed.fun/e/0DFgfkcXOsDP,https://likessb.com/e/7yjgl1x56n08.html,https://ds2play.com/e/fzfvfq3ngig0", + }, { "label": "DoodStream", "meta": { @@ -34,5 +45,16 @@ exports[`Frembed handle imdb black mirror s4e2 1`] = ` "sourceId": "external_fr", "url": "https://johnalwayssame.com/e/cqy9oue7sv0g", }, + { + "error": [Error: TypeError: fetch failed], + "isExternal": true, + "label": "ds2play.com", + "meta": { + "countryCode": "fr", + "title": "Black Mirror 4x2", + }, + "sourceId": "external", + "url": "https://ds2play.com/e/fzfvfq3ngig0", + }, ] `; diff --git a/src/handler/__snapshots__/FrenchCloud.test.ts.snap b/src/handler/__snapshots__/FrenchCloud.test.ts.snap index f82f089..d72bc76 100644 --- a/src/handler/__snapshots__/FrenchCloud.test.ts.snap +++ b/src/handler/__snapshots__/FrenchCloud.test.ts.snap @@ -33,6 +33,16 @@ exports[`FrenchCloud handle imdb the devil's bath 1`] = ` "sourceId": "external_fr", "url": "https://mixdrop.ag/e/l7v73zqrfdj19z", }, + { + "error": [Error: Fetcher error: 404: Not Found], + "isExternal": true, + "label": "streamtape.com", + "meta": { + "countryCode": "fr", + }, + "sourceId": "external", + "url": "https://streamtape.com/e/gjA1OQ4klyHxgJ", + }, { "label": "DoodStream", "meta": { diff --git a/src/handler/__snapshots__/VerHdLink.test.ts.snap b/src/handler/__snapshots__/VerHdLink.test.ts.snap index 42ce3d8..3e1571d 100644 --- a/src/handler/__snapshots__/VerHdLink.test.ts.snap +++ b/src/handler/__snapshots__/VerHdLink.test.ts.snap @@ -33,6 +33,16 @@ exports[`VerHdLink handle titanic 1`] = ` "sourceId": "external_mx", "url": "https://mixdrop.ag/e/vn0wx308fq984q", }, + { + "error": [Error: Fetcher error: 404: Not Found], + "isExternal": true, + "label": "streamtape.com", + "meta": { + "countryCode": "mx", + }, + "sourceId": "external", + "url": "https://streamtape.com/e/Bjp2vjrdBxsK82", + }, { "label": "SuperVideo", "meta": { diff --git a/src/types.ts b/src/types.ts index e29581f..190b617 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,7 @@ import { Manifest, ManifestConfig } from 'stremio-addon-sdk'; +export const TIMEOUT = 'timeout'; + export interface Context { id: string; ip: string; @@ -25,9 +27,9 @@ export interface Meta { export interface UrlResult { url: URL; isExternal?: boolean; - blocked?: BlockedReason; + error?: unknown; label: string; - sourceId?: string; + sourceId: string; meta: Meta; requestHeaders?: Record; } diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 725a08f..f87ba76 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,7 +1,7 @@ import CachePolicy from 'http-cache-semantics'; import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; -import { Context } from '../types'; +import { Context, TIMEOUT } from '../types'; import { BlockedError, NotFoundError } from '../error'; import { clearTimeout } from 'node:timers'; @@ -90,7 +90,7 @@ export class Fetcher { this.logger.info(`Fetch ${request.method} ${url}`, ctx); const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 10000); + const timer = setTimeout(() => controller.abort(TIMEOUT), 10000); let response; try { diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 3db011f..d95c5ea 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -3,8 +3,8 @@ import { ExtractorRegistry } from '../extractor'; import { StreamResolver } from './StreamResolver'; import { Handler, MeineCloud, MostraGuarda } from '../handler'; import { Fetcher } from './Fetcher'; -import { Context, CountryCode, UrlResult } from '../types'; -import { NotFoundError } from '../error'; +import { Context, CountryCode, TIMEOUT, UrlResult } from '../types'; +import { BlockedError, NotFoundError } from '../error'; jest.mock('../utils/Fetcher'); const logger = winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }); @@ -35,7 +35,7 @@ describe('resolve', () => { expect(streams).toStrictEqual([{ name: 'WebStreamr', - title: 'āŒ Error with handler "meinecloud". Please create an issue if this persists. Request-id: id', + title: 'šŸ”— MeineCloud\nāŒ Request failed. Request-id: id.', ytId: 'E4WlUXrJgy4', }]); @@ -57,7 +57,7 @@ describe('resolve', () => { expect(streams).toMatchSnapshot(); }); - test('adds Cloudflare blocked info', async () => { + test('adds error info', async () => { class MockHandler implements Handler { readonly id = 'mockhandler'; @@ -68,21 +68,46 @@ describe('resolve', () => { readonly countryCodes: CountryCode[] = ['de']; readonly handle = async (): Promise<(UrlResult | undefined)[]> => { - return [{ - url: new URL('https://example.com'), - isExternal: true, - blocked: 'cloudflare_challenge', - label: 'example.com', - sourceId: '', - meta: { - countryCode: 'de', + return [ + { + url: new URL('https://example.com'), + isExternal: true, + error: new BlockedError('cloudflare_challenge'), + label: 'hoster.com', + sourceId: '', + meta: { + countryCode: 'de', + }, }, - }]; + { + url: new URL('https://example2.com'), + isExternal: true, + error: new TypeError(), + label: 'hoster.com', + sourceId: '', + meta: { + countryCode: 'de', + }, + }, + { + url: new URL('https://example2.com'), + isExternal: true, + error: TIMEOUT, + label: 'hoster.com', + sourceId: '', + meta: { + countryCode: 'de', + }, + }, + ]; }; } const streams = await streamResolver.resolve(ctx, [new MockHandler()], 'movie', 'tt11655566'); expect(streams).toMatchSnapshot(); + + const streamsWithoutExternalUrls = await streamResolver.resolve({ ...ctx, config: { ...ctx.config, excludeExternalUrls: 'on' } }, [new MockHandler()], 'movie', 'tt11655566'); + expect(streamsWithoutExternalUrls).toMatchSnapshot(); }); test('ignores not found errors', async () => { diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index b4149f5..c73fedf 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -2,7 +2,7 @@ import { Stream } from 'stremio-addon-sdk'; import { flag } from 'country-emoji'; import winston from 'winston'; import bytes from 'bytes'; -import { Context, UrlResult } from '../types'; +import { Context, TIMEOUT, UrlResult } from '../types'; import { Handler } from '../handler'; import { BlockedError, NotFoundError } from '../error'; import { languageFromCountryCode } from './languageFromCountryCode'; @@ -41,28 +41,11 @@ export class StreamResolver { return; } - /* istanbul ignore next */ - if (error instanceof BlockedError) { - this.logger.warn(`${handler.id}: Request was blocked. Reason: ${error.reason}`, ctx); - - streams.push({ - name: process.env['MANIFEST_NAME'] || 'WebStreamr', - title: `āš ļø Request was blocked for handler "${handler.id}".`, - ytId: 'E4WlUXrJgy4', - }); - - return; - } - streams.push({ name: process.env['MANIFEST_NAME'] || 'WebStreamr', - title: `āŒ Error with handler "${handler.id}". Please create an issue if this persists. Request-id: ${ctx.id}`, + title: [`šŸ”— ${handler.label}`, this.logErrorAndReturnNiceString(ctx, handler.id, error)].join('\n'), ytId: 'E4WlUXrJgy4', }); - - const cause = (error as Error & { cause?: unknown }).cause; - - this.logger.error(`${handler.id} error: ${error}, cause: ${cause}`, ctx); } }); await Promise.all(handlerPromises); @@ -84,37 +67,71 @@ export class StreamResolver { this.logger.info(`Return ${urlResults.length} streams`, ctx); streams.push( - ...urlResults.map(urlResult => ({ - [urlResult.isExternal ? 'externalUrl' : 'url']: urlResult.url.href, - name: this.buildName(urlResult), - title: this.buildTitle(urlResult), - behaviorHints: { - ...(urlResult.sourceId && { bingeGroup: `webstreamr-${urlResult.sourceId}` }), - ...(urlResult.requestHeaders !== undefined && { - notWebReady: true, - proxyHeaders: { request: urlResult.requestHeaders }, - }), - ...(urlResult.meta.bytes && { videoSize: urlResult.meta.bytes }), - }, - })), + ...urlResults.filter(urlResult => !urlResult.isExternal || this.showExternalUrls(ctx) || urlResult.error) + .map(urlResult => ({ + ...this.buildUrl(ctx, urlResult), + name: this.buildName(ctx, urlResult), + title: this.buildTitle(ctx, urlResult), + behaviorHints: { + ...(urlResult.sourceId && { bingeGroup: `webstreamr-${urlResult.sourceId}` }), + ...(urlResult.requestHeaders !== undefined && { + notWebReady: true, + proxyHeaders: { request: urlResult.requestHeaders }, + }), + ...(urlResult.meta.bytes && { videoSize: urlResult.meta.bytes }), + }, + })), ); return streams; }; - private readonly buildName = (urlResult: UrlResult): string => { + private readonly showExternalUrls = (ctx: Context): boolean => !('excludeExternalUrls' in ctx.config); + + private readonly buildUrl = (ctx: Context, urlResult: UrlResult): { externalUrl: string } | { url: string } | { ytId: string } => { + if (!urlResult.isExternal) { + return { url: urlResult.url.href }; + } + + if (this.showExternalUrls(ctx)) { + return { externalUrl: urlResult.url.href }; + } + + return { ytId: 'E4WlUXrJgy4' }; + }; + + private readonly buildName = (ctx: Context, urlResult: UrlResult): string => { let name = process.env['MANIFEST_NAME'] || 'WebStreamr'; name += urlResult.meta.height ? ` ${urlResult.meta.height}P` : ' N/A'; - if (urlResult.isExternal) { + if (urlResult.isExternal && this.showExternalUrls(ctx)) { name += ` āš ļø external`; } return name; }; - private readonly buildTitle = (urlResult: UrlResult): string => { + private readonly logErrorAndReturnNiceString = (ctx: Context, source: string, error: unknown): string => { + if (error instanceof BlockedError) { + this.logger.warn(`${source}: Request was blocked. Reason: ${error.reason}`, ctx); + + return 'āš ļø Request was blocked.'; + } + + if (error === TIMEOUT) { + this.logger.warn(`${source}: Request timed out.`, ctx); + + return '🐢 Request timed out.'; + } + + const cause = (error as Error & { cause?: unknown }).cause; + this.logger.error(`${source} error: ${error}, cause: ${cause}`, ctx); + + return `āŒ Request failed. Request-id: ${ctx.id}.`; + }; + + private readonly buildTitle = (ctx: Context, urlResult: UrlResult): string => { const titleLines = []; if (urlResult.meta.title) { @@ -131,8 +148,8 @@ export class StreamResolver { titleLines.push(`šŸ”— ${urlResult.label}`); - if (urlResult.blocked) { - titleLines.push('āš ļø Request was blocked.'); + if (urlResult.error) { + titleLines.push(this.logErrorAndReturnNiceString(ctx, urlResult.sourceId, urlResult.error)); } return titleLines.join('\n'); diff --git a/src/utils/__snapshots__/StreamResolver.test.ts.snap b/src/utils/__snapshots__/StreamResolver.test.ts.snap index f6ad097..f0a580f 100644 --- a/src/utils/__snapshots__/StreamResolver.test.ts.snap +++ b/src/utils/__snapshots__/StreamResolver.test.ts.snap @@ -1,15 +1,60 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`resolve adds Cloudflare blocked info 1`] = ` +exports[`resolve adds error info 1`] = ` [ { "behaviorHints": {}, "externalUrl": "https://example.com/", "name": "WebStreamr N/A āš ļø external", "title": "🌐 German šŸ‡©šŸ‡Ŗ -šŸ”— example.com +šŸ”— hoster.com āš ļø Request was blocked.", }, + { + "behaviorHints": {}, + "externalUrl": "https://example2.com/", + "name": "WebStreamr N/A āš ļø external", + "title": "🌐 German šŸ‡©šŸ‡Ŗ +šŸ”— hoster.com +āŒ Request failed. Request-id: id.", + }, + { + "behaviorHints": {}, + "externalUrl": "https://example2.com/", + "name": "WebStreamr N/A āš ļø external", + "title": "🌐 German šŸ‡©šŸ‡Ŗ +šŸ”— hoster.com +🐢 Request timed out.", + }, +] +`; + +exports[`resolve adds error info 2`] = ` +[ + { + "behaviorHints": {}, + "name": "WebStreamr N/A", + "title": "🌐 German šŸ‡©šŸ‡Ŗ +šŸ”— hoster.com +āš ļø Request was blocked.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr N/A", + "title": "🌐 German šŸ‡©šŸ‡Ŗ +šŸ”— hoster.com +āŒ Request failed. Request-id: id.", + "ytId": "E4WlUXrJgy4", + }, + { + "behaviorHints": {}, + "name": "WebStreamr N/A", + "title": "🌐 German šŸ‡©šŸ‡Ŗ +šŸ”— hoster.com +🐢 Request timed out.", + "ytId": "E4WlUXrJgy4", + }, ] `;