diff --git a/src/error/NotFoundError.ts b/src/error/NotFoundError.ts new file mode 100644 index 0000000..0e1f90f --- /dev/null +++ b/src/error/NotFoundError.ts @@ -0,0 +1 @@ +export class NotFoundError extends Error {} diff --git a/src/error/index.ts b/src/error/index.ts new file mode 100644 index 0000000..cb2a6ad --- /dev/null +++ b/src/error/index.ts @@ -0,0 +1 @@ +export * from './NotFoundError'; diff --git a/src/extractor/Dropload.ts b/src/extractor/Dropload.ts index a98a71a..9d6bdb4 100644 --- a/src/extractor/Dropload.ts +++ b/src/extractor/Dropload.ts @@ -1,7 +1,8 @@ +import bytes from 'bytes'; import { Extractor } from './types'; import { extractUrlFromPacked, Fetcher } from '../utils'; import { Context } from '../types'; -import bytes from 'bytes'; +import { NotFoundError } from '../error'; export class Dropload implements Extractor { readonly id = 'dropload'; @@ -22,6 +23,10 @@ export class Dropload implements Extractor { url.pathname = url.pathname.replace('/e/', '').replace('/embed-', '/'); const html = await this.fetcher.text(ctx, url); + if (html.includes('File Not Found')) { + throw new NotFoundError(); + } + const heightMatch = html.match(/\d{3,}x(\d{3,}),/) as string[]; const sizeMatch = html.match(/([\d.]+ ?[GM]B)/) as string[]; diff --git a/src/extractor/ExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts index 1f5d597..6bf5fcd 100644 --- a/src/extractor/ExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -24,4 +24,17 @@ describe('ExtractorRegistry', () => { expect(urlResult2).toBe(urlResult1); }); + + test('returns from memory cache if possible', async () => { + const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); + const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de'); + + expect(urlResult2).toBe(urlResult1); + }); + + test('ignores not found errors', async () => { + const urlResult = await extractorRegistry.handle(ctx, new URL('https://dropload.io/asdfghijklmn.html'), 'de'); + + expect(urlResult).toBeUndefined(); + }); }); diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 24eb579..6386c34 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -7,6 +7,7 @@ import { DoodStream } from './DoodStream'; import { Dropload } from './Dropload'; import { SuperVideo } from './SuperVideo'; import { ExternalUrl } from './ExternalUrl'; +import { NotFoundError } from '../error'; export class ExtractorRegistry { private readonly logger: winston.Logger; @@ -36,6 +37,10 @@ export class ExtractorRegistry { try { urlResult = await extractor.extract(ctx, url, countryCode); } catch (error) { + if (error instanceof NotFoundError) { + return undefined; + } + this.logger.warn(`${extractor.id} error: ${error}`, ctx); return undefined; } diff --git a/src/utils/Fetcher.test.ts b/src/utils/Fetcher.test.ts index 061fc68..b4fcf90 100644 --- a/src/utils/Fetcher.test.ts +++ b/src/utils/Fetcher.test.ts @@ -1,8 +1,9 @@ import winston from 'winston'; -import axios from 'axios'; +import axios, { AxiosError } from 'axios'; import AxiosMockAdapter from 'axios-mock-adapter'; import { Fetcher } from './Fetcher'; import { Context } from '../types'; +import { NotFoundError } from '../error'; const axiosMock = new AxiosMockAdapter(axios); @@ -97,4 +98,16 @@ describe('fetch', () => { }, ); }); + + test('converts 404 to custom NotFoundError', async () => { + axiosMock.onGet().reply(404); + + await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(NotFoundError); + }); + + test('passes through other errors', async () => { + axiosMock.onGet().networkError(); + + await expect(fetcher.text(ctx, new URL('https://some-url.test/'))).rejects.toBeInstanceOf(AxiosError); + }); }); diff --git a/src/utils/Fetcher.ts b/src/utils/Fetcher.ts index 93444fe..96ea129 100644 --- a/src/utils/Fetcher.ts +++ b/src/utils/Fetcher.ts @@ -1,8 +1,9 @@ -import { AxiosInstance, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios'; +import { AxiosInstance, AxiosRequestConfig, AxiosResponseHeaders, RawAxiosResponseHeaders, isAxiosError } from 'axios'; import TTLCache from '@isaacs/ttlcache'; import UserAgent from 'user-agents'; import winston from 'winston'; import { Context } from '../types'; +import { NotFoundError } from '../error'; export class Fetcher { private readonly axios: AxiosInstance; @@ -18,25 +19,19 @@ export class Fetcher { readonly text = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { this.logger.info(`Fetch ${url}`, ctx); - const response = await this.axios.get(url.href, this.getConfig(ctx, url, config)); - - return response.data; + return (await this.errorWrapper(() => this.axios.get(url.href, this.getConfig(ctx, url, config)))).data; }; readonly textPost = async (ctx: Context, url: URL, data: unknown, config?: AxiosRequestConfig): Promise => { this.logger.info(`Fetch post ${url}`, ctx); - const response = await this.axios.post(url.href, data, this.getConfig(ctx, url, config)); - - return response.data; + return (await this.errorWrapper(() => this.axios.post(url.href, data, this.getConfig(ctx, url, config)))).data; }; readonly head = async (ctx: Context, url: URL, config?: AxiosRequestConfig): Promise => { this.logger.info(`Fetch head ${url}`, ctx); - const response = await this.axios.head(url.href, this.getConfig(ctx, url, config)); - - return response.headers; + return (await this.errorWrapper(() => this.axios.head(url.href, this.getConfig(ctx, url, config)))).headers; }; private readonly getConfig = (ctx: Context, url: URL, config?: AxiosRequestConfig): AxiosRequestConfig => { @@ -64,4 +59,15 @@ export class Fetcher { return userAgent; }; + + private readonly errorWrapper = async (callable: () => T): Promise => { + try { + return await callable(); + } catch (error) { + if (isAxiosError(error) && error.response?.status === 404) { + throw new NotFoundError(); + } + throw error; + } + }; } diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 2aecf07..844befe 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -1,9 +1,10 @@ import winston from 'winston'; import { ExtractorRegistry } from '../extractor'; import { StreamResolver } from './StreamResolver'; -import { MeineCloud, MostraGuarda } from '../handler'; +import { Handler, MeineCloud, MostraGuarda } from '../handler'; import { Fetcher } from './Fetcher'; import { Context } from '../types'; +import { NotFoundError } from '../error'; jest.mock('../utils/Fetcher'); const logger = winston.createLogger({ transports: [new winston.transports.Console({ level: 'nope' })] }); @@ -55,4 +56,17 @@ describe('resolve', () => { const streams = await streamResolver.resolve(ctx, [meineCloud, mostraGuarda], 'movie', 'tt29141112'); expect(streams).toMatchSnapshot(); }); + + test('ignores not found errors', async () => { + const mockHandler: Handler = { + id: 'mockhandler', + label: 'MockHandler', + contentTypes: ['movie'], + languages: ['de'], + handle: jest.fn().mockRejectedValue(new NotFoundError()), + }; + + const streams = await streamResolver.resolve(ctx, [mockHandler], 'movie', 'tt12345678'); + expect(streams).toStrictEqual([]); + }); }); diff --git a/src/utils/StreamResolver.ts b/src/utils/StreamResolver.ts index cd89d3e..b97344e 100644 --- a/src/utils/StreamResolver.ts +++ b/src/utils/StreamResolver.ts @@ -4,6 +4,7 @@ import winston from 'winston'; import bytes from 'bytes'; import { Context, UrlResult } from '../types'; import { Handler } from '../handler'; +import { NotFoundError } from '../error'; export class StreamResolver { private readonly logger: winston.Logger; @@ -35,6 +36,10 @@ export class StreamResolver { urlResults.push(...(handlerUrlResults.filter(handlerUrlResult => handlerUrlResult !== undefined))); } catch (err) { + if (err instanceof NotFoundError) { + return; + } + streams.push({ name: 'WebStreamr', title: `❌ Error with handler "${handler.id}". Please create an issue if this persists. Request-id: ${ctx.id}`, diff --git a/src/utils/__fixtures__/Fetcher/https:dropload.ioasdfghijklmn.html b/src/utils/__fixtures__/Fetcher/https:dropload.ioasdfghijklmn.html new file mode 100644 index 0000000..ac1315b --- /dev/null +++ b/src/utils/__fixtures__/Fetcher/https:dropload.ioasdfghijklmn.html @@ -0,0 +1,160 @@ + + + + + + Dropload - Revolution Video Hosting + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ + +
+
+
+ + + +
+
+ +
+ + +
+ + Create an Account +
+ +
+
+ + +
+
+ + + + +
+
+
+
+ + + + +
+
+
+
+
+ +
+
+

File Not Found

+

The file you were looking for could not be found, sorry for any inconvenience.

+ Possible causes of this error could be:

+
    +
  • The file expired +
  • The file was deleted by its owner +
  • The file was deleted by administration because it didn't comply with our Terms of Use +
+
+
+
+
+
+ +
+ + + + + + + + + +