diff --git a/src/extractor/DoodStream.ts b/src/extractor/DoodStream.ts index 1a9e519..25a6a32 100644 --- a/src/extractor/DoodStream.ts +++ b/src/extractor/DoodStream.ts @@ -1,32 +1,32 @@ import randomstring from 'randomstring'; import * as cheerio from 'cheerio'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher, guessFromTitle } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; import { NotFoundError } from '../error'; -export class DoodStream implements Extractor { +export class DoodStream extends Extractor { public readonly id = 'doodstream'; public readonly label = 'DoodStream'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go|dooodster|dooood/); - public readonly normalize = (url: URL): URL => { + public override readonly normalize = (url: URL): URL => { const videoId = url.pathname.split('/').slice(-1)[0] as string; return new URL(`http://dood.to/e/${videoId}`); }; - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const html = await this.fetcher.text(ctx, new URL(url)); const passMd5Match = html.match(/\/pass_md5\/[\w-]+\/([\w-]+)/); diff --git a/src/extractor/Dropload.ts b/src/extractor/Dropload.ts index bf11a39..327f967 100644 --- a/src/extractor/Dropload.ts +++ b/src/extractor/Dropload.ts @@ -1,28 +1,28 @@ import bytes from 'bytes'; import * as cheerio from 'cheerio'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { extractUrlFromPacked, Fetcher } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; import { NotFoundError } from '../error'; -export class Dropload implements Extractor { +export class Dropload extends Extractor { public readonly id = 'dropload'; public readonly label = 'Dropload'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/); - public readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); + public override readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const html = await this.fetcher.text(ctx, url); if (html.includes('File Not Found')) { diff --git a/src/extractor/ExternalUrl.ts b/src/extractor/ExternalUrl.ts index 1799fd9..e07b927 100644 --- a/src/extractor/ExternalUrl.ts +++ b/src/extractor/ExternalUrl.ts @@ -1,25 +1,25 @@ -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; -export class ExternalUrl implements Extractor { +export class ExternalUrl extends Extractor { public readonly id = 'external'; public readonly label = 'External'; - public readonly ttl = 3600000; // 1h + public override readonly ttl = 3600000; // 1h private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/); - public readonly normalize = (url: URL): URL => url; - - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise => { try { // Make sure the URL is accessible, but avoid causing noise and delays doing this await this.fetcher.head(ctx, url, { noFlareSolverr: true, timeout: 1000 }); diff --git a/src/extractor/Extractor.ts b/src/extractor/Extractor.ts new file mode 100644 index 0000000..f24456d --- /dev/null +++ b/src/extractor/Extractor.ts @@ -0,0 +1,40 @@ +import { Context, CountryCode, UrlResult } from '../types'; +import { NotFoundError } from '../error'; + +export abstract class Extractor { + public abstract readonly id: string; + + public abstract readonly label: string; + + public readonly ttl: number = 900000; // 15m + + public abstract readonly supports: (ctx: Context, url: URL) => boolean; + + public readonly normalize = (url: URL): URL => url; + + protected abstract readonly extractInternal: (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined) => Promise; + + public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined): Promise => { + try { + return await this.extractInternal(ctx, url, countryCode, title); + } catch (error) { + if (error instanceof NotFoundError) { + return []; + } + + return [ + { + url, + isExternal: true, + error, + label: url.host, + sourceId: `${this.id}`, + meta: { + countryCode, + ...(title && { title }), + }, + }, + ]; + } + }; +} diff --git a/src/extractor/ExtractorRegistry.test.ts b/src/extractor/ExtractorRegistry.test.ts index fa46fd0..3341988 100644 --- a/src/extractor/ExtractorRegistry.test.ts +++ b/src/extractor/ExtractorRegistry.test.ts @@ -48,10 +48,12 @@ describe('ExtractorRegistry', () => { }); test('returns external url for error', async () => { - const urlResults1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de); - expect(urlResults1).toMatchSnapshot(); + const urlResults = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de); + expect(urlResults).toMatchSnapshot(); + }); - const urlResults2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked.html'), CountryCode.de, 'a title!'); - expect(urlResults2).toMatchSnapshot(); + test('returns external url for error with title', async () => { + const urlResults = await extractorRegistry.handle(ctx, new URL('https://dropload.io/mocked-blocked-2.html'), CountryCode.de, 'a title!'); + expect(urlResults).toMatchSnapshot(); }); }); diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 2ac83fb..a9668ba 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -1,8 +1,7 @@ import TTLCache from '@isaacs/ttlcache'; import winston from 'winston'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Context, CountryCode, UrlResult } from '../types'; -import { NotFoundError } from '../error'; export class ExtractorRegistry { private readonly logger: winston.Logger; @@ -30,30 +29,7 @@ export class ExtractorRegistry { this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx); - try { - urlResults = await extractor.extract(ctx, normalizedUrl, countryCode, title); - } catch (error) { - if (error instanceof NotFoundError) { - this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl }); - - return []; - } - - return [ - { - url, - isExternal: true, - error, - label: url.host, - sourceId: `${extractor.id}`, - meta: { - countryCode, - ...(title && { title }), - }, - }, - ]; - } - + urlResults = await extractor.extract(ctx, normalizedUrl, countryCode, title); this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl }); return urlResults; diff --git a/src/extractor/Fsst.ts b/src/extractor/Fsst.ts index 28f305e..c192aab 100644 --- a/src/extractor/Fsst.ts +++ b/src/extractor/Fsst.ts @@ -1,26 +1,24 @@ import * as cheerio from 'cheerio'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; -export class Fsst implements Extractor { +export class Fsst extends Extractor { public readonly id = 'fsst'; public readonly label = 'Fsst'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/fsst/); - public readonly normalize = (url: URL): URL => url; - - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const html = await this.fetcher.text(ctx, url); const $ = cheerio.load(html); diff --git a/src/extractor/KinoGer.ts b/src/extractor/KinoGer.ts index 1a5b82e..e02462e 100644 --- a/src/extractor/KinoGer.ts +++ b/src/extractor/KinoGer.ts @@ -1,27 +1,27 @@ import crypto from 'crypto'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher, guessFromTitle } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; /** @see https://github.com/Gujal00/ResolveURL/blob/master/script.module.resolveurl/lib/resolveurl/plugins/kinoger.py */ -export class KinoGer implements Extractor { +export class KinoGer extends Extractor { public readonly id = 'kinoger'; public readonly label = 'KinoGer'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/kinoger\.re|shiid4u\.upn\.one|moflix\.upns\.xyz|player\.upn\.one|wasuytm\.store|ultrastream\.online/); - public readonly normalize = (url: URL): URL => new URL(`${url.origin}/api/v1/video?id=${url.hash.slice(1)}`); + public override readonly normalize = (url: URL): URL => new URL(`${url.origin}/api/v1/video?id=${url.hash.slice(1)}`); - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const hexData = await this.fetcher.text(ctx, url, { headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36' } }); const encrypted = Buffer.from(hexData, 'hex'); diff --git a/src/extractor/Soaper.ts b/src/extractor/Soaper.ts index c5312f6..4770f4a 100644 --- a/src/extractor/Soaper.ts +++ b/src/extractor/Soaper.ts @@ -1,4 +1,4 @@ -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher, guessFromPlaylist } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; @@ -7,24 +7,22 @@ interface SoaperInfoResponsePartial { val_bak: string; } -export class Soaper implements Extractor { +export class Soaper extends Extractor { public readonly id = 'soaper'; public readonly label = 'Soaper'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/soaper/) && null !== url.pathname.match(/^\/(episode|movie)_/); - public readonly normalize = (url: URL): URL => url; - - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode, title: string | undefined): Promise => { const movieOrEpisodeId = (url.pathname.match(/\/\w+_(\w+)/) as string[])[1] as string; const form = new URLSearchParams(); diff --git a/src/extractor/SuperVideo.ts b/src/extractor/SuperVideo.ts index 7b84bc6..68d923d 100644 --- a/src/extractor/SuperVideo.ts +++ b/src/extractor/SuperVideo.ts @@ -1,32 +1,32 @@ import bytes from 'bytes'; import * as cheerio from 'cheerio'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { extractUrlFromPacked, Fetcher } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; import { NotFoundError } from '../error'; -export class SuperVideo implements Extractor { +export class SuperVideo extends Extractor { public readonly id = 'supervideo'; public readonly label = 'SuperVideo'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/); - public readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); + public override readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const html = await this.fetcher.text(ctx, url); if (html.includes('This video can be watched as embed only')) { - return await this.extract(ctx, new URL(`/e${url.pathname}`, url.origin), countryCode); + return await this.extractInternal(ctx, new URL(`/e${url.pathname}`, url.origin), countryCode); } if (/'The file was deleted|The file expired/.test(html)) { diff --git a/src/extractor/VidSrc.ts b/src/extractor/VidSrc.ts index 44cd427..6722ad9 100644 --- a/src/extractor/VidSrc.ts +++ b/src/extractor/VidSrc.ts @@ -1,28 +1,26 @@ import * as cheerio from 'cheerio'; import slugify from 'slugify'; -import { Extractor } from './types'; +import { Extractor } from './Extractor'; import { Fetcher, guessFromPlaylist } from '../utils'; import { Context, CountryCode, UrlResult } from '../types'; import { NotFoundError } from '../error'; -export class VidSrc implements Extractor { +export class VidSrc extends Extractor { public readonly id = 'vidsrc'; public readonly label = 'VidSrc'; - public readonly ttl = 900000; // 15m - private readonly fetcher: Fetcher; public constructor(fetcher: Fetcher) { + super(); + this.fetcher = fetcher; } public readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/vidsrc/); - public readonly normalize = (url: URL): URL => url; - - public readonly extract = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { + protected readonly extractInternal = async (ctx: Context, url: URL, countryCode: CountryCode): Promise => { const html = await this.fetcher.text(ctx, url); const $ = cheerio.load(html); diff --git a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap index 29d0dbb..fe777c7 100644 --- a/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap +++ b/src/extractor/__snapshots__/ExtractorRegistry.test.ts.snap @@ -33,7 +33,7 @@ exports[`ExtractorRegistry returns external url for error 1`] = ` ] `; -exports[`ExtractorRegistry returns external url for error 2`] = ` +exports[`ExtractorRegistry returns external url for error with title 1`] = ` [ { "error": [Error: Fetcher error: 403: Forbidden @@ -45,7 +45,7 @@ exports[`ExtractorRegistry returns external url for error 2`] = ` "title": "a title!", }, "sourceId": "dropload", - "url": "https://dropload.io/mocked-blocked.html", + "url": "https://dropload.io/mocked-blocked-2.html", }, ] `; diff --git a/src/extractor/index.ts b/src/extractor/index.ts index 066aeec..6f9ba04 100644 --- a/src/extractor/index.ts +++ b/src/extractor/index.ts @@ -1,16 +1,16 @@ -import { Extractor } from './types'; import { DoodStream } from './DoodStream'; import { Dropload } from './Dropload'; +import { ExternalUrl } from './ExternalUrl'; +import { Extractor } from './Extractor'; import { Fsst } from './Fsst'; -import { SuperVideo } from './SuperVideo'; import { KinoGer } from './KinoGer'; import { Soaper } from './Soaper'; +import { SuperVideo } from './SuperVideo'; import { VidSrc } from './VidSrc'; -import { ExternalUrl } from './ExternalUrl'; import { Fetcher } from '../utils'; +export * from './Extractor'; export * from './ExtractorRegistry'; -export * from './types'; export const createExtractors = (fetcher: Fetcher): Extractor[] => [ new DoodStream(fetcher), diff --git a/src/extractor/types.ts b/src/extractor/types.ts deleted file mode 100644 index 05cc2f1..0000000 --- a/src/extractor/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Context, CountryCode, UrlResult } from '../types'; - -export interface Extractor { - readonly id: string; - - readonly label: string; - - readonly ttl: number; - - readonly supports: (ctx: Context, url: URL) => boolean; - - readonly normalize: (url: URL) => URL; - - readonly extract: (ctx: Context, url: URL, countryCode: CountryCode, title?: string | undefined) => Promise; -} diff --git a/src/utils/StreamResolver.test.ts b/src/utils/StreamResolver.test.ts index 6326806..633512b 100644 --- a/src/utils/StreamResolver.test.ts +++ b/src/utils/StreamResolver.test.ts @@ -77,18 +77,16 @@ describe('resolve', () => { }; } - class MockExtractor implements Extractor { + class MockExtractor extends Extractor { public readonly id = 'mockextractor'; public readonly label = 'MockExtractor'; - public readonly ttl = 1; + public override readonly ttl = 1; public readonly supports = (): boolean => true; - public readonly normalize = (url: URL): URL => url; - - public readonly extract = async (): Promise => + protected readonly extractInternal = async (): Promise => [ { url: new URL('https://example.com'), diff --git a/src/utils/__fixtures__/Fetcher/https:dropload.iomocked-blocked-2.html.error b/src/utils/__fixtures__/Fetcher/https:dropload.iomocked-blocked-2.html.error new file mode 100644 index 0000000..8a1c7cd --- /dev/null +++ b/src/utils/__fixtures__/Fetcher/https:dropload.iomocked-blocked-2.html.error @@ -0,0 +1 @@ +Fetcher error: 403: Forbidden