diff --git a/src/extractor/DoodStream.ts b/src/extractor/DoodStream.ts index 8fe3935..75c65b8 100644 --- a/src/extractor/DoodStream.ts +++ b/src/extractor/DoodStream.ts @@ -20,11 +20,14 @@ export class DoodStream implements Extractor { readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/); - readonly extract = async (ctx: Context, url: URL, meta: Meta) => { + readonly normalize = (url: URL): URL => { const videoId = url.pathname.split('/').slice(-1)[0] as string; - const normalizedUrl = new URL(`http://dood.to/e/${videoId}`); - const html = await this.fetcher.text(ctx, new URL(normalizedUrl)); + return new URL(`http://dood.to/e/${videoId}`); + }; + + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { + const html = await this.fetcher.text(ctx, new URL(url)); const passMd5Match = html.match(/\/pass_md5\/[\w-]+\/([\w-]+)/); if (!passMd5Match) { diff --git a/src/extractor/Dropload.ts b/src/extractor/Dropload.ts index b6eaa8e..137228d 100644 --- a/src/extractor/Dropload.ts +++ b/src/extractor/Dropload.ts @@ -20,9 +20,10 @@ export class Dropload implements Extractor { readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/); + readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { - const normalizedUrl = new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); - const html = await this.fetcher.text(ctx, normalizedUrl); + const html = await this.fetcher.text(ctx, url); if (html.includes('File Not Found')) { throw new NotFoundError(); diff --git a/src/extractor/ExternalUrl.ts b/src/extractor/ExternalUrl.ts index 6fa1411..2d571f0 100644 --- a/src/extractor/ExternalUrl.ts +++ b/src/extractor/ExternalUrl.ts @@ -17,6 +17,8 @@ export class ExternalUrl implements Extractor { readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/); + readonly normalize = (url: URL): URL => url; + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { // We only want to make sure that the URL is accessible await this.fetcher.head(ctx, url, { noFlareSolverr: true }); diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts index 57bb6f5..d6883bd 100644 --- a/src/extractor/ExtractorRegistry.ts +++ b/src/extractor/ExtractorRegistry.ts @@ -30,23 +30,25 @@ export class ExtractorRegistry { } readonly handle = async (ctx: Context, url: URL, meta: Meta): Promise => { - let urlResults = this.urlResultCache.get(url.href) ?? []; - if (this.urlResultCache.has(url.href)) { - return urlResults.map(urlResult => ({ ...urlResult, ttl: this.urlResultCache.getRemainingTTL(url.href) })); - } - const extractor = this.extractors.find(extractor => extractor.supports(ctx, url)); if (!extractor) { return []; } + const normalizedUrl = extractor.normalize(url); + + let urlResults = this.urlResultCache.get(normalizedUrl.href) ?? []; + if (this.urlResultCache.has(normalizedUrl.href)) { + return urlResults.map(urlResult => ({ ...urlResult, ttl: this.urlResultCache.getRemainingTTL(normalizedUrl.href) })); + } + this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx); try { - urlResults = await extractor.extract(ctx, url, meta); + urlResults = await extractor.extract(ctx, normalizedUrl, meta); } catch (error) { if (error instanceof NotFoundError) { - this.urlResultCache.set(url.href, urlResults, { ttl: extractor.ttl }); + this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl }); return []; } @@ -63,7 +65,7 @@ export class ExtractorRegistry { ]; } - this.urlResultCache.set(url.href, urlResults, { ttl: extractor.ttl }); + this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl }); return urlResults; }; diff --git a/src/extractor/Fsst.ts b/src/extractor/Fsst.ts index a04c2ca..93f0920 100644 --- a/src/extractor/Fsst.ts +++ b/src/extractor/Fsst.ts @@ -18,6 +18,8 @@ export class Fsst implements Extractor { readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/fsst/); + readonly normalize = (url: URL): URL => url; + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { const html = await this.fetcher.text(ctx, url); diff --git a/src/extractor/Soaper.ts b/src/extractor/Soaper.ts index 9b70b4e..a0fd676 100644 --- a/src/extractor/Soaper.ts +++ b/src/extractor/Soaper.ts @@ -22,6 +22,8 @@ export class Soaper implements Extractor { readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/soaper/) && null !== url.pathname.match(/^\/(episode|movie)_/); + readonly normalize = (url: URL): URL => url; + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { const movieOrEpisodeId = (url.pathname.match(/\/\w+_(\w+)/) as string[])[1] as string; diff --git a/src/extractor/SuperVideo.ts b/src/extractor/SuperVideo.ts index 93cd630..a8220f3 100644 --- a/src/extractor/SuperVideo.ts +++ b/src/extractor/SuperVideo.ts @@ -19,9 +19,10 @@ export class SuperVideo implements Extractor { readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/); + readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); + readonly extract = async (ctx: Context, url: URL, meta: Meta) => { - const normalizedUrl = new URL(url.href.replace('/e/', '/').replace('/embed-', '/')); - const html = await this.fetcher.text(ctx, normalizedUrl); + const html = await this.fetcher.text(ctx, url); const heightAndSizeMatch = html.match(/\d{3,}x(\d{3,}), ([\d.]+ ?[GM]B)/) as string[]; diff --git a/src/extractor/types.ts b/src/extractor/types.ts index c793fae..65ec9b4 100644 --- a/src/extractor/types.ts +++ b/src/extractor/types.ts @@ -9,5 +9,7 @@ export interface Extractor { readonly supports: (ctx: Context, url: URL) => boolean; + readonly normalize: (url: URL) => URL; + readonly extract: (ctx: Context, url: URL, meta: Meta) => Promise; }