Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

refactor(extractor): introduce URL normalization

Browse source
This commit is contained in:
WebStreamr 2025-06-10 15:03:05 +00:00
parent 7b933bfbac
commit 2d615ce582
No known key found for this signature in database
8 changed files with 30 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/extractor/DoodStream.ts
View file

@ -20,11 +20,14 @@ export class DoodStream implements Extractor {
readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dood|do[0-9]go/);
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
readonly normalize = (url: URL): URL => {
const videoId = url.pathname.split('/').slice(-1)[0] as string;
const normalizedUrl = new URL(`http://dood.to/e/${videoId}`);
const html = await this.fetcher.text(ctx, new URL(normalizedUrl));
return new URL(`http://dood.to/e/${videoId}`);
};
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
const html = await this.fetcher.text(ctx, new URL(url));
const passMd5Match = html.match(/\/pass_md5\/[\w-]+\/([\w-]+)/);
if (!passMd5Match) {

5
src/extractor/Dropload.ts
View file

@ -20,9 +20,10 @@ export class Dropload implements Extractor {
readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/dropload/);
readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
const normalizedUrl = new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
const html = await this.fetcher.text(ctx, normalizedUrl);
const html = await this.fetcher.text(ctx, url);
if (html.includes('File Not Found')) {
throw new NotFoundError();

2
src/extractor/ExternalUrl.ts
View file

@ -17,6 +17,8 @@ export class ExternalUrl implements Extractor {
readonly supports = (ctx: Context, url: URL): boolean => !('excludeExternalUrls' in ctx.config) && null !== url.host.match(/.*/);
readonly normalize = (url: URL): URL => url;
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
// We only want to make sure that the URL is accessible
await this.fetcher.head(ctx, url, { noFlareSolverr: true });

18
src/extractor/ExtractorRegistry.ts
View file

@ -30,23 +30,25 @@ export class ExtractorRegistry {
}
readonly handle = async (ctx: Context, url: URL, meta: Meta): Promise<UrlResult[]> => {
let urlResults = this.urlResultCache.get(url.href) ?? [];
if (this.urlResultCache.has(url.href)) {
return urlResults.map(urlResult => ({ ...urlResult, ttl: this.urlResultCache.getRemainingTTL(url.href) }));
}
const extractor = this.extractors.find(extractor => extractor.supports(ctx, url));
if (!extractor) {
return [];
}
const normalizedUrl = extractor.normalize(url);
let urlResults = this.urlResultCache.get(normalizedUrl.href) ?? [];
if (this.urlResultCache.has(normalizedUrl.href)) {
return urlResults.map(urlResult => ({ ...urlResult, ttl: this.urlResultCache.getRemainingTTL(normalizedUrl.href) }));
}
this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx);
try {
urlResults = await extractor.extract(ctx, url, meta);
urlResults = await extractor.extract(ctx, normalizedUrl, meta);
} catch (error) {
if (error instanceof NotFoundError) {
this.urlResultCache.set(url.href, urlResults, { ttl: extractor.ttl });
this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl });
return [];
}
@ -63,7 +65,7 @@ export class ExtractorRegistry {
];
}
this.urlResultCache.set(url.href, urlResults, { ttl: extractor.ttl });
this.urlResultCache.set(normalizedUrl.href, urlResults, { ttl: extractor.ttl });
return urlResults;
};

2
src/extractor/Fsst.ts
View file

@ -18,6 +18,8 @@ export class Fsst implements Extractor {
readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/fsst/);
readonly normalize = (url: URL): URL => url;
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
const html = await this.fetcher.text(ctx, url);

2
src/extractor/Soaper.ts
View file

@ -22,6 +22,8 @@ export class Soaper implements Extractor {
readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/soaper/) && null !== url.pathname.match(/^\/(episode|movie)_/);
readonly normalize = (url: URL): URL => url;
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
const movieOrEpisodeId = (url.pathname.match(/\/\w+_(\w+)/) as string[])[1] as string;

5
src/extractor/SuperVideo.ts
View file

@ -19,9 +19,10 @@ export class SuperVideo implements Extractor {
readonly supports = (_ctx: Context, url: URL): boolean => null !== url.host.match(/supervideo/);
readonly normalize = (url: URL): URL => new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
readonly extract = async (ctx: Context, url: URL, meta: Meta) => {
const normalizedUrl = new URL(url.href.replace('/e/', '/').replace('/embed-', '/'));
const html = await this.fetcher.text(ctx, normalizedUrl);
const html = await this.fetcher.text(ctx, url);
const heightAndSizeMatch = html.match(/\d{3,}x(\d{3,}), ([\d.]+ ?[GM]B)/) as string[];

2
src/extractor/types.ts
View file

@ -9,5 +9,7 @@ export interface Extractor {
readonly supports: (ctx: Context, url: URL) => boolean;
readonly normalize: (url: URL) => URL;
readonly extract: (ctx: Context, url: URL, meta: Meta) => Promise<UrlResult[]>;
}