Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

fix(extractor): do not throw for broken/outdated link

Browse source
This commit is contained in:
WebStreamr 2025-05-16 22:28:17 +02:00
parent b790cc7954
commit 63a79f8758
No known key found for this signature in database
6 changed files with 264 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/extractor/EmbedExtractorRegistry.test.ts → src/extractor/ExtractorRegistry.test.ts
View file

@ -9,15 +9,19 @@ const logger = winston.createLogger({ transports: [new winston.transports.Consol
const fetcher = new Fetcher();
const extractorRegistry = new ExtractorRegistry(logger, fetcher);
describe('EmbedExtractorRegistry', () => {
describe('ExtractorRegistry', () => {
const ctx: Context = { ip: '127.0.0.1' };
test('returns undefined when no embed extractor can be found', async () => {
test('returns undefined when no extractor can be found', async () => {
const urlResult = await extractorRegistry.handle(ctx, new URL('https://some-url.test'), 'en');
expect(urlResult).toBeUndefined();
});
test('returns undefined when extractor fails', async () => {
expect(await extractorRegistry.handle(ctx, new URL('https://mixdrop.my/e/123456789'), 'en')).toBeUndefined();
});
test('returns from memory cache if possible', async () => {
const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');
const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');

20
src/extractor/ExtractorRegistry.ts
View file

@ -10,12 +10,12 @@ import { Mixdrop } from './Mixdrop';
export class ExtractorRegistry {
private readonly logger: winston.Logger;
private readonly embedExtractors: Extractor[];
private readonly extractors: Extractor[];
private readonly urlResultCache: TTLCache<string, UrlResult>;
constructor(logger: winston.Logger, fetcher: Fetcher) {
this.logger = logger;
this.embedExtractors = [
this.extractors = [
new DoodStream(fetcher),
new Dropload(fetcher),
new Mixdrop(fetcher),
@ -30,18 +30,22 @@ export class ExtractorRegistry {
return urlResult;
}
const embedExtractor = this.embedExtractors.find(embedExtractor => embedExtractor.supports(url));
if (undefined === embedExtractor) {
const extractor = this.extractors.find(embedExtractor => embedExtractor.supports(url));
if (undefined === extractor) {
return undefined;
}
this.logger.info(`Extract stream URL using ${embedExtractor.id} extractor from ${url}`);
this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`);
urlResult = await embedExtractor.extract(ctx, url, countryCode);
if (urlResult) {
this.urlResultCache.set(url.href, urlResult, { ttl: embedExtractor.ttl });
try {
urlResult = await extractor.extract(ctx, url, countryCode);
} catch (error) {
this.logger.warn(`${extractor.id} error: ` + error);
return undefined;
}
this.urlResultCache.set(url.href, urlResult, { ttl: extractor.ttl });
return urlResult;
};
}

2
src/extractor/types.ts
View file

@ -9,5 +9,5 @@ export interface Extractor {
readonly supports: (url: URL) => boolean;
readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise<UrlResult | undefined>;
readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise<UrlResult>;
}

2
src/utils/StreamResolver.ts
View file

@ -41,7 +41,7 @@ export class StreamResolver {
ytId: 'E4WlUXrJgy4',
});
this.logger.error(`${handler.id} error: ` + err);
this.logger.error(`${handler.id} error: ${err}`);
}
});
await Promise.all(handlerPromises);

52
src/utils/__fixtures__/Fetcher/https:mixdrop.mye123456789 generated Normal file
View file

File diff suppressed because one or more lines are too long

192
src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789 generated Normal file
View file

@ -0,0 +1,192 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta name="viewport" content="width=device-width; initial-scale=1.0">
<title>MixDrop - File not found</title>
<meta name="description" content="" />
<meta name="keywords" content="mixdrop,share file,share video,upload file,upload video" />
<meta property="og:title" content="MixDrop - File not found" />
<meta property="og:description" content="" />
<meta name="verified-code" content="035b8906-2381-4a3a-9d4c-3b2ae07293e4">
<!--<meta name="robots" content="nosnippet">-->
<meta name="a.validate.02" content="ctm1Vhh9zx0o7ew5xlkxmA8MjHBDjK2gY_5x" />
<link rel="icon" type="image/png" href="/imgs/v2/favicon-32x32.png" sizes="32x32" />
<link rel="icon" type="image/png" href="/imgs/v2/favicon-16x16.png" sizes="16x16" />
<meta name="csrf" content="40758e7869c55031abee715dea547b6a">
<meta name="clckd" content="ebd626d580ab61986c788e761c305e41" />
<meta name="monetag" content="d1dda856d50c7dd239c4daa5c545e3d3">
<meta name="41e891e66eaaca8fc8294c24cc35a2b1eee3aa09" content="41e891e66eaaca8fc8294c24cc35a2b1eee3aa09" />
<link rel="stylesheet" href="/js/slidebars/slidebars.css?v=0.1" />
<link rel="stylesheet" href="/css/style.v2.0.2.min.css" />
<script src="https://code.jquery.com/jquery-3.6.4.min.js" integrity="sha256-oP6HI9z1XaZNBrJURtCoUT5SUnxFr8s3BzRl+cbzUq8=" crossorigin="anonymous"></script>
</head>
<body>
<div class="container" canvas="container">
<div class="panel header">
<div class="wrapper">
<a href="/"><img src="/imgs/v2/logo.png" class="logo" alt="mixdrop" /></a>
<div class="menu">
<a href="javascript:void(0)" class="btn btn1 openModal" data-modal="signup">SIGNUP</a>
<a href="javascript:void(0)" class="btn btn2 openModal" data-modal="login">LOGIN</a>
</div> <div class="menu-mobile"><img src="/imgs/v2/menu.png" alt="menu"></div>
</div>
</div>
<div class="panel download error txt-c">
<div class="wrapper">
<img src="/imgs/illustration-notfound.png" style="width: 250px;max-width: 80%;" />
<h2>WE ARE SORRY</h2>
<h3>We can't find the file you are looking for.</h3>
<script data-cfasync="false" async type="text/javascript" src="//xx.apptdinsteps.com/sBq05a7a7j1Yhgl/117018"></script> <div style="padding: 30px 0">
</div>
</div>
</div>
<footer class="panel">
<div class="wrapper">
<div class="col-3">
<a href="/"><img src="/imgs/v2/logo.png" class="logo" alt="mixdrop" /></a>
<a href="javascript:void(0)" class="openModal" data-modal="contact">Contact</a>
<a href="/news/">News</a>
<a href="/faq/">FAQ</a>
</div>
<div class="col-3">
<div class="hdr">POLICY</div>
<a href="/terms/">Terms And Conditions</a>
<a href="/privacy/">Privacy Policy</a>
<a href="/copyright/">Copyright Policy</a>
<a href="javascript:void(0)" class="openModal" data-modal="abuse">Report Abuse</a>
</div>
<div class="col-3">
<div class="hdr">SERVICES</div>
<a href="/panel/">User Panel</a>
<a href="/api/">API</a>
</div>
<!--
<div class="col-lang">
<div class="langSwitch">
<a href="javascript:void(0)" title="English"><img src="/imgs/flag_en.png" alt="english"> EN</a>
<div class="langOptions" style="display: none;">
</div>
</div>
</div>-->
</div>
</footer>
</div>
<div class="uploadbox">
<div class="header">Uploads ( <span>0 / 0</span> ) <i class="toggle"></i></div>
<div class="scroll">
<div class="tbl">
</div>
</div>
</div>
<div id="goUp" class="scrollTo floating" data-scrollto='.container'></div>
<div class="modal" id="modal-login">
<div class="modal-wrapper">
<form action="/actions/global.php">
<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
<input type="password" name="password" placeholder="Password" class="ic ic-in-pass" />
<div class="2fa-content" style="display:none">
<label class="hdr">2FA Confirmation Code</label>
<p>A confirmation code was sent to your e-mail address, please confirm it below</p>
<input type="text" name="code" placeholder="" style="text-transform: uppercase;font-size: 2rem;letter-spacing: .5rem;" />
</div>
<button type="button" class="submit btn2">LOGIN</button>
<a href="javascript:void(0)" class="forgot">Forgot your password?</a>
<div class="mbError">&nbsp;</div>
<input type="hidden" name="step" value="0" />
<input type="hidden" name="a" value="login" />
</form>
</div>
</div>
<div class="modal" id="modal-recover">
<div class="modal-wrapper">
<form action="/actions/global.php">
<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
<button type="button" class="submit btn2">RECOVER</button>
<div class="mbError">&nbsp;</div>
<input type="hidden" name="a" value="recover" />
</form>
</div>
</div>
<div class="modal" id="modal-signup">
<div class="modal-wrapper">
<form action="/actions/global.php">
<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
<input type="password" name="password" placeholder="Password" class="ic ic-in-pass" />
<input type="password" name="passwordconfirm" placeholder="Repeat Password" class="ic ic-in-pass" />
<button type="button" class="submit btn2">SIGNUP</button>
<a href="javascript:void(0)" class="mlogin">Already have an account?</a>
<div class="mbError">&nbsp;</div>
<input type="hidden" name="a" value="signup" />
</form>
</div>
</div>
<div class="modal" id="modal-contact">
<div class="modal-wrapper">
<form action="/actions/global.php">
<input type="text" name="name" placeholder="Your name" class="ic ic-in-user" />
<input type="text" name="email" placeholder="Your E-Mail" class="ic ic-in-mail" />
<textarea name="message" placeholder="Your message" class="ic ic-in-compose" rows="6"></textarea>
<button type="button" class="submit btn2">SEND MESSAGE</button>
<div class="mbError">&nbsp;</div>
<input type="hidden" name="a" value="contact" />
</form>
</div>
</div>
<div class="modal" id="modal-abuse">
<div class="modal-wrapper">
<form action="/actions/global.php">
<select name="reason" class="ic ic-in-list">
<option>Reason</option>
<option value="copyright">Copyright infringement</option>
<option value="cp">Child Pornography</option>
<option value="terrorism">Terrorism Content</option>
</select>
<input type="text" name="name" placeholder="Your name" class="ic ic-in-user" />
<input type="text" name="email" placeholder="Your E-Mail" class="ic ic-in-mail" />
<textarea name="message" placeholder="Your message" class="ic ic-in-compose" rows="6"></textarea>
<button type="button" class="submit btn2">SEND MESSAGE</button>
<div class="mbError">&nbsp;</div>
<input type="hidden" name="a" value="abuse" />
</form>
</div>
</div>
<div off-canvas="menu right push" class="toggleMenu">
<a href="javascript:void(0)" class="openModal" data-modal="login">LOGIN</a>
<a href="javascript:void(0)" class="openModal" data-modal="signup">SIGNUP</a>
<a href="javascript:void(0)" class="scrollTo" data-scrollto=".landing">HOME</a>
<a href="/panel/">USER PANEL</a>
<a href="javascript:void(0)" class="scrollTo" data-scrollto=".share">SHARE</a>
<a href="javascript:void(0)" class="scrollTo" data-scrollto=".embed">EMBED</a>
<a href="" class="scrollTo" data-scrollto=".affiliate">AFFILIATE</a>
<a href="/faq/">FAQ</a>
<a href="/news/">NEWS</a>
<a href="javascript:void(0)" class="openModal" data-modal="contact">CONTACT</a>
</div>
<script src="https://code.jquery.com/ui/1.13.2/jquery-ui.min.js" integrity="sha256-lSjKY0/srUM9BE3dPm+c4fBo1dky2v27Gdjm2uoZaL0=" crossorigin="anonymous"></script>
<script src="/js/modal/modal.js?v=2.0.1"></script>
<script src="/js/slidebars/slidebars.min.js?v=2.0.1"></script>
<script src="/js/circular-progress/circle-progress.min.js?v=2.0.1"></script>
<script src="/js/jquery-upload/js/jquery.iframe-transport.js?v=2.0.1"></script>
<script src="/js/jquery-upload/js/jquery.fileupload.js?v=2.0.1"></script>
<script src="/js/jquery-upload/js/main.js?v=2.0.1"></script>
<script src="/panel/js/scroll/perfect-scrollbar.min.js?v=2.0.1"></script>
<link rel="stylesheet" href="/panel/js/scroll/perfect-scrollbar.css?v=2.0.1" />
<script src="/js/script.v2.min.js?v=2.0.1"></script>
<script src="/js/ads.js"></script>
<script defer data-domain="mixdrop.ag/f" src="https://mdstats.info/js/script.js"></script>
</body>
</html>