feat(extractor): add external URL support

This could still be a useful fallback on some devices since it indicates
that we found a potential embed URL but could not extract the video URL.

src/extractor/ExternalUrl.ts

@@ -0,0 +1,34 @@
+import { Extractor } from './types';
+import { Fetcher } from '../utils';
+import { Context } from '../types';
+
+export class ExternalUrl implements Extractor {
+  readonly id = 'external';
+
+  readonly label = 'External';
+
+  readonly ttl = 3600000; // 1h
+
+  private readonly fetcher: Fetcher;
+
+  constructor(fetcher: Fetcher) {
+    this.fetcher = fetcher;
+  }
+
+  readonly supports = (url: URL): boolean => null !== url.host.match(/.*/);
+
+  readonly extract = async (ctx: Context, url: URL, countryCode: string) => {
+    // We only want to make sure that the URL is accessible
+    await this.fetcher.text(ctx, url);
+
+    return {
+      url: url,
+      isExternal: true,
+      label: `${url.host}`,
+      sourceId: `${this.id}_${countryCode.toLowerCase()}`,
+      height: 0,
+      bytes: 0,
+      countryCode,
+    };
+  };
+}

src/extractor/ExtractorRegistry.test.ts

@@ -18,10 +18,6 @@ describe('ExtractorRegistry', () => {
     expect(urlResult).toBeUndefined();
   });
 
-  test('returns undefined when extractor fails', async () => {
-    expect(await extractorRegistry.handle(ctx, new URL('https://mixdrop.my/e/123456789'), 'en')).toBeUndefined();
-  });
-
   test('returns from memory cache if possible', async () => {
     const urlResult1 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');
     const urlResult2 = await extractorRegistry.handle(ctx, new URL('https://dropload.io/lyo2h1snpe5c.html'), 'de');

src/extractor/ExtractorRegistry.ts

@@ -6,6 +6,7 @@ import { Fetcher } from '../utils';
 import { DoodStream } from './DoodStream';
 import { Dropload } from './Dropload';
 import { SuperVideo } from './SuperVideo';
+import { ExternalUrl } from './ExternalUrl';
 
 export class ExtractorRegistry {
   private readonly logger: winston.Logger;
@@ -18,6 +19,7 @@ export class ExtractorRegistry {
       new DoodStream(fetcher),
       new Dropload(fetcher),
       new SuperVideo(fetcher),
+      new ExternalUrl(fetcher), // fallback extractor which must come last
     ];
     this.urlResultCache = new TTLCache({ max: 1024 });
   }
@@ -28,11 +30,7 @@ export class ExtractorRegistry {
       return urlResult;
     }
 
-    const extractor = this.extractors.find(extractor => extractor.supports(url));
-    if (undefined === extractor) {
-      return undefined;
-    }
-
+    const extractor = this.extractors.find(extractor => extractor.supports(url)) as Extractor;
     this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx);
 
     try {

src/handler/Frembed.test.ts

@@ -27,7 +27,7 @@ describe('Frembed', () => {
   test('handle imdb black mirror s4e2', async () => {
     const streams = (await handler.handle(ctx, 'series', 'tt2085059:4:2')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(1);
+    expect(streams).toHaveLength(3);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'DoodStream',
@@ -40,5 +40,25 @@ describe('Frembed', () => {
       },
     });
     expect(streams[0]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
+    expect(streams[1]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'netu.frembed.art',
+      sourceId: 'external_fr',
+      height: 0,
+      bytes: 0,
+      countryCode: 'fr',
+    });
+    expect(streams[1]?.url.href).toMatch(/netu\.frembed\.art/);
+    expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'johnalwayssame.com',
+      sourceId: 'external_fr',
+      height: 0,
+      bytes: 0,
+      countryCode: 'fr',
+    });
+    expect(streams[2]?.url.href).toMatch(/johnalwayssame\.com/);
   });
 });

src/handler/FrenchCloud.test.ts

@@ -27,7 +27,7 @@ describe('FrenchCloud', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(3);
+    expect(streams).toHaveLength(4);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -47,6 +47,16 @@ describe('FrenchCloud', () => {
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
     expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'mixdrop.ag',
+      sourceId: 'external_fr',
+      height: 0,
+      bytes: 0,
+      countryCode: 'fr',
+    });
+    expect(streams[2]?.url.href).toMatch(/mixdrop\.ag/);
+    expect(streams[3]).toStrictEqual({
       url: expect.any(URL),
       label: 'DoodStream',
       sourceId: 'doodstream_fr',
@@ -57,6 +67,6 @@ describe('FrenchCloud', () => {
         Referer: 'http://dood.to/',
       },
     });
-    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
+    expect(streams[3]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/handler/MeineCloud.test.ts

@@ -27,7 +27,7 @@ describe('MeineCloud', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(3);
+    expect(streams).toHaveLength(4);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -47,6 +47,16 @@ describe('MeineCloud', () => {
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
     expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'mixdrop.ag',
+      sourceId: 'external_de',
+      height: 0,
+      bytes: 0,
+      countryCode: 'de',
+    });
+    expect(streams[2]?.url.href).toMatch(/mixdrop\.ag/);
+    expect(streams[3]).toStrictEqual({
       url: expect.any(URL),
       label: 'DoodStream',
       sourceId: 'doodstream_de',
@@ -57,6 +67,6 @@ describe('MeineCloud', () => {
         Referer: 'http://dood.to/',
       },
     });
-    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
+    expect(streams[3]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/handler/MostraGuarda.test.ts

@@ -27,7 +27,7 @@ describe('MostraGuarda', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(3);
+    expect(streams).toHaveLength(4);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -47,6 +47,16 @@ describe('MostraGuarda', () => {
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
     expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'mixdrop.ag',
+      sourceId: 'external_it',
+      height: 0,
+      bytes: 0,
+      countryCode: 'it',
+    });
+    expect(streams[2]?.url.href).toMatch(/mixdrop\.ag/);
+    expect(streams[3]).toStrictEqual({
       url: expect.any(URL),
       label: 'DoodStream',
       sourceId: 'doodstream_it',
@@ -57,6 +67,6 @@ describe('MostraGuarda', () => {
         Referer: 'http://dood.to/',
       },
     });
-    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
+    expect(streams[3]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/handler/VerHdLink.test.ts

@@ -27,7 +27,7 @@ describe('VerHdLink', () => {
   test('handle titanic', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt0120338')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(4);
+    expect(streams).toHaveLength(6);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -47,6 +47,16 @@ describe('VerHdLink', () => {
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
     expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'mixdrop.ag',
+      sourceId: 'external_mx',
+      height: 0,
+      bytes: 0,
+      countryCode: 'mx',
+    });
+    expect(streams[2]?.url.href).toMatch(/mixdrop\.ag/);
+    expect(streams[3]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
       sourceId: 'supervideo_es',
@@ -54,8 +64,8 @@ describe('VerHdLink', () => {
       bytes: 1610612736,
       countryCode: 'es',
     });
-    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
-    expect(streams[3]).toStrictEqual({
+    expect(streams[3]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[4]).toStrictEqual({
       url: expect.any(URL),
       label: 'Dropload',
       sourceId: 'dropload_es',
@@ -63,6 +73,16 @@ describe('VerHdLink', () => {
       bytes: 1610612736,
       countryCode: 'es',
     });
-    expect(streams[3]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[4]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[5]).toStrictEqual({
+      url: expect.any(URL),
+      isExternal: true,
+      label: 'mixdrop.ag',
+      sourceId: 'external_es',
+      height: 0,
+      bytes: 0,
+      countryCode: 'es',
+    });
+    expect(streams[5]?.url.href).toMatch(/mixdrop\.ag/);
   });
 });

src/types.ts

@@ -12,6 +12,7 @@ export type Config = Record<string, string>;
 
 export interface UrlResult {
   url: URL;
+  isExternal?: boolean;
   label: string;
   sourceId: string;
   height: number;

src/utils/StreamResolver.test.ts

@@ -119,6 +119,22 @@ describe('resolve', () => {
           },
         },
       },
+      {
+        externalUrl: 'https://mixdrop.ag/e/3nzwveprim63or6',
+        name: 'WebStreamr external',
+        title: 'mixdrop.ag | 🇩🇪',
+        behaviorHints: {
+          bingeGroup: 'webstreamr-external_de',
+        },
+      },
+      {
+        externalUrl: 'https://mixdrop.ag/e/vk196d6xfzwwo1',
+        name: 'WebStreamr external',
+        title: 'mixdrop.ag | 🇮🇹',
+        behaviorHints: {
+          bingeGroup: 'webstreamr-external_it',
+        },
+      },
     ]);
   });
 });

src/utils/StreamResolver.ts

@@ -68,6 +68,9 @@ export class StreamResolver {
         if (urlResult.height) {
           name += ` ${urlResult.height}p`;
         }
+        if (urlResult.isExternal) {
+          name += ` external`;
+        }
 
         let title = urlResult.label;
         if (urlResult.bytes) {
@@ -78,7 +81,7 @@ export class StreamResolver {
         }
 
         return {
-          url: urlResult.url.href,
+          [urlResult.isExternal ? 'externalUrl' : 'url']: urlResult.url.href,
           name,
           title,
           behaviorHints: {

src/utils/__fixtures__/Fetcher/https:johnalwayssame.comecqy9oue7sv0g

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Redirecting...</title>
+</head>
+<body>
+<script>
+        if (typeof localStorage !== 'undefined') {
+            const permanentToken = localStorage.getItem('permanentToken');
+            if (permanentToken) {
+                const currentUrl = new URL(window.location.href);
+                currentUrl.searchParams.set('permanentToken', permanentToken);
+                window.location.href = currentUrl.toString();
+            } else {
+                window.location.href = 'https://kellywhatcould.com/e/cqy9oue7sv0g';
+            }
+        } else {
+            window.location.href = 'https://kellywhatcould.com/e/cqy9oue7sv0g';
+        }
+    </script>
+</body>
+</html>

src/utils/__fixtures__/Fetcher/https:mixdrop.myf123456789

@@ -1,192 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
-<meta name="viewport" content="width=device-width; initial-scale=1.0">
-<title>MixDrop - File not found</title>
-<meta name="description" content="" />
-<meta name="keywords" content="mixdrop,share file,share video,upload file,upload video" />
-<meta property="og:title" content="MixDrop - File not found" />
-<meta property="og:description" content="" />
-<meta name="verified-code" content="035b8906-2381-4a3a-9d4c-3b2ae07293e4">
-<!--<meta name="robots" content="nosnippet">-->
-<meta name="a.validate.02" content="ctm1Vhh9zx0o7ew5xlkxmA8MjHBDjK2gY_5x" />
-<link rel="icon" type="image/png" href="/imgs/v2/favicon-32x32.png" sizes="32x32" />
-<link rel="icon" type="image/png" href="/imgs/v2/favicon-16x16.png" sizes="16x16" />
-<meta name="csrf" content="40758e7869c55031abee715dea547b6a">
-<meta name="clckd" content="ebd626d580ab61986c788e761c305e41" />
-<meta name="monetag" content="d1dda856d50c7dd239c4daa5c545e3d3">
-<meta name="41e891e66eaaca8fc8294c24cc35a2b1eee3aa09" content="41e891e66eaaca8fc8294c24cc35a2b1eee3aa09" />
-<link rel="stylesheet" href="/js/slidebars/slidebars.css?v=0.1" />
-<link rel="stylesheet" href="/css/style.v2.0.2.min.css" />
-<script src="https://code.jquery.com/jquery-3.6.4.min.js" integrity="sha256-oP6HI9z1XaZNBrJURtCoUT5SUnxFr8s3BzRl+cbzUq8=" crossorigin="anonymous"></script>
-</head>
-<body>
-<div class="container" canvas="container">
-	<div class="panel header">
-		<div class="wrapper">
-			<a href="/"><img src="/imgs/v2/logo.png" class="logo" alt="mixdrop" /></a>
-
-			
-	<div class="menu">
-		<a href="javascript:void(0)" class="btn btn1 openModal" data-modal="signup">SIGNUP</a>
-		<a href="javascript:void(0)" class="btn btn2 openModal" data-modal="login">LOGIN</a>
-	</div>			<div class="menu-mobile"><img src="/imgs/v2/menu.png" alt="menu"></div>
-		</div>
-	</div>
-<div class="panel download error txt-c">
-	<div class="wrapper">
-				<img src="/imgs/illustration-notfound.png" style="width: 250px;max-width: 80%;" />
-		<h2>WE ARE SORRY</h2>
-		<h3>We can't find the file you are looking for.</h3>
-				<script data-cfasync="false" async type="text/javascript" src="//xx.apptdinsteps.com/sBq05a7a7j1Yhgl/117018"></script>		<div style="padding: 30px 0">
-				
-						</div>
-			</div>
-</div>
-
-	<footer class="panel">
-		<div class="wrapper">
-			<div class="col-3">
-				<a href="/"><img src="/imgs/v2/logo.png" class="logo"  alt="mixdrop" /></a>
-				<a href="javascript:void(0)" class="openModal" data-modal="contact">Contact</a>
-				<a href="/news/">News</a>
-				<a href="/faq/">FAQ</a>
-			</div>
-			<div class="col-3">
-				<div class="hdr">POLICY</div>
-				<a href="/terms/">Terms And Conditions</a>
-				<a href="/privacy/">Privacy Policy</a>
-				<a href="/copyright/">Copyright Policy</a>
-				<a href="javascript:void(0)" class="openModal" data-modal="abuse">Report Abuse</a>
-			</div>
-			<div class="col-3">
-				<div class="hdr">SERVICES</div>
-				<a href="/panel/">User Panel</a>
-				<a href="/api/">API</a>
-			</div>
-			<!--
-			<div class="col-lang">
-				<div class="langSwitch">
-					<a href="javascript:void(0)" title="English"><img src="/imgs/flag_en.png" alt="english"> EN</a>
-					<div class="langOptions" style="display: none;">
-
-					</div>
-				</div>
-			</div>-->
-		</div>
-	</footer>
-</div>
-<div class="uploadbox">
-	<div class="header">Uploads ( <span>0 / 0</span> ) <i class="toggle"></i></div>
-	<div class="scroll">
-		<div class="tbl">
-		</div>
-	</div>
-</div>
-<div id="goUp" class="scrollTo floating" data-scrollto='.container'></div>
-<div class="modal" id="modal-login">
-	<div class="modal-wrapper">
-		<form action="/actions/global.php">
-			<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
-			<input type="password" name="password" placeholder="Password" class="ic ic-in-pass" />
-			<div class="2fa-content" style="display:none">
-				<label class="hdr">2FA Confirmation Code</label>
-
-				<p>A confirmation code was sent to your e-mail address, please confirm it below</p>
-				<input type="text" name="code" placeholder="" style="text-transform: uppercase;font-size: 2rem;letter-spacing: .5rem;" />
-			</div>
-			<button type="button" class="submit btn2">LOGIN</button>
-			<a href="javascript:void(0)" class="forgot">Forgot your password?</a>
-			<div class="mbError">&nbsp;</div>
-						<input type="hidden" name="step" value="0" />
-			<input type="hidden" name="a" value="login" />
-		</form>
-	</div>
-</div>
-
-<div class="modal" id="modal-recover">
-	<div class="modal-wrapper">
-		<form action="/actions/global.php">
-			<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
-			<button type="button" class="submit btn2">RECOVER</button>
-			<div class="mbError">&nbsp;</div>
-			<input type="hidden" name="a" value="recover" />
-		</form>
-	</div>
-</div>
-
-<div class="modal" id="modal-signup">
-	<div class="modal-wrapper">
-		<form action="/actions/global.php">
-						<input type="text" name="email" placeholder="E-Mail" class="ic ic-in-mail" />
-			<input type="password" name="password" placeholder="Password" class="ic ic-in-pass" />
-			<input type="password" name="passwordconfirm" placeholder="Repeat Password" class="ic ic-in-pass" />
-			<button type="button" class="submit btn2">SIGNUP</button>
-			<a href="javascript:void(0)" class="mlogin">Already have an account?</a>
-			<div class="mbError">&nbsp;</div>
-						<input type="hidden" name="a" value="signup" />
-		</form>
-	</div>
-</div>
-
-<div class="modal" id="modal-contact">
-	<div class="modal-wrapper">
-		<form action="/actions/global.php">
-			<input type="text" name="name" placeholder="Your name" class="ic ic-in-user" />
-			<input type="text" name="email" placeholder="Your E-Mail" class="ic ic-in-mail" />
-			<textarea name="message" placeholder="Your message" class="ic ic-in-compose" rows="6"></textarea>
-			<button type="button" class="submit btn2">SEND MESSAGE</button>
-			<div class="mbError">&nbsp;</div>
-			<input type="hidden" name="a" value="contact" />
-		</form>
-	</div>
-</div>
-
-<div class="modal" id="modal-abuse">
-	<div class="modal-wrapper">
-		<form action="/actions/global.php">
-			<select name="reason" class="ic ic-in-list">
-				<option>Reason</option>
-				<option value="copyright">Copyright infringement</option>
-				<option value="cp">Child Pornography</option>
-				<option value="terrorism">Terrorism Content</option>
-			</select>
-			<input type="text" name="name" placeholder="Your name" class="ic ic-in-user" />
-			<input type="text" name="email" placeholder="Your E-Mail" class="ic ic-in-mail" />
-			<textarea name="message" placeholder="Your message" class="ic ic-in-compose" rows="6"></textarea>
-			<button type="button" class="submit btn2">SEND MESSAGE</button>
-			<div class="mbError">&nbsp;</div>
-			<input type="hidden" name="a" value="abuse" />
-		</form>
-	</div>
-</div>
-
-<div off-canvas="menu right push" class="toggleMenu">
-		<a href="javascript:void(0)" class="openModal" data-modal="login">LOGIN</a>
-	<a href="javascript:void(0)" class="openModal" data-modal="signup">SIGNUP</a>
-		<a href="javascript:void(0)" class="scrollTo" data-scrollto=".landing">HOME</a>
-	<a href="/panel/">USER PANEL</a>
-	<a href="javascript:void(0)" class="scrollTo" data-scrollto=".share">SHARE</a>
-	<a href="javascript:void(0)" class="scrollTo" data-scrollto=".embed">EMBED</a>
-	<a href="" class="scrollTo" data-scrollto=".affiliate">AFFILIATE</a>
-	<a href="/faq/">FAQ</a>
-	<a href="/news/">NEWS</a>
-	<a href="javascript:void(0)" class="openModal" data-modal="contact">CONTACT</a>
-</div>
-<script src="https://code.jquery.com/ui/1.13.2/jquery-ui.min.js" integrity="sha256-lSjKY0/srUM9BE3dPm+c4fBo1dky2v27Gdjm2uoZaL0=" crossorigin="anonymous"></script>
-<script src="/js/modal/modal.js?v=2.0.1"></script>
-<script src="/js/slidebars/slidebars.min.js?v=2.0.1"></script>
-<script src="/js/circular-progress/circle-progress.min.js?v=2.0.1"></script>
-<script src="/js/jquery-upload/js/jquery.iframe-transport.js?v=2.0.1"></script>
-<script src="/js/jquery-upload/js/jquery.fileupload.js?v=2.0.1"></script>
-<script src="/js/jquery-upload/js/main.js?v=2.0.1"></script>
-<script src="/panel/js/scroll/perfect-scrollbar.min.js?v=2.0.1"></script>
-<link rel="stylesheet" href="/panel/js/scroll/perfect-scrollbar.css?v=2.0.1" />
-<script src="/js/script.v2.min.js?v=2.0.1"></script>
-<script src="/js/ads.js"></script>
-
-	<script defer data-domain="mixdrop.ag/f" src="https://mdstats.info/js/script.js"></script>
-</body>
-</html>