feat(extractor): try to semi-blindly implement DoodStream support

package-lock.json

@@ -16,6 +16,7 @@
         "cheerio": "^1.0.0",
         "country-emoji": "^1.5.6",
         "express": "^5.1.0",
+        "randomstring": "^1.3.1",
         "slugify": "^1.6.6",
         "unpacker": "^1.0.1",
         "user-agents": "^1.1.536",
@@ -29,6 +30,7 @@
         "@types/jest": "^29.5.14",
         "@types/make-fetch-happen": "^10.0.4",
         "@types/node": "^22.15.3",
+        "@types/randomstring": "^1.3.0",
         "@types/stremio-addon-sdk": "^1.6.11",
         "@types/user-agents": "^1.0.4",
         "axios-mock-adapter": "^2.1.0",
@@ -1634,6 +1636,13 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/randomstring": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@types/randomstring/-/randomstring-1.3.0.tgz",
+      "integrity": "sha512-kCP61wludjY7oNUeFiMxfswHB3Wn/aC03Cu82oQsNTO6OCuhVN/rCbBs68Cq6Nkgjmp2Sh3Js6HearJPkk7KQA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/range-parser": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
@@ -6128,6 +6137,30 @@
       ],
       "license": "MIT"
     },
+    "node_modules/randombytes": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
+      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
+      "license": "MIT",
+      "dependencies": {
+        "safe-buffer": "^5.1.0"
+      }
+    },
+    "node_modules/randomstring": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/randomstring/-/randomstring-1.3.1.tgz",
+      "integrity": "sha512-lgXZa80MUkjWdE7g2+PZ1xDLzc7/RokXVEQOv5NN2UOTChW1I8A9gha5a9xYBOqgaSoI6uJikDmCU8PyRdArRQ==",
+      "license": "MIT",
+      "dependencies": {
+        "randombytes": "2.1.0"
+      },
+      "bin": {
+        "randomstring": "bin/randomstring"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/range-parser": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",

package.json

@@ -37,6 +37,7 @@
     "cheerio": "^1.0.0",
     "country-emoji": "^1.5.6",
     "express": "^5.1.0",
+    "randomstring": "^1.3.1",
     "slugify": "^1.6.6",
     "unpacker": "^1.0.1",
     "user-agents": "^1.1.536",
@@ -50,6 +51,7 @@
     "@types/jest": "^29.5.14",
     "@types/make-fetch-happen": "^10.0.4",
     "@types/node": "^22.15.3",
+    "@types/randomstring": "^1.3.0",
     "@types/stremio-addon-sdk": "^1.6.11",
     "@types/user-agents": "^1.0.4",
     "axios-mock-adapter": "^2.1.0",

src/embed-extractor/DoodStream.ts

@@ -0,0 +1,50 @@
+import randomstring from 'randomstring';
+import { EmbedExtractor } from './types';
+import { Fetcher } from '../utils';
+import { Context } from '../types';
+
+// DoodStream does not return the pass_md5 from some IPs like e.g. Oracle cloud
+// In such cases a VPN might be needed
+export class DoodStream implements EmbedExtractor {
+  readonly id = 'doodstream';
+
+  readonly label = 'DoodStream';
+
+  readonly ttl = 900000; // 15m
+
+  private readonly fetcher: Fetcher;
+
+  constructor(fetcher: Fetcher) {
+    this.fetcher = fetcher;
+  }
+
+  readonly supports = (url: URL): boolean => null !== url.host.match(/dood/);
+
+  readonly extract = async (ctx: Context, url: URL, countryCode: string) => {
+    const videoId = url.pathname.split('/').slice(-1)[0] as string;
+    const normalizedUrl = new URL(`http://dood.to/e/${videoId}`);
+
+    const html = await this.fetcher.text(ctx, new URL(normalizedUrl));
+
+    const passMd5Match = html.match(/\/pass_md5\/[\w-]+\/([\w-]+)/);
+    if (!passMd5Match) {
+      return undefined;
+    }
+
+    const token = passMd5Match[1] as string;
+
+    const baseUrl = await this.fetcher.text(ctx, new URL(`http://dood.to${passMd5Match[0]}`));
+
+    return {
+      url: new URL(`${baseUrl + randomstring.generate(10)}?token=${token}&expiry=${Date.now()}`),
+      label: this.label,
+      sourceId: `${this.id}_${countryCode.toLowerCase()}`,
+      height: 0,
+      bytes: 0,
+      countryCode,
+      requestHeaders: {
+        Referer: 'http://dood.to/',
+      },
+    };
+  };
+}

src/embed-extractor/EmbedExtractorRegistry.ts

@@ -3,6 +3,7 @@ import winston from 'winston';
 import { EmbedExtractor } from './types';
 import { Context, UrlResult } from '../types';
 import { Fetcher } from '../utils';
+import { DoodStream } from './DoodStream';
 import { Dropload } from './Dropload';
 import { SuperVideo } from './SuperVideo';
 
@@ -14,6 +15,7 @@ export class EmbedExtractorRegistry {
   constructor(logger: winston.Logger, fetcher: Fetcher) {
     this.logger = logger;
     this.embedExtractors = [
+      new DoodStream(fetcher),
       new Dropload(fetcher),
       new SuperVideo(fetcher),
     ];
@@ -34,7 +36,9 @@ export class EmbedExtractorRegistry {
     this.logger.info(`Extract stream URL using ${embedExtractor.id} extractor from ${url}`);
 
     urlResult = await embedExtractor.extract(ctx, url, countryCode);
-    this.urlResultCache.set(url.href, urlResult, { ttl: embedExtractor.ttl });
+    if (urlResult) {
+      this.urlResultCache.set(url.href, urlResult, { ttl: embedExtractor.ttl });
+    }
 
     return urlResult;
   };

src/embed-extractor/types.ts

@@ -9,5 +9,5 @@ export interface EmbedExtractor {
 
   readonly supports: (url: URL) => boolean;
 
-  readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise<UrlResult>;
+  readonly extract: (ctx: Context, url: URL, countryCode: string) => Promise<UrlResult | undefined>;
 }

src/handler/FrenchCloud.test.ts

@@ -27,7 +27,7 @@ describe('FrenchCloud', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(2);
+    expect(streams).toHaveLength(3);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -46,5 +46,17 @@ describe('FrenchCloud', () => {
       countryCode: 'fr',
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      label: 'DoodStream',
+      sourceId: 'doodstream_fr',
+      height: 0,
+      bytes: 0,
+      countryCode: 'fr',
+      requestHeaders: {
+        Referer: 'http://dood.to/',
+      },
+    });
+    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/handler/MeineCloud.test.ts

@@ -27,7 +27,7 @@ describe('MeineCloud', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(2);
+    expect(streams).toHaveLength(3);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -46,5 +46,17 @@ describe('MeineCloud', () => {
       countryCode: 'de',
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      label: 'DoodStream',
+      sourceId: 'doodstream_de',
+      height: 0,
+      bytes: 0,
+      countryCode: 'de',
+      requestHeaders: {
+        Referer: 'http://dood.to/',
+      },
+    });
+    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/handler/MostraGuarda.test.ts

@@ -27,7 +27,7 @@ describe('MostraGuarda', () => {
   test('handle imdb the devil\'s bath', async () => {
     const streams = (await handler.handle(ctx, 'movie', 'tt29141112')).filter(stream => stream !== undefined);
 
-    expect(streams).toHaveLength(2);
+    expect(streams).toHaveLength(3);
     expect(streams[0]).toStrictEqual({
       url: expect.any(URL),
       label: 'SuperVideo',
@@ -46,5 +46,17 @@ describe('MostraGuarda', () => {
       countryCode: 'it',
     });
     expect(streams[1]?.url.href).toMatch(/^https:\/\/.*?.m3u8/);
+    expect(streams[2]).toStrictEqual({
+      url: expect.any(URL),
+      label: 'DoodStream',
+      sourceId: 'doodstream_it',
+      height: 0,
+      bytes: 0,
+      countryCode: 'it',
+      requestHeaders: {
+        Referer: 'http://dood.to/',
+      },
+    });
+    expect(streams[2]?.url.href).toMatch(/^https:\/\/.*?token.*?expiry/);
   });
 });

src/types.ts

@@ -12,5 +12,6 @@ export interface UrlResult {
   sourceId: string;
   height: number;
   bytes: number;
-  countryCode: string | undefined;
+  countryCode?: string;
+  requestHeaders?: Record<string, string>;
 }

src/utils/StreamResolver.test.ts

@@ -87,6 +87,34 @@ describe('resolve', () => {
           group: 'webstreamr-supervideo_de',
         },
       },
+      {
+        url: expect.any(String),
+        name: 'WebStreamr',
+        title: 'DoodStream | 🇩🇪',
+        behaviourHints: {
+          group: 'webstreamr-doodstream_de',
+          notWebReady: true,
+          proxyHeaders: {
+            request: {
+              Referer: 'http://dood.to/',
+            },
+          },
+        },
+      },
+      {
+        url: expect.any(String),
+        name: 'WebStreamr',
+        title: 'DoodStream | 🇮🇹',
+        behaviourHints: {
+          group: 'webstreamr-doodstream_it',
+          notWebReady: true,
+          proxyHeaders: {
+            request: {
+              Referer: 'http://dood.to/',
+            },
+          },
+        },
+      },
     ]);
   });
 });

src/utils/StreamResolver.ts

@@ -78,6 +78,10 @@ export class StreamResolver {
           title,
           behaviourHints: {
             group: `webstreamr-${urlResult.sourceId}`,
+            ...(urlResult.requestHeaders !== undefined && {
+              notWebReady: true,
+              proxyHeaders: { request: urlResult.requestHeaders },
+            }),
           },
         };
       }),

src/utils/__fixtures__/Fetcher/http:dood.toegy8l8mb2i311

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head> <meta charset="utf-8"> <title>Video not found | DoodStream</title> <link rel="preconnect" href="//i.doodcdn.io"><link rel="preconnect" href="//cdnjs.cloudflare.com"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script> <link rel="stylesheet" href="//i.doodcdn.io/theme_2/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="//i.doodcdn.io/theme_2/css/style.css" /></head><body> <style type="text/css"> .not_found img {height:250px !important;} @media (max-width: 768px){.not_found h1{font-size:1.8rem;}.not_found img {height:200px !important;margin: 0 auto 15px;}} @media (max-width: 480px){.not_found h1{font-size:1.4rem;}.not_found img {height:120px !important;margin: 0 auto 15px;}.not_found p{margin-top: -5px;}} </style> <div class="container pt-5 not_found"> <div class="row"> <div class="col-md-12 pt-2 text-center"> <img src="//i.doodcdn.io/img/no_video_3.svg"> <h1>Not Found</h1> <p>video you are looking for is not found.</p> </div> </div> </div><script data-cfasync="false" src="//d1f05vr3sjsuy7.cloudfront.net/?srvfd=908056"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/gHzOaAdOhbZ/71405"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/r67c0fc81985e5/70849"></script><script type="text/javascript" data-cfasync="false">/*<![CDATA[/* */(function(){var g=window,r="ca0967284e67bc53cb5b626985619403",w=[["siteId",317+185*82*507-3600486],["minBid",0],["popundersPerIP","0"],["delayBetween",300],["default","https://strettechoco.com/iYa7dlULceKw/27615"],["defaultPerDay",0],["topmostLayer","never"]],n=["d3d3LmJsb2NrYWRzbm90LmNvbS9iYWphLm1pbi5jc3M=","ZG5oZmk1bm4yZHQ2Ny5jbG91ZGZyb250Lm5ldC9RVHd0eC91YWxnZWJyYS5taW4uanM=","d3d3LmJibHZwbGd1b21hLmNvbS9mYWphLm1pbi5jc3M=","d3d3LndqZ3BzcGt4empiZGd1LmNvbS9NTmlIWS9rYWxnZWJyYS5taW4uanM="],f=-1,v,q,h=function(){clearTimeout(q);f++;if(n[f]&&!(1766620801000<(new Date).getTime()&&1<f)){v=g.document.createElement("script");v.type="text/javascript";v.async=!0;var o=g.document.getElementsByTagName("script")[0];v.src="https://"+atob(n[f]);v.crossOrigin="anonymous";v.onerror=h;v.onload=function(){clearTimeout(q);g[r.slice(0,16)+r.slice(0,16)]||h()};q=setTimeout(h,5E3);o.parentNode.insertBefore(v,o)}};if(!g[r]){try{Object.freeze(g[r]=w)}catch(e){}h()}})();/*]]>/* */</script></body></html>

src/utils/__fixtures__/Fetcher/http:dood.toerw7rxdfrbg09

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head> <meta charset="utf-8"> <title>Video not found | DoodStream</title> <link rel="preconnect" href="//i.doodcdn.io"><link rel="preconnect" href="//cdnjs.cloudflare.com"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script> <link rel="stylesheet" href="//i.doodcdn.io/theme_2/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="//i.doodcdn.io/theme_2/css/style.css" /></head><body> <style type="text/css"> .not_found img {height:250px !important;} @media (max-width: 768px){.not_found h1{font-size:1.8rem;}.not_found img {height:200px !important;margin: 0 auto 15px;}} @media (max-width: 480px){.not_found h1{font-size:1.4rem;}.not_found img {height:120px !important;margin: 0 auto 15px;}.not_found p{margin-top: -5px;}} </style> <div class="container pt-5 not_found"> <div class="row"> <div class="col-md-12 pt-2 text-center"> <img src="//i.doodcdn.io/img/no_video_3.svg"> <h1>Not Found</h1> <p>video you are looking for is not found.</p> </div> </div> </div><script data-cfasync="false" src="//d1f05vr3sjsuy7.cloudfront.net/?srvfd=908056"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/gHzOaAdOhbZ/71405"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/r67c0fc81985e5/70849"></script><script type="text/javascript" data-cfasync="false">/*<![CDATA[/* */(function(){var g=window,r="ca0967284e67bc53cb5b626985619403",w=[["siteId",317+185*82*507-3600486],["minBid",0],["popundersPerIP","0"],["delayBetween",300],["default","https://strettechoco.com/iYa7dlULceKw/27615"],["defaultPerDay",0],["topmostLayer","never"]],n=["d3d3LmJsb2NrYWRzbm90LmNvbS9iYWphLm1pbi5jc3M=","ZG5oZmk1bm4yZHQ2Ny5jbG91ZGZyb250Lm5ldC9RVHd0eC91YWxnZWJyYS5taW4uanM=","d3d3LmJibHZwbGd1b21hLmNvbS9mYWphLm1pbi5jc3M=","d3d3LndqZ3BzcGt4empiZGd1LmNvbS9NTmlIWS9rYWxnZWJyYS5taW4uanM="],f=-1,v,q,h=function(){clearTimeout(q);f++;if(n[f]&&!(1766620801000<(new Date).getTime()&&1<f)){v=g.document.createElement("script");v.type="text/javascript";v.async=!0;var o=g.document.getElementsByTagName("script")[0];v.src="https://"+atob(n[f]);v.crossOrigin="anonymous";v.onerror=h;v.onload=function(){clearTimeout(q);g[r.slice(0,16)+r.slice(0,16)]||h()};q=setTimeout(h,5E3);o.parentNode.insertBefore(v,o)}};if(!g[r]){try{Object.freeze(g[r]=w)}catch(e){}h()}})();/*]]>/* */</script></body></html>

src/utils/__fixtures__/Fetcher/http:dood.topass_md5164387212-80-155-1747404408-4733f0a25bf777152a8aca9615644da37uebebipnnhusa4xnyea1er4

@@ -0,0 +1 @@
+https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rtpmbzgb5tornooa/k9wk0js17e~

src/utils/__fixtures__/Fetcher/http:dood.topass_md5165649626-80-155-1747405113-cbcdd8c31b71609b43725432d299545caw2v2d8uhbrj2ky54d573ujk

@@ -0,0 +1 @@
+https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbpl3xog2gtqnw2xa/awn9rgotuu~

src/utils/__fixtures__/Fetcher/http:dood.topass_md5204946325-80-155-1747404729-174faab43b2ec6d6f27d9bd6ba46ddbey0n5z6g48kft9apq8imziegv

@@ -0,0 +1 @@
+https://ty1053vs.cloudatacdn.com/u5kjz2cvh7blsdgge4mmgoifjigorrf6o6mlfwqbndqkhkcvu43dcrk7wfga/6zwayqucuf~