perf(extractor): do not guess size from mp4 for DoodStream
This commit is contained in:
WebStreamr
parent
06e0ba23d7
commit
b8c5c1a4f3
23 changed files with 49 additions and 54 deletions
|
|
@ -14,10 +14,6 @@ describe('DoodStream', () => {
|
|||
expect(await extractorRegistry.handle(ctx, new URL('http://dood.to/e/sk1m9eumzyjj'))).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('doodster', async () => {
|
||||
expect(await extractorRegistry.handle(ctx, new URL('https://dooodster.com/e/1cfcevn6dg8shrfvht22odxw2lty18hr'))).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('missing pass_md5 -> not found', async () => {
|
||||
expect(await extractorRegistry.handle(ctx, new URL('https://dood.to/e/gy8l8mb2i311'))).toMatchSnapshot();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import bytes from 'bytes';
|
||||
import * as cheerio from 'cheerio';
|
||||
import randomstring from 'randomstring';
|
||||
import { NotFoundError } from '../error';
|
||||
import { Context, Format, Meta, UrlResult } from '../types';
|
||||
import { guessSizeFromMp4 } from '../utils/size';
|
||||
import { Extractor } from './Extractor';
|
||||
|
||||
export class DoodStream extends Extractor {
|
||||
|
|
@ -40,13 +40,14 @@ export class DoodStream extends Extractor {
|
|||
const $ = cheerio.load(html);
|
||||
const title = $('title').text().trim().replace(/ - DoodStream$/, '').trim();
|
||||
|
||||
const downloadHtml = await this.fetcher.text(ctx, new URL(url.href.replace('/e/', '/d/')));
|
||||
const sizeMatch = downloadHtml.match(/([\d.]+ ?[GM]B)/);
|
||||
|
||||
let mp4Url: URL;
|
||||
let bytes: number | undefined;
|
||||
if (baseUrl.includes('cloudflarestorage')) {
|
||||
mp4Url = new URL(baseUrl);
|
||||
} else {
|
||||
mp4Url = new URL(`${baseUrl}${randomstring.generate(10)}?token=${token}&expiry=${Date.now()}`);
|
||||
bytes = await guessSizeFromMp4(ctx, this.fetcher, mp4Url, { headers: { Referer: url.href } });
|
||||
}
|
||||
|
||||
return [
|
||||
|
|
@ -59,7 +60,7 @@ export class DoodStream extends Extractor {
|
|||
meta: {
|
||||
...meta,
|
||||
title,
|
||||
...(bytes && { bytes }),
|
||||
...(sizeMatch && { bytes: bytes.parse(sizeMatch[1] as string) as number }),
|
||||
},
|
||||
requestHeaders: {
|
||||
Referer: url.origin,
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
{}
|
||||
1
src/extractor/__fixtures__/DoodStream/http:dood.toddfx8me4un4ul
generated
Normal file
1
src/extractor/__fixtures__/DoodStream/http:dood.toddfx8me4un4ul
generated
Normal file
File diff suppressed because one or more lines are too long
1
src/extractor/__fixtures__/DoodStream/http:dood.todsk1m9eumzyjj
generated
Normal file
1
src/extractor/__fixtures__/DoodStream/http:dood.todsk1m9eumzyjj
generated
Normal file
File diff suppressed because one or more lines are too long
1
src/extractor/__fixtures__/DoodStream/http:dood.todwfpwtsgyr1xi
generated
Normal file
1
src/extractor/__fixtures__/DoodStream/http:dood.todwfpwtsgyr1xi
generated
Normal file
|
|
@ -0,0 +1 @@
|
|||
<!doctype html><html lang="en"><head> <meta charset="utf-8"> <title>Video not found | DoodStream</title> <link rel="preconnect" href="//i.doodcdn.io"><link rel="preconnect" href="//cdnjs.cloudflare.com"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script> <link rel="stylesheet" href="//i.doodcdn.io/theme_2/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="//i.doodcdn.io/theme_2/css/style.css" /></head><body> <style type="text/css"> .not_found img {height:250px !important;} @media (max-width: 768px){.not_found h1{font-size:1.8rem;}.not_found img {height:200px !important;margin: 0 auto 15px;}} @media (max-width: 480px){.not_found h1{font-size:1.4rem;}.not_found img {height:120px !important;margin: 0 auto 15px;}.not_found p{margin-top: -5px;}} </style> <div class="container pt-5 not_found"> <div class="row"> <div class="col-md-12 pt-2 text-center"> <img src="//i.doodcdn.io/img/no_video_3.svg"> <h1>Not Found</h1> <p>video you are looking for is not found.</p> </div> </div> </div><script data-cfasync="false" src="//d1f05vr3sjsuy7.cloudfront.net/?srvfd=908056"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/gHzOaAdOhbZ/71405"></script><script data-cfasync="false" async type="text/javascript" src="//faqirsgoliard.top/r67c0fc81985e5/70849"></script><script type="text/javascript" data-cfasync="false">/*<![CDATA[/* */(function(){var g=window,r="ca0967284e67bc53cb5b626985619403",w=[["siteId",317+185*82*507-3600486],["minBid",0],["popundersPerIP","0"],["delayBetween",300],["default","https://strettechoco.com/iYa7dlULceKw/27615"],["defaultPerDay",0],["topmostLayer","never"]],n=["d3d3LmJsb2NrYWRzbm90LmNvbS9iYWphLm1pbi5jc3M=","ZG5oZmk1bm4yZHQ2Ny5jbG91ZGZyb250Lm5ldC9RVHd0eC91YWxnZWJyYS5taW4uanM=","d3d3LmJibHZwbGd1b21hLmNvbS9mYWphLm1pbi5jc3M=","d3d3LndqZ3BzcGt4empiZGd1LmNvbS9NTmlIWS9rYWxnZWJyYS5taW4uanM="],f=-1,v,q,h=function(){clearTimeout(q);f++;if(n[f]&&!(1766620801000<(new Date).getTime()&&1<f)){v=g.document.createElement("script");v.type="text/javascript";v.async=!0;var o=g.document.getElementsByTagName("script")[0];v.src="https://"+atob(n[f]);v.crossOrigin="anonymous";v.onerror=h;v.onload=function(){clearTimeout(q);g[r.slice(0,16)+r.slice(0,16)]||h()};q=setTimeout(h,5E3);o.parentNode.insertBefore(v,o)}};if(!g[r]){try{Object.freeze(g[r]=w)}catch(e){}h()}})();/*]]>/* */</script></body></html>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
https://ijs155l.cloudatacdn.com/u5kj2rz5qtdlsdgge4hrgj2flz5cjodea6auatveaphxogr2vvukxfzmwzva/egprvdjvpr~
|
||||
|
|
@ -6,6 +6,7 @@ exports[`DoodStream can guess height from title 1`] = `
|
|||
"format": "mp4",
|
||||
"label": "DoodStream",
|
||||
"meta": {
|
||||
"bytes": 549139251,
|
||||
"countryCodes": [],
|
||||
"title": "Black Mirror S04E02 FRENCH 720p WEB x264-RiPiT",
|
||||
},
|
||||
|
|
@ -44,7 +45,7 @@ exports[`DoodStream dood.to 1`] = `
|
|||
"format": "mp4",
|
||||
"label": "DoodStream",
|
||||
"meta": {
|
||||
"bytes": 791702409,
|
||||
"bytes": 1395864371,
|
||||
"countryCodes": [],
|
||||
"title": "des-teufels-bad-2024",
|
||||
},
|
||||
|
|
@ -58,23 +59,4 @@ exports[`DoodStream dood.to 1`] = `
|
|||
]
|
||||
`;
|
||||
|
||||
exports[`DoodStream doodster 1`] = `
|
||||
[
|
||||
{
|
||||
"format": "mp4",
|
||||
"label": "DoodStream",
|
||||
"meta": {
|
||||
"countryCodes": [],
|
||||
"title": "Brokeback Mountain 2005",
|
||||
},
|
||||
"requestHeaders": {
|
||||
"Referer": "http://dood.to",
|
||||
},
|
||||
"sourceId": "doodstream_",
|
||||
"ttl": 21600000,
|
||||
"url": "https://ijs155l.cloudatacdn.com/u5kj2rz5qtdlsdgge4hrgj2flz5cjodea6auatveaphxogr2vvukxfzmwzva/egprvdjvpr~mocked-random-string?token=fbsg3301phkef2s4swo8bssg&expiry=639837296000",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`DoodStream missing pass_md5 -> not found 1`] = `[]`;
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
{"server":"nginx","date":"Sun, 08 Jun 2025 19:20:59 GMT","content-type":"video/mp4","content-length":791702409,"last-modified":"Wed, 03 Apr 2024 09:53:31 GMT","connection":"keep-alive","etag":"660d271b-2f306b89","access-control-allow-origin":"*","accept-ranges":"bytes"}
|
||||
|
|
@ -1 +0,0 @@
|
|||
{}
|
||||
1
src/utils/__fixtures__/StreamResolver/http:dood.tod1srct94x5yfz
generated
Normal file
1
src/utils/__fixtures__/StreamResolver/http:dood.tod1srct94x5yfz
generated
Normal file
File diff suppressed because one or more lines are too long
1
src/utils/__fixtures__/StreamResolver/http:dood.tod37o04m33q3g5
generated
Normal file
1
src/utils/__fixtures__/StreamResolver/http:dood.tod37o04m33q3g5
generated
Normal file
File diff suppressed because one or more lines are too long
1
src/utils/__fixtures__/StreamResolver/http:dood.todsk1m9eumzyjj
generated
Normal file
1
src/utils/__fixtures__/StreamResolver/http:dood.todsk1m9eumzyjj
generated
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
https://do365ki.cloudatacdn.com/u5kjzcovmpd3sdgge7tc4jacdzaikwer6ff4qd463b6qj6lodlgpnsbkffvq/xy59ygfkku~
|
||||
|
|
@ -1 +1 @@
|
|||
https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~
|
||||
https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rmcr6fv2tcniineq/6aqs1w2xla~
|
||||
|
|
@ -0,0 +1 @@
|
|||
https://et1095c.cloudatacdn.com/u5kjxj7s27d3sdgge5woezkbi7milmjrckehcrwuw63gsvytfwxlfmjzvu3a/jli1t787d1~
|
||||
|
|
@ -1 +0,0 @@
|
|||
https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~
|
||||
|
|
@ -237,12 +237,12 @@ exports[`resolve returns sorted results 1`] = `
|
|||
"Referer": "http://dood.to",
|
||||
},
|
||||
},
|
||||
"videoSize": 791702409,
|
||||
"videoSize": 1395864371,
|
||||
},
|
||||
"name": "WebStreamr 🇩🇪",
|
||||
"title": "des-teufels-bad-2024
|
||||
💾 755.03 MB 🔗 DoodStream from MeineCloud",
|
||||
"url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000",
|
||||
💾 1.3 GB 🔗 DoodStream from MeineCloud",
|
||||
"url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rmcr6fv2tcniineq/6aqs1w2xla~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000",
|
||||
},
|
||||
{
|
||||
"behaviorHints": {
|
||||
|
|
@ -253,11 +253,12 @@ exports[`resolve returns sorted results 1`] = `
|
|||
"Referer": "http://dood.to",
|
||||
},
|
||||
},
|
||||
"videoSize": 1181116006,
|
||||
},
|
||||
"name": "WebStreamr 🇮🇹",
|
||||
"title": "the-devils-bath-2024-sd-sub-ita
|
||||
🔗 DoodStream from MostraGuarda",
|
||||
"url": "https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000",
|
||||
💾 1.1 GB 🔗 DoodStream from MostraGuarda",
|
||||
"url": "https://et1095c.cloudatacdn.com/u5kjxj7s27d3sdgge5woezkbi7milmjrckehcrwuw63gsvytfwxlfmjzvu3a/jli1t787d1~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -317,12 +318,12 @@ exports[`resolve returns sorted results 2`] = `
|
|||
"Referer": "http://dood.to",
|
||||
},
|
||||
},
|
||||
"videoSize": 791702409,
|
||||
"videoSize": 1395864371,
|
||||
},
|
||||
"name": "WebStreamr 🇩🇪",
|
||||
"title": "des-teufels-bad-2024
|
||||
💾 755.03 MB 🔗 DoodStream from MeineCloud",
|
||||
"url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rlyusvlvslr3u3ta/bibyo6w0dm~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000",
|
||||
💾 1.3 GB 🔗 DoodStream from MeineCloud",
|
||||
"url": "https://aa360cc.cloudatacdn.com/u5kjv4jxytd3sdgge5uogji5dg4huat2pxrm2qibdbm5rmcr6fv2tcniineq/6aqs1w2xla~mocked-random-string?token=7uebebipnnhusa4xnyea1er4&expiry=639837296000",
|
||||
},
|
||||
{
|
||||
"behaviorHints": {
|
||||
|
|
@ -333,11 +334,12 @@ exports[`resolve returns sorted results 2`] = `
|
|||
"Referer": "http://dood.to",
|
||||
},
|
||||
},
|
||||
"videoSize": 1181116006,
|
||||
},
|
||||
"name": "WebStreamr 🇮🇹",
|
||||
"title": "the-devils-bath-2024-sd-sub-ita
|
||||
🔗 DoodStream from MostraGuarda",
|
||||
"url": "https://kk892as.cloudatacdn.com/u5kj7j7s27d3sdgge5woezkbi4pu672wktq3aqujw47rbx4declm2mk2k2gq/4msi5kvncy~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000",
|
||||
💾 1.1 GB 🔗 DoodStream from MostraGuarda",
|
||||
"url": "https://et1095c.cloudatacdn.com/u5kjxj7s27d3sdgge5woezkbi7milmjrckehcrwuw63gsvytfwxlfmjzvu3a/jli1t787d1~mocked-random-string?token=aw2v2d8uhbrj2ky54d573ujk&expiry=639837296000",
|
||||
},
|
||||
{
|
||||
"behaviorHints": {
|
||||
|
|
@ -381,6 +383,22 @@ exports[`resolve returns source errors as stream 2`] = `
|
|||
|
||||
exports[`resolve returns ytId instead of url for YouTube video 1`] = `
|
||||
[
|
||||
{
|
||||
"behaviorHints": {
|
||||
"bingeGroup": "webstreamr-doodstream_en",
|
||||
"notWebReady": true,
|
||||
"proxyHeaders": {
|
||||
"request": {
|
||||
"Referer": "http://dood.to",
|
||||
},
|
||||
},
|
||||
"videoSize": 1932735283,
|
||||
},
|
||||
"name": "WebStreamr 🇺🇸",
|
||||
"title": "Solaris 1972 PROPER 1080p BluRay x264-SADPANDA - ENG SUBS - tt0069293
|
||||
💾 1.8 GB 🔗 DoodStream from PrimeWire",
|
||||
"url": "https://do365ki.cloudatacdn.com/u5kjzcovmpd3sdgge7tc4jacdzaikwer6ff4qd463b6qj6lodlgpnsbkffvq/xy59ygfkku~mocked-random-string?token=aej8q7ajug1gt0nd4lbs8zgd&expiry=639837296000",
|
||||
},
|
||||
{
|
||||
"behaviorHints": {
|
||||
"bingeGroup": "webstreamr-youtube_en",
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
import { Context } from '../types';
|
||||
import { CustomRequestInit, Fetcher } from './Fetcher';
|
||||
|
||||
export const guessSizeFromMp4 = async (ctx: Context, fetcher: Fetcher, url: URL, init?: CustomRequestInit): Promise<number | undefined> => {
|
||||
export const guessSizeFromMp4 = async (ctx: Context, fetcher: Fetcher, url: URL, init?: CustomRequestInit): Promise<number> => {
|
||||
const mp4Head = await fetcher.head(ctx, url, init);
|
||||
|
||||
if (mp4Head['content-length']) {
|
||||
return parseInt(mp4Head['content-length'] as string);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
return parseInt(mp4Head['content-length'] as string);
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue