From f6303c1a6e99a723a8a10e130765952f9614a001 Mon Sep 17 00:00:00 2001 From: RavensRain Date: Sun, 11 Sep 2022 10:41:21 +0200 Subject: [PATCH 1/3] Ability to reduce unnecessary bandwidth usage --- @types/crunchyTypes.d.ts | 3 +- @types/messageHandler.d.ts | 4 ++- config/cli-defaults.yml | 3 +- crunchy.ts | 30 ++++++++++++++----- docs/DOCUMENTATION.md | 9 ++++++ .../src/serviceHandler/crunchyroll.ts | 2 +- .../DownloadSelector/DownloadSelector.tsx | 5 +++- gui/react/src/provider/Store.tsx | 2 ++ modules/module.app-args.ts | 2 +- modules/module.args.ts | 15 ++++++++++ 10 files changed, 61 insertions(+), 14 deletions(-) diff --git a/@types/crunchyTypes.d.ts b/@types/crunchyTypes.d.ts index c972acd..1f983fa 100644 --- a/@types/crunchyTypes.d.ts +++ b/@types/crunchyTypes.d.ts @@ -25,7 +25,8 @@ export type CrunchyDownloadOptions = { mkvmergeOptions: string[], defaultSub: LanguageItem, defaultAudio: LanguageItem, - ccTag: string + ccTag: string, + dlVideoOnce: boolean } export type CurnchyMultiDownload = { diff --git a/@types/messageHandler.d.ts b/@types/messageHandler.d.ts index 98ec628..323e94e 100644 --- a/@types/messageHandler.d.ts +++ b/@types/messageHandler.d.ts @@ -31,6 +31,7 @@ export type QueueItem = { season: string }, q: number, + dlVideoOnce: boolean, dubLang: string[], image: string } @@ -40,6 +41,7 @@ export type ResolveItemsData = { dubLang: string[], all: boolean, but: boolean, + dlVideoOnce: boolean, e: string, fileName: string, q: number, @@ -95,7 +97,7 @@ export type FuniStreamData = { force?: 'Y'|'y'|'N'|'n'|'C'|'c', callbackMaker?: forceMuxer: AvailableMuxer | undefined, simul: boolean, skipSubMux: boolean, nocleanup: boolean, override: string[], videoTitle: string, ffmpegOptions: string[], mkvmergeOptions: string[], defaultAudio: LanguageItem, defaultSub: LanguageItem, ccTag: string } export type FuniSubsData = { nosubs?: boolean, sub: boolean, dlsubs: string[], ccTag: string } -export type DownloadData = { id: string, e: string, dubLang: string[], dlsubs: string[], fileName: string, q: number, novids: boolean, noaudio: boolean } +export type DownloadData = { id: string, e: string, dubLang: string[], dlsubs: string[], fileName: string, q: number, novids: boolean, noaudio: boolean, dlVideoOnce: boolean } export type AuthResponse = ResponseBase; export type FuniSearchReponse = ResponseBase; diff --git a/config/cli-defaults.yml b/config/cli-defaults.yml index fe38a8b..b6940e3 100644 --- a/config/cli-defaults.yml +++ b/config/cli-defaults.yml @@ -1,4 +1,5 @@ q: 0 nServer: 1 mp4mux: false -noCleanUp: false \ No newline at end of file +noCleanUp: false +dlVideoOnce: true \ No newline at end of file diff --git a/crunchy.ts b/crunchy.ts index d3c9992..ec222a1 100644 --- a/crunchy.ts +++ b/crunchy.ts @@ -62,7 +62,7 @@ export default class Crunchy implements ServiceClass { public async cli() { console.log(`\n=== Multi Downloader NX ${packageJson.version} ===\n`); const argv = yargs.appArgv(this.cfg.cli); - + // load binaries this.cfg.bin = await yamlCfg.loadBinCfg(); if (argv.allDubs) { @@ -900,6 +900,7 @@ export default class Crunchy implements ServiceClass { } let dlFailed = false; + let dlVideoOnce = false; // Variable to save if best selected video quality was downloaded for (const mMeta of medias.data) { console.log(`[INFO] Requesting: [${mMeta.mediaId}] ${mediaName}`); @@ -1030,6 +1031,7 @@ export default class Crunchy implements ServiceClass { plQuality: { str: string, dim: string, + CODECS: string, RESOLUTION: { width: number, height: number @@ -1039,6 +1041,8 @@ export default class Crunchy implements ServiceClass { // set quality const plResolution = pl.attributes.RESOLUTION; const plResolutionText = `${plResolution.width}x${plResolution.height}`; + // set codecs + const plCodecs = pl.attributes.CODECS; // parse uri const plUri = new URL(pl.uri); let plServer = plUri.hostname; @@ -1072,27 +1076,36 @@ export default class Crunchy implements ServiceClass { plQuality.push({ str: qualityStrAdd, dim: plResolutionText, + CODECS: plCodecs, RESOLUTION: plResolution }); } } options.x = options.x > plServerList.length ? 1 : options.x; - + const plSelectedServer = plServerList[options.x - 1]; const plSelectedList = plStreams[plSelectedServer]; plQuality.sort((a, b) => { - const aMatch = a.dim.match(/[0-9]+/) || []; - const bMatch = b.dim.match(/[0-9]+/) || []; + const aMatch: RegExpMatchArray | never[] = a.dim.match(/[0-9]+/) || []; + const bMatch: RegExpMatchArray | never[] = b.dim.match(/[0-9]+/) || []; return parseInt(aMatch[0]) - parseInt(bMatch[0]); }); - let quality = options.q; - if (quality > plQuality.length) { + let quality = options.q === 0 ? plQuality.length : options.q; + if(quality > plQuality.length) { console.log(`[WARN] The requested quality of ${options.q} is greater than the maximun ${plQuality.length}.\n[WARN] Therefor the maximum will be capped at ${plQuality.length}.`); quality = plQuality.length; } - const selPlUrl = quality === 0 ? plSelectedList[plQuality[plQuality.length - 1].dim as string] : - plSelectedList[plQuality.map(a => a.dim)[quality - 1]] ? plSelectedList[plQuality.map(a => a.dim)[quality - 1]] : ''; + // When best selected video quality is already downloaded + if(dlVideoOnce && options.dlVideoOnce) { + // Select the lowest resolution with the same codecs + while(quality !=1 && plQuality[quality - 1].CODECS == plQuality[quality - 2].CODECS) { + console.log(quality); + console.log(plQuality[quality - 1].CODECS == plQuality[quality - 2].CODECS); + quality--; + } + } + const selPlUrl = plSelectedList[plQuality.map(a => a.dim)[quality - 1]] ? plSelectedList[plQuality.map(a => a.dim)[quality - 1]] : ''; console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`); console.log(`[INFO] Available qualities:\n\t${plQuality.map((a, ind) => `[${ind+1}] ${a.str}`).join('\n\t')}`); @@ -1162,6 +1175,7 @@ export default class Crunchy implements ServiceClass { path: `${tsFile}.ts`, lang: lang }); + dlVideoOnce = true; } } else{ diff --git a/docs/DOCUMENTATION.md b/docs/DOCUMENTATION.md index 0330180..518bbb6 100644 --- a/docs/DOCUMENTATION.md +++ b/docs/DOCUMENTATION.md @@ -120,6 +120,15 @@ For special episodes: S1-4 OR S1,S2,S3,S4 where S is the special letter | Both | `-q ${qualityLevel}` | `number` | `No`| `NaN` | `0`| `q: ` | Set the quality level. Use 0 to use the maximum quality. +#### `--dlVideoOnce` +| **Service** | **Usage** | **Type** | **Required** | **Alias** | **Default** |**cli-default Entry** +| --- | --- | --- | --- | --- | --- | ---| +| Both | `--dlVideoOnce ` | `boolean` | `No`| `NaN` | `true`| `dlVideoOnce: ` | + +If selected, the best selected quality will be downloaded only for the first language, +then the worst video quality with the same audio quality will be downloaded for every other language. +By the later merge of the videos, no quality difference will be present. +This will speed up the download speed, if multiple languages are selected. #### `-x` | **Service** | **Usage** | **Type** | **Required** | **Alias** | **Choices** | **Default** |**cli-default Entry** | --- | --- | --- | --- | --- | --- | --- | ---| diff --git a/gui/electron/src/serviceHandler/crunchyroll.ts b/gui/electron/src/serviceHandler/crunchyroll.ts index 9f99fa8..ddfa3dd 100644 --- a/gui/electron/src/serviceHandler/crunchyroll.ts +++ b/gui/electron/src/serviceHandler/crunchyroll.ts @@ -92,7 +92,7 @@ class CrunchyHandler extends Base implements MessageHandler { }); if (res.isOk) { for (const select of res.value) { - if (!(await this.crunchy.downloadEpisode(select, {..._default, skipsubs: false, callbackMaker: this.makeProgressHandler.bind(this), q: data.q, fileName: data.fileName, dlsubs: data.dlsubs, force: 'y', + if (!(await this.crunchy.downloadEpisode(select, {..._default, skipsubs: false, callbackMaker: this.makeProgressHandler.bind(this), q: data.q, fileName: data.fileName, dlsubs: data.dlsubs, dlVideoOnce: data.dlVideoOnce, force: 'y', novids: data.novids }))) { const er = new Error(`Unable to download episode ${data.e} from ${data.id}`); er.name = 'Download error'; diff --git a/gui/react/src/components/AddToQueue/DownloadSelector/DownloadSelector.tsx b/gui/react/src/components/AddToQueue/DownloadSelector/DownloadSelector.tsx index ce909c3..09ed9bb 100644 --- a/gui/react/src/components/AddToQueue/DownloadSelector/DownloadSelector.tsx +++ b/gui/react/src/components/AddToQueue/DownloadSelector/DownloadSelector.tsx @@ -27,7 +27,8 @@ const DownloadSelector: React.FC = ({ onFinish }) => { const subLang = messageHandler?.handleDefault('dlsubs'); const q = messageHandler?.handleDefault('q'); const fileName = messageHandler?.handleDefault('fileName'); - const result = await Promise.all([dubLang, subLang, q, fileName]); + const dlVideoOnce = messageHandler?.handleDefault('dlVideoOnce'); + const result = await Promise.all([dubLang, subLang, q, fileName, dlVideoOnce]); dispatch({ type: 'downloadOptions', payload: { @@ -36,6 +37,7 @@ const DownloadSelector: React.FC = ({ onFinish }) => { dlsubs: result[1], q: result[2], fileName: result[3], + dlVideoOnce: result[4], } }); setAvailableDubs(await messageHandler?.availableDubCodes() ?? []); @@ -142,6 +144,7 @@ const DownloadSelector: React.FC = ({ onFinish }) => { + List episodes diff --git a/gui/react/src/provider/Store.tsx b/gui/react/src/provider/Store.tsx index 20008c1..0bda8d8 100644 --- a/gui/react/src/provider/Store.tsx +++ b/gui/react/src/provider/Store.tsx @@ -9,6 +9,7 @@ export type DownloadOptions = { dubLang: typeof dubLanguageCodes, dlsubs: string[], fileName: string, + dlVideoOnce: boolean, all: boolean, but: boolean, novids: boolean, @@ -61,6 +62,7 @@ const initialState: StoreState = { dubLang: [ 'jpn' ], dlsubs: [ 'all' ], fileName: '', + dlVideoOnce: false, all: false, but: false, noaudio: false, diff --git a/modules/module.app-args.ts b/modules/module.app-args.ts index 40a52ee..140b1ba 100644 --- a/modules/module.app-args.ts +++ b/modules/module.app-args.ts @@ -2,7 +2,7 @@ import yargs, { Choices } from 'yargs'; import { args, AvailableMuxer, groups } from './module.args'; import { LanguageItem } from './module.langsData'; -let argvC: { [x: string]: unknown; ccTag: string, defaultAudio: LanguageItem, defaultSub: LanguageItem, ffmpegOptions: string[], mkvmergeOptions: string[], force: 'Y'|'y'|'N'|'n'|'C'|'c', skipUpdate: boolean, videoTitle: string, override: string[], fsRetryTime: number, forceMuxer: AvailableMuxer|undefined; username: string|undefined, password: string|undefined, silentAuth: boolean, skipSubMux: boolean, downloadArchive: boolean, addArchive: boolean, but: boolean, auth: boolean | undefined; dlFonts: boolean | undefined; search: string | undefined; 'search-type': string; page: number | undefined; 'search-locale': string; new: boolean | undefined; 'movie-listing': string | undefined; series: string | undefined; s: string | undefined; e: string | undefined; q: number; x: number; kstream: number; partsize: number; hslang: string; dlsubs: string[]; novids: boolean | undefined; noaudio: boolean | undefined; nosubs: boolean | undefined; dubLang: string[]; all: boolean; fontSize: number; allDubs: boolean; timeout: number; simul: boolean; mp4: boolean; skipmux: boolean | undefined; fileName: string; numbers: number; nosess: string; debug: boolean | undefined; nocleanup: boolean; help: boolean | undefined; service: 'funi' | 'crunchy'; update: boolean; fontName: string | undefined; _: (string | number)[]; $0: string; }; +let argvC: { [x: string]: unknown; ccTag: string, defaultAudio: LanguageItem, defaultSub: LanguageItem, ffmpegOptions: string[], mkvmergeOptions: string[], force: 'Y'|'y'|'N'|'n'|'C'|'c', skipUpdate: boolean, videoTitle: string, override: string[], fsRetryTime: number, forceMuxer: AvailableMuxer|undefined; username: string|undefined, password: string|undefined, silentAuth: boolean, skipSubMux: boolean, downloadArchive: boolean, addArchive: boolean, but: boolean, auth: boolean | undefined; dlFonts: boolean | undefined; search: string | undefined; 'search-type': string; page: number | undefined; 'search-locale': string; new: boolean | undefined; 'movie-listing': string | undefined; series: string | undefined; s: string | undefined; e: string | undefined; q: number; x: number; kstream: number; partsize: number; hslang: string; dlsubs: string[]; novids: boolean | undefined; noaudio: boolean | undefined; nosubs: boolean | undefined; dubLang: string[]; all: boolean; fontSize: number; allDubs: boolean; timeout: number; simul: boolean; mp4: boolean; skipmux: boolean | undefined; fileName: string; numbers: number; nosess: string; debug: boolean | undefined; nocleanup: boolean; help: boolean | undefined; service: 'funi' | 'crunchy'; update: boolean; fontName: string | undefined; _: (string | number)[]; $0: string; dlVideoOnce: boolean; }; export type ArgvType = typeof argvC; diff --git a/modules/module.args.ts b/modules/module.args.ts index 2d70de9..3948a03 100644 --- a/modules/module.args.ts +++ b/modules/module.args.ts @@ -176,6 +176,21 @@ const args: TAppArg[] = [ type: 'number', usage: '${qualityLevel}' }, + { + name: 'dlVideoOnce', + describe: 'Download only once the video with the best selected quality', + type: 'boolean', + group: 'dl', + service: 'both', + docDescribe: 'If selected, the best selected quality will be downloaded only for the first language,' + + '\nthen the worst video quality with the same audio quality will be downloaded for every other language.' + + '\nBy the later merge of the videos, no quality difference will be present.' + + '\nThis will speed up the download speed, if multiple languages are selected.', + usage: '', + default: { + default: true + } + }, { name: 'x', group: 'dl', -- 2.45.2 From cce7956b0710dcfd35990ce28f7a3bd1d2763a5a Mon Sep 17 00:00:00 2001 From: RavensRain Date: Sun, 11 Sep 2022 13:22:05 +0200 Subject: [PATCH 2/3] remove useless console.log and set dlVideoDownload as non-default --- config/cli-defaults.yml | 2 +- crunchy.ts | 2 -- modules/module.args.ts | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config/cli-defaults.yml b/config/cli-defaults.yml index b6940e3..85682dc 100644 --- a/config/cli-defaults.yml +++ b/config/cli-defaults.yml @@ -2,4 +2,4 @@ q: 0 nServer: 1 mp4mux: false noCleanUp: false -dlVideoOnce: true \ No newline at end of file +dlVideoOnce: false \ No newline at end of file diff --git a/crunchy.ts b/crunchy.ts index ec222a1..f4344f0 100644 --- a/crunchy.ts +++ b/crunchy.ts @@ -1100,8 +1100,6 @@ export default class Crunchy implements ServiceClass { if(dlVideoOnce && options.dlVideoOnce) { // Select the lowest resolution with the same codecs while(quality !=1 && plQuality[quality - 1].CODECS == plQuality[quality - 2].CODECS) { - console.log(quality); - console.log(plQuality[quality - 1].CODECS == plQuality[quality - 2].CODECS); quality--; } } diff --git a/modules/module.args.ts b/modules/module.args.ts index 3948a03..bf702f1 100644 --- a/modules/module.args.ts +++ b/modules/module.args.ts @@ -188,7 +188,7 @@ const args: TAppArg[] = [ + '\nThis will speed up the download speed, if multiple languages are selected.', usage: '', default: { - default: true + default: false } }, { -- 2.45.2 From 67ccb5daaaf2f05a11001c1c0a090df9ee8b597f Mon Sep 17 00:00:00 2001 From: RavensRain Date: Mon, 12 Sep 2022 19:14:51 +0200 Subject: [PATCH 3/3] set dlVideoOnce to crunchy only --- modules/module.args.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module.args.ts b/modules/module.args.ts index bf702f1..94b1351 100644 --- a/modules/module.args.ts +++ b/modules/module.args.ts @@ -181,7 +181,7 @@ const args: TAppArg[] = [ describe: 'Download only once the video with the best selected quality', type: 'boolean', group: 'dl', - service: 'both', + service: 'crunchy', docDescribe: 'If selected, the best selected quality will be downloaded only for the first language,' + '\nthen the worst video quality with the same audio quality will be downloaded for every other language.' + '\nBy the later merge of the videos, no quality difference will be present.' -- 2.45.2