add format selection to most services

README.md

@@ -5,12 +5,13 @@
 ## Changes made
 
 - Name change
+- Format selection (90% done, Atmos needs to be added still)
 
 ### Planned changes
 
 - Focus less on the individual streaming service and give best priority to the highest quality rip available
 - Format selection
-	- Yes, including Dolby Atmos and whatever weird streaming format you prefer, so long as it is provided directly by the ripping service.
+  - Yes, including Dolby Atmos and whatever weird streaming format you prefer, so long as it is provided directly by the ripping service.
 - Client selection
 
 ## Usage
@@ -109,4 +110,4 @@ Types used across the project. The purpose of many of these is to make sure all
 
 As stated above, verdana is a fork of [Lucida](https://codeberg.org/lucida/lucida).
 
-Both Lucida and verdana are partially inspired by [OrpheusDL](https://github.com/yarrm80s/orpheusdl), a Python program for music archival which can be used similarly to verdana/Lucida. Some scripts inside verdana/Lucida are modeled after OrpheusDL modules.
+Both Lucida and verdana are partially inspired by [OrpheusDL](https://github.com/yarrm80s/orpheusdl), a Python program for music archival which can be used similarly to verdana/Lucida. Some scripts inside verdana/Lucida are modeled after OrpheusDL modules.

src/index.ts

@@ -101,12 +101,15 @@ class Verdana {
 		const moduleNames = Object.keys(this.modules)
 		return Object.fromEntries(results.map((e, i) => [moduleNames[i], e]))
 	}
-	async getFormat(id: string): Promise<{ [key: string]: Format | null }> {
-		const results = await Promise.all(
-			Object.values(this.modules).map((e) => e.getFormat(id))
-		)
-		const moduleNames = Object.keys(this.modules)
-		return Object.fromEntries(results.map((e, i) => [moduleNames[i], e]))
+	async getFormatsByUrl(url: string): Promise<Format[]> {
+		const urlObj = new URL(url)
+		for (const i in this.modules) {
+			const matches = this.modules[i].hostnames.includes(urlObj.hostname)
+			if (!matches) continue
+
+			return this.modules[i].getFormats(url)
+		}
+		throw new Error(`Couldn't find module for hostname ${urlObj.hostname}`)
 	}
 }
 

src/streamers/deezer/main.ts

@@ -9,7 +9,8 @@ import {
 	SearchResults,
 	StreamerAccount,
 	StreamerWithLogin,
-	Track
+	Track,
+	Format
 } from '../../types.js'
 import { BLOWFISH_SECRET, CLIENT_ID, CLIENT_SECRET, GW_LIGHT_URL } from './constants.js'
 import { createHash } from 'crypto'
@@ -64,7 +65,7 @@ interface APIMethod {
 }
 
 export default class Deezer implements StreamerWithLogin {
-	hostnames = ['deezer.com', 'www.deezer.com', 'deezer.page.link']
+	hostnames = ['deezer.com', 'www.deezer.com', 'link.deezer.com']
 	testData = {
 		'https://www.deezer.com/us/artist/1194083': {
 			type: 'artist',
@@ -273,15 +274,19 @@ export default class Deezer implements StreamerWithLogin {
 
 	async #unshortenUrl(url: URL): Promise<URL> {
 		const res = await fetch(url, { redirect: 'manual', dispatcher: this.dispatcher })
-		const location = res.headers.get('Location')
+		const location = res.headers.get('location')
 
-		if (res.status != 302 || !location) throw new Error('URL not supported')
+		if (!location) throw new Error('URL not supported')
 
-		return new URL(location)
+		const decoded = new URL(location).searchParams.get('awf')
+
+		if (!decoded) throw new Error('Could not decode link.deezer.com link')
+
+		return new URL(decoded)
 	}
 
 	async #getInfoFromUrl(url: URL): Promise<{ type: ItemType; id: number }> {
-		if (url.hostname == 'deezer.page.link') url = await this.#unshortenUrl(url)
+		if (url.hostname == 'link.deezer.com') url = await this.#unshortenUrl(url)
 
 		const match = url.pathname.match(/^\/(?:[a-z]{2}\/)?(track|album|artist|playlist)\/(\d+)\/?$/)
 		if (!match) throw new Error('URL not supported')
@@ -359,7 +364,7 @@ export default class Deezer implements StreamerWithLogin {
 		})
 	}
 
-	async #getStream(track: DeezerTrack): Promise<GetStreamResponse> {
+	async #getStream(track: DeezerTrack, format: DeezerFormat = 9): Promise<GetStreamResponse> {
 		if ('FALLBACK' in track) track = track.FALLBACK!
 
 		const countries = track?.AVAILABLE_COUNTRIES?.STREAM_ADS
@@ -367,36 +372,23 @@ export default class Deezer implements StreamerWithLogin {
 		if (!countries.includes(this.country!))
 			throw new Error("Track not available in the account's country")
 
-		let format: DeezerFormat = DeezerFormat.MP3_128
-		const formatsToCheck = [
-			{
-				format: DeezerFormat.FLAC,
-				filesize: track.FILESIZE_FLAC
-			},
-			{
-				format: DeezerFormat.MP3_320,
-				filesize: track.FILESIZE_MP3_320
-			}
-		]
-		for (const f of formatsToCheck) {
-			if (f.filesize != '0' && this.availableFormats.has(f.format)) {
-				format = f.format
-				break
-			}
-		}
+		if (!this.availableFormats.has(format)) throw new Error('Format is not available.')
 
 		const id = track.SNG_ID
 		const trackToken = track.TRACK_TOKEN
 		const trackTokenExpiry = track.TRACK_TOKEN_EXPIRE
 
-		let mimeType = ''
+		let mimeType
+		let extension
 		switch (format) {
 			case DeezerFormat.MP3_128:
 			case DeezerFormat.MP3_320:
 				mimeType = 'audio/mpeg'
+				extension = 'mp3'
 				break
 			case DeezerFormat.FLAC:
 				mimeType = 'audio/flac'
+				extension = 'flac'
 		}
 
 		// download
@@ -465,7 +457,8 @@ export default class Deezer implements StreamerWithLogin {
 				.on('error', function (e) {
 					throw new Error('Error while decrypting track stream:' + e)
 				}),
-			mimeType
+			mimeType,
+			extension
 		}
 	}
 
@@ -535,8 +528,8 @@ export default class Deezer implements StreamerWithLogin {
 				return {
 					type: 'track',
 					metadata: parseTrack(trackPage.DATA, trackPage.LYRICS),
-					getStream: () => {
-						return this.#getStream(trackPage.DATA)
+					getStream: (formatId: DeezerFormat) => {
+						return this.#getStream(trackPage.DATA, formatId)
 					}
 				}
 			}
@@ -568,4 +561,43 @@ export default class Deezer implements StreamerWithLogin {
 			return track
 		} else throw new Error(`Not available on Deezer.`)
 	}
+
+	async getFormats(url: string): Promise<Format[]> {
+		const { type, id } = await this.#getInfoFromUrl(new URL(url))
+		if (type != 'track') throw new Error('URL not recognized')
+		const trackPage = await this.#getTrackPage(id)
+
+		const formats: Format[] = []
+		const trackData = trackPage.DATA
+
+		const readableFormats = [
+			{ size: trackData.FILESIZE_MP3_128, format: 1 },
+			{ size: trackData.FILESIZE_MP3_320, format: 3 },
+			{ size: trackData.FILESIZE_FLAC, format: 9 }
+		]
+
+		for (const i in readableFormats) {
+			if (readableFormats[i].size != '0') {
+				formats.push({
+					id: readableFormats[i].format,
+					rippable: true,
+					size: parseInt(readableFormats[i].size),
+					...formatHelper(readableFormats[i].format)
+				} as Format)
+			}
+		}
+
+		return formats
+	}
+}
+
+function formatHelper(format: DeezerFormat) {
+	switch (format) {
+		case DeezerFormat.MP3_128:
+			return { extension: 'mp3', mimeType: 'audio/mp3', bitrate: 128 }
+		case DeezerFormat.MP3_320:
+			return { extension: 'mp3', mimeType: 'audio/mp3', bitrate: 320 }
+		case DeezerFormat.FLAC:
+			return { extension: 'flac', mimeType: 'audio/flac' }
+	}
 }

src/streamers/deezer/parse.ts

@@ -136,6 +136,7 @@ export interface DeezerTrack {
 	MD5_ORIGIN: string
 	MEDIA_VERSION: string
 
+	FILESIZE_MP3_128: string
 	FILESIZE_MP3_320: string
 	FILESIZE_FLAC: string
 

src/streamers/qobuz/main.ts

@@ -7,7 +7,8 @@ import {
 	GetByUrlResponse,
 	GetStreamResponse,
 	Track,
-	StreamerAccount
+	StreamerAccount,
+	Format
 } from '../../types.js'
 import { DEFAULT_HEADERS } from './constants.js'
 import {
@@ -174,12 +175,12 @@ export default class Qobuz implements StreamerWithLogin {
 		}
 	}
 
-	async #getFileUrl(trackId: string, qualityId = 27): Promise<GetStreamResponse> {
+	async #getFileUrl(trackId: string, formatId: number = 27): Promise<GetStreamResponse> {
 		if (!this.token) throw new Error('Not logged in.')
 
 		const params: { [key: string]: string } = {
 			track_id: trackId.toString(),
-			format_id: qualityId.toString(),
+			format_id: formatId.toString(),
 			intent: 'stream',
 			sample: 'false',
 			app_id: this.appId,
@@ -198,6 +199,7 @@ export default class Qobuz implements StreamerWithLogin {
 		const streamResponse = await fetch(trackFileResponse.url, { dispatcher: this.dispatcher })
 		return {
 			mimeType: trackFileResponse.mime_type,
+			extension: formatId == 5 ? 'mp3' : 'flac',
 			sizeBytes: parseInt(<string>streamResponse.headers.get('Content-Length')),
 			stream: Readable.fromWeb(streamResponse.body!)
 		}
@@ -205,10 +207,10 @@ export default class Qobuz implements StreamerWithLogin {
 
 	async #getTrackMetadata(trackId: string) {
 		return parseTrack(
-			<RawTrack>await this.#get('track/get', {
+			(await this.#get('track/get', {
 				track_id: trackId,
 				app_id: this.appId
-			})
+			})) as RawTrack
 		)
 	}
 
@@ -347,4 +349,45 @@ export default class Qobuz implements StreamerWithLogin {
 			return track
 		}
 	}
+	async getFormats(url: string): Promise<Format[]> {
+		const [type, id] = this.#getUrlParts(url)
+		if (type != 'track') throw new Error('URL not recognized.')
+
+		// get raw track
+		const rawTrack = (await this.#get('track/get', {
+			track_id: id,
+			app_id: this.appId
+		})) as RawTrack
+
+		// define all rippable formats
+		const acceptable = [
+			{ id: 5, rippable: true, bitrate: 320, extension: 'mp3', mimeType: 'audio/mp3' },
+			{ id: 6, rippable: true, bitDepth: 16, extension: 'flac', mimeType: 'audio/flac' },
+			{
+				id: 7,
+				rippable: true,
+				bitDepth: 24,
+				sampleRate: 96,
+				extension: 'flac',
+				mimeType: 'audio/flac'
+			},
+			{
+				id: 27,
+				rippable: true,
+				bitDepth: 24,
+				sampleRate: 192,
+				extension: 'flac',
+				mimeType: 'audio/flac'
+			}
+		] as Format[]
+
+		// slice by maximum rate provided by qobuz
+		let maxIndex: number | undefined
+		if (rawTrack.maximum_bit_depth == 16) maxIndex = 2
+		else if (rawTrack.maximum_bit_depth == 24) maxIndex = 3
+		else if (rawTrack.maximum_sampling_rate == 192) maxIndex = 4
+		else maxIndex = 1
+
+		return acceptable.slice(0, maxIndex)
+	}
 }

src/streamers/qobuz/parse.ts

@@ -97,17 +97,21 @@ export function parseAlbum(raw: RawAlbum) {
 
 export interface RawTrack {
 	title: string
-	version?: string
+	version: string
 	id: number
-	copyright?: string
+	copyright: string
 	performer: RawArtist
-	album?: RawAlbum
-	track_number?: number
-	media_number?: number
+	album: RawAlbum
+	track_number: number
+	media_number: number
 	duration: number
 	parental_warning: boolean
 	isrc: string
-	performers?: string
+	performers: string
+	audio_info: { replaygain_track_gain: number; replaygain_track_peak: 1 }
+	maximum_sampling_rate: number
+	maximum_bit_depth: number
+	maximum_channel_count: number
 }
 
 export function parseTrack(raw: RawTrack): Track {

src/streamers/soundcloud/main.ts

@@ -1,3 +1,5 @@
+// todo: rewrite. this is just terrible.
+
 import { Dispatcher, fetch, HeadersInit } from 'undici'
 import { DEFAULT_HEADERS } from './constants.js'
 import {
@@ -9,6 +11,8 @@ import {
 	StreamerAccount,
 	TrackGetByUrlResponse,
 	Format
+	TrackGetByUrlResponse,
+	Format
 } from '../../types.js'
 import {
 	parseAlbum,
@@ -517,3 +521,14 @@ export default class Soundcloud implements Streamer {
 		return formats.reverse()
 	}
 }
+
+function getExtensionFromMime(mime: string) {
+	switch (mime) {
+		case 'audio/mp4':
+			return 'm4a'
+		case 'audio/flac':
+			return 'flac'
+		default:
+			throw new Error(`Unable to resolve extension from mime type: ${mime}`)
+	}
+}

src/streamers/spotify/main.ts

@@ -8,6 +8,7 @@ import {
 	parsePlaylist
 } from './parse.js'
 import {
+	Format,
 	GetByUrlResponse,
 	ItemType,
 	SearchResults,
@@ -94,6 +95,7 @@ class Spotify implements StreamerWithLogin {
 						const streamData = await this.client.get.trackStream(id)
 						return {
 							mimeType: 'audio/ogg',
+							extension: 'ogg',
 							sizeBytes: streamData.sizeBytes,
 							stream: streamData.stream
 						}
@@ -136,6 +138,7 @@ class Spotify implements StreamerWithLogin {
 						const streamData = await this.client.get.episodeStream(id)
 						return {
 							mimeType: 'audio/ogg',
+							extension: 'ogg',
 							sizeBytes: streamData.sizeBytes,
 							stream: streamData.stream
 						}
@@ -189,6 +192,9 @@ class Spotify implements StreamerWithLogin {
 			explicit: info.allowExplicit
 		}
 	}
+	async getFormats(url: string): Promise<Format[]> {
+		return []
+	}
 }
 
 export default Spotify

src/streamers/tidal/constants.ts

@@ -1,3 +1,13 @@
+import { TidalFormat } from './parse.js'
+
 export const TIDAL_AUTH_BASE = 'https://auth.tidal.com/v1/'
 export const TIDAL_API_BASE = 'https://listen.tidal.com/v1/'
 export const TIDAL_SUBSCRIPTION_BASE = 'https://api.tidal.com/v1/'
+
+export const TIDAL_FORMATS = [
+	'LOW',
+	'HIGH',
+	'LOSSLESS',
+	'HI_RES',
+	'HI_RES_LOSSLESS'
+] as TidalFormat[]

src/streamers/tidal/main.ts

@@ -9,15 +9,22 @@ import {
 	SearchResults,
 	Streamer,
 	Track,
-	StreamerAccount
+	StreamerAccount,
+	Format
 } from '../../types.js'
-import { TIDAL_AUTH_BASE, TIDAL_API_BASE, TIDAL_SUBSCRIPTION_BASE } from './constants.js'
+import {
+	TIDAL_AUTH_BASE,
+	TIDAL_API_BASE,
+	TIDAL_SUBSCRIPTION_BASE,
+	TIDAL_FORMATS
+} from './constants.js'
 import {
 	Contributor,
 	RawAlbum,
 	RawArtist,
 	RawPlaylist,
 	RawTrack,
+	TidalFormat,
 	addCredits,
 	parseAlbum,
 	parseArtist,
@@ -61,7 +68,7 @@ interface SubscriptionData {
 		type: string
 		offlineGracePeriod: number
 	}
-	highestSoundQuality: string
+	highestSoundQuality: TidalFormat
 	premiumAccess: boolean
 	canGetTrial: boolean
 	paymentType: string
@@ -96,6 +103,7 @@ export default class Tidal implements Streamer {
 	userId: number | undefined
 	dispatcher: Dispatcher | undefined
 	sessionId: string | undefined
+	highestQuality: TidalFormat | undefined
 	hostnames = ['tidal.com', 'www.tidal.com', 'listen.tidal.com']
 	testData = {
 		'https://tidal.com/browse/artist/3908662': {
@@ -357,12 +365,12 @@ export default class Tidal implements Streamer {
 	}
 	async #getFileUrl(
 		trackId: number | string,
-		quality = 'HI_RES_LOSSLESS'
+		format: TidalFormat = 'HI_RES_LOSSLESS'
 	): Promise<GetStreamResponse> {
 		interface PlaybackInfo {
 			manifest: string
 			manifestMimeType: string
-			audioQuality: 'LOW' | 'HIGH' | 'LOSSLESS' | 'HI_RES' | 'HI_RES_LOSSLESS'
+			audioQuality: TidalFormat
 		}
 
 		if (!this.sessionId) await this.getAccountInfo()
@@ -372,7 +380,7 @@ export default class Tidal implements Streamer {
 			{
 				playbackmode: 'STREAM',
 				assetpresentation: 'FULL',
-				audioquality: quality,
+				audioquality: format,
 				prefetch: 'false',
 				sessionId: this.sessionId as string
 			}
@@ -389,6 +397,7 @@ export default class Tidal implements Streamer {
 			const streamResponse = await fetch(manifest.urls[0], { dispatcher: this.dispatcher })
 			return {
 				mimeType: manifest.mimeType,
+				extension: getExtensionFromMime(manifest.mimeType),
 				sizeBytes: parseInt(<string>streamResponse.headers.get('Content-Length')),
 				stream: Readable.fromWeb(streamResponse.body!)
 			}
@@ -434,6 +443,7 @@ export default class Tidal implements Streamer {
 						if (code == 0)
 							resolve({
 								mimeType: 'audio/mp4',
+								extension: 'm4a',
 								stream: fs.createReadStream(args[args.length - 1]).once('end', async function () {
 									await fs.promises.rm(folder, { recursive: true })
 								})
@@ -468,12 +478,13 @@ export default class Tidal implements Streamer {
 
 				return {
 					mimeType: 'audio/flac',
+					extension: 'flac',
 					stream: ffmpegProc.stdout
 				}
 			}
 		}
 	}
-	async #getFfArgs(audioQuality: 'LOW' | 'HIGH' | 'LOSSLESS' | 'HI_RES' | 'HI_RES_LOSSLESS') {
+	async #getFfArgs(audioQuality: TidalFormat) {
 		switch (audioQuality) {
 			case 'LOW':
 			case 'HIGH':
@@ -519,8 +530,8 @@ export default class Tidal implements Streamer {
 			case 'track':
 				return {
 					type,
-					getStream: () => {
-						return this.#getFileUrl(id)
+					getStream: (format: TidalFormat = 'HI_RES_LOSSLESS') => {
+						return this.#getFileUrl(id, format)
 					},
 					metadata: await this.#getTrack(id)
 				}
@@ -562,6 +573,7 @@ export default class Tidal implements Streamer {
 			await this.#get(`users/${this.userId}/subscription`, {}, TIDAL_SUBSCRIPTION_BASE)
 		)
 
+		this.highestQuality = subscription.highestSoundQuality
 		return {
 			valid: true,
 			premium: subscription.premiumAccess,
@@ -569,4 +581,46 @@ export default class Tidal implements Streamer {
 			explicit: true
 		}
 	}
+	async getFormats(url: string): Promise<Format[]> {
+		const [type, id] = this.#getUrlParts(url)
+		if (type != 'track') throw new Error('URL unrecognized')
+
+		// get raw track info
+		const trackResponse = <RawTrack>await this.#get(`tracks/${id}`)
+		const highestTrackQuality = TIDAL_FORMATS.indexOf(trackResponse.audioQuality) + 1
+
+		// if we don't have it already, get the account highest quality, to mark certain tracks as rippable
+		if (!this.highestQuality) await this.getAccountInfo()
+		const highestAccountQuality = TIDAL_FORMATS.indexOf(this.highestQuality as TidalFormat) + 1
+
+		let formats: Format[] = []
+		const acceptable = TIDAL_FORMATS.slice(0, highestTrackQuality)
+
+		for (const i in acceptable) {
+			let bitrate
+			if (acceptable[i] == 'LOW') bitrate = 96
+			if (acceptable[i] == 'HIGH') bitrate = 320
+
+			formats.push({
+				id: acceptable[i] as TidalFormat,
+				rippable: parseInt(i) < highestAccountQuality,
+				mimeType: acceptable[i] == 'LOW' || acceptable[i] == 'HIGH' ? 'audio/mp4' : 'audio/flac',
+				extension: acceptable[i] == 'LOW' || acceptable[i] == 'HIGH' ? 'm4a' : 'flac',
+				bitrate
+			})
+		}
+
+		return formats
+	}
+}
+
+function getExtensionFromMime(mime: string) {
+	switch (mime) {
+		case 'audio/mp4':
+			return 'm4a'
+		case 'audio/flac':
+			return 'flac'
+		default:
+			throw new Error(`Unable to resolve extension from mime type: ${mime}`)
+	}
 }

src/streamers/tidal/parse.ts

@@ -1,6 +1,8 @@
 import { Album, Artist, Playlist, Track } from '../../types.js'
 import { DOMParser } from 'xmldom-qsa'
 
+export type TidalFormat = 'LOW' | 'HIGH' | 'LOSSLESS' | 'HI_RES' | 'HI_RES_LOSSLESS'
+
 export interface RawArtist {
 	id: string
 	url?: string
@@ -137,6 +139,7 @@ export interface RawTrack {
 	title: string
 	version?: string
 	album: RawAlbum
+	audioQuality: TidalFormat
 }
 
 export function parseTrack(raw: RawTrack): Track {

src/types.ts

@@ -114,16 +114,22 @@ export interface SearchResults {
 }
 
 export interface Format {
-	id: string;
-	extension: string;
-	bitrate: number;
-	atmos: boolean;
+	id: string | number
+	rippable: boolean // if it looks like the stream is rippable
+	size?: number
+	extension?: string
+	mimeType?: string
+	sampleRate?: number
+	bitDepth?: number // {x}-bit FLAC
+	bitrate?: number // {x}kbps MP3
+	atmos?: boolean
 }
 
 export interface GetStreamResponse {
 	sizeBytes?: number
 	stream: NodeJS.ReadableStream
 	mimeType: string
+	extension: string
 }
 
 // got a better name for this?
@@ -137,28 +143,33 @@ export type GetByUrlResponse =
 
 export interface TrackGetByUrlResponse {
 	type: 'track'
-	getStream(format?: string | null): Promise<GetStreamResponse>
+	getStream(formatId?: string | number): Promise<GetStreamResponse>
 	metadata: Track
 }
+
 export interface EpisodeGetByUrlResponse {
 	type: 'episode'
-	getStream(format?: string | null): Promise<GetStreamResponse>
+	getStream(formatId?: string | number): Promise<GetStreamResponse>
 	metadata: Episode
 }
+
 export interface ArtistGetByUrlResponse {
 	type: 'artist'
 	metadata: Artist
 }
+
 export interface AlbumGetByUrlResponse {
 	type: 'album'
 	tracks: Track[]
 	metadata: Album
 }
+
 export interface PlaylistGetByUrlResponse {
 	type: 'playlist'
 	tracks: Track[]
 	metadata: Playlist
 }
+
 export interface PodcastGetByUrlResponse {
 	type: 'podcast'
 	episodes: Episode[]
@@ -190,7 +201,7 @@ export interface Streamer {
 		| ((url: string, limit?: number) => Promise<GetByUrlResponse>)
 	disconnect?(): Promise<void>
 	getAccountInfo(): Promise<StreamerAccount>
-	getFormat?(id: string): Promise<Format>
+	getFormats(url: string): Promise<Format[]>
 }
 
 export interface StreamerWithLogin extends Streamer {