From b8da59086b1c244705d4914aedeb501fc5eeebe7 Mon Sep 17 00:00:00 2001 From: reidlab Date: Sat, 26 Apr 2025 18:40:13 -0700 Subject: [PATCH] =?UTF-8?q?downlederigo=E0=B3=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++-- config.example.toml | 9 +++- flake.nix | 2 + package-lock.json | 7 +++ package.json | 1 + src/cache.ts | 87 +++++++++++++++++++++++++++++++++--- src/config.ts | 2 + src/downloader/index.ts | 50 +++++++++++++++++++++ src/downloader/keygen.ts | 2 +- src/downloader/streamInfo.ts | 21 ++++++--- 10 files changed, 170 insertions(+), 19 deletions(-) create mode 100644 src/downloader/index.ts diff --git a/README.md b/README.md index 28fb5c4..128fc14 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,12 @@ a self-hostable web-ui apple music downloader widevine decryptor with questionab `WIDEVINE_CLIENT_ID` is uhm owie. this thing kind of Sucks to obtain and i would totally recommend finding a not-so-legal spot you can obtain this from (in fact, i found one on github LOL), rather than extracting it yourself. if you want to do through the pain like i did, check [this guide](forum.videohelp.com/threads/408031-Dumping-Your-own-L3-CDM-with-Android-Studio) out!! once you have your `client_id.bin` file, convert it to base64 and slap it in the env var (`cat client_id.bin | base64 -w 0`) -`WIDEVINE_PRIVATE_KEY` is essentially the same process of obtainment, you'll get it from the same guide!! i'm not sure how to easily find one of these on the web, but i'm sure y'all end users (user count: 0 (TRVTHNVKE)) can pull through. this is also in base64 (`cat private_key.pem | base64 -w 0`) +`WIDEVINE_PRIVATE_KEY` is essentially the same process of obtainment, you'll get it from the same guide!! i'm not sure how to easily find one of these on the web, but i'm sure you end users (user count: 0 (TRVTHNVKE)) can pull through. this is also in base64 (`cat private_key.pem | base64 -w 0`) ### config most of the config is talked on in [`config.example.toml`](./config.example.toml), just copy it over to `config.toml` and go wild! i tried to make the error reporting for invalid configurations pretty good and digestable - +## limitations / the formats -## the formats - -currently you can only get basic widevine ones +currently you can only get basic widevine ones, everything related to playready and fairplay is not supported, sorry!! someday i will get this working, at least for playready. it's just that no one has written a library yet diff --git a/config.example.toml b/config.example.toml index 230f467..b959732 100644 --- a/config.example.toml +++ b/config.example.toml @@ -4,6 +4,12 @@ port = 2000 [downloader] +# path to the ffmpeg binary +# will get from PATH if simply "ffmpeg" +ffmpeg_path = "ffmpeg" +# path to the yt-dlp binary +# will get from PATH if simply "yt-dlp" +ytdlp_path = "yt-dlp" [downloader.cache] # where to store downloaded files (music, lyrics, etc.) @@ -13,5 +19,6 @@ directory = "cache" ttl = 3600 # (1 hour) [downloader.api] -# two letter language code (ISO 639-1) and two letter country code (ISO 3166-1 alpha-2) +# two letter language code (ISO 639-1), followed by a dash (-) and a two letter country code (ISO 3166-1 alpha-2) +# recommended to use something similar to your `ITUA` env variable language = "en-US" diff --git a/flake.nix b/flake.nix index bac324a..0f12677 100644 --- a/flake.nix +++ b/flake.nix @@ -38,6 +38,8 @@ packages = with pkgs; [ nodejs nodePackages.npm + + ffmpeg yt-dlp ]; }; }); diff --git a/package-lock.json b/package-lock.json index 5a840fe..d3bce2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "parse-hls": "^1.0.7", "pssh-tools": "^1.2.0", "source-map-support": "^0.5.21", + "timeago.js": "^4.0.2", "toml": "^3.0.0", "zod": "^3.24.2", "zod-validation-error": "^3.4.0" @@ -3488,6 +3489,12 @@ "dev": true, "license": "MIT" }, + "node_modules/timeago.js": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/timeago.js/-/timeago.js-4.0.2.tgz", + "integrity": "sha512-a7wPxPdVlQL7lqvitHGGRsofhdwtkoSXPGATFuSOA2i1ZNQEPLrGnj68vOp2sOJTCFAQVXPeNMX/GctBaO9L2w==", + "license": "MIT" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index bc7b16f..b790938 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "parse-hls": "^1.0.7", "pssh-tools": "^1.2.0", "source-map-support": "^0.5.21", + "timeago.js": "^4.0.2", "toml": "^3.0.0", "zod": "^3.24.2", "zod-validation-error": "^3.4.0" diff --git a/src/cache.ts b/src/cache.ts index 40b5249..283a8bb 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,23 +1,98 @@ import fs from "node:fs"; import path from "node:path"; +import timeago from "timeago.js"; import { config } from "./config.js"; import * as log from "./log.js"; +// DO NOT READ FURTHER INTO THIS FILE +// COGNITIVE DISSONANCE WARNING + +// TODO: hourly cache reports +// TODO: swap to sqlite +// TODO: make async fs calls +// TODO: rework EVERYTHING + interface CacheEntry { fileName: string; - expiry: number; + expiry: number; // milliseconds, not seconds } -const ttl = config.downloader.cache.ttl * 1000; -const file = path.join(config.downloader.cache.directory, "cache.json"); +const cacheTtl = config.downloader.cache.ttl * 1000; +const cacheFile = path.join(config.downloader.cache.directory, "cache.json"); if (!fs.existsSync(config.downloader.cache.directory)) { log.debug("cache directory not found, creating it"); fs.mkdirSync(config.downloader.cache.directory, { recursive: true }); } -if (!fs.existsSync(file)) { +if (!fs.existsSync(cacheFile)) { log.debug("cache file not found, creating it"); - fs.writeFileSync(file, JSON.stringify([]), { encoding: "utf-8" }); + fs.writeFileSync(cacheFile, JSON.stringify([]), { encoding: "utf-8" }); } -// SUPER TODO: implement this +let cache = JSON.parse(fs.readFileSync(cacheFile, { encoding: "utf-8" })) as CacheEntry[]; + +// TODO: change how this works +// this is so uncomfy +cache.push = function(...items: CacheEntry[]): number { + for (const entry of items) { + log.debug(`cache entry ${entry.fileName} added, expires ${timeago.format(entry.expiry)}`); + setTimeout(() => { + log.debug(`cache entry ${entry.fileName} expired, cleaning`); + removeCacheEntry(entry.fileName); + rewriteCache(); + }, entry.expiry - Date.now()); + } + + return Array.prototype.push.apply(this, items); +}; + +function rewriteCache(): void { + // cache is in fact []. i checked + fs.writeFileSync(cacheFile, JSON.stringify(cache), { encoding: "utf-8" }); +} + +function removeCacheEntry(fileName: string): void { + cache = cache.filter((entry) => { return entry.fileName !== fileName; }); + try { + fs.unlinkSync(path.join(config.downloader.cache.directory, fileName)); + } catch (err) { + log.error(`could not remove cache entry ${fileName}`); + log.error("this could result in 2 effects:"); + log.error("1. the cache entry will be removed, and the file never existed, operation is perfect, ignore this"); + log.error("2. the cache entry will be removed, but the file exists, so it will remain in the filesystem"); + log.error("if you experience the latter, the manual deletion of the file is required to fix this."); + } +} + +// clear cache entries that are expired +// this is for when the program is killed when cache entries are present +// those could expire while the program is not running, therefore not being cleaned +let expiryLogPrinted = false; +for (const entry of cache) { + if (entry.expiry < Date.now()) { + if (!expiryLogPrinted) { log.info("old expired cache entries are present, cleaning them"); } + expiryLogPrinted = true; + log.debug(`cache entry ${entry.fileName} expired ${timeago.format(entry.expiry)}; cleaning`); + removeCacheEntry(entry.fileName); + rewriteCache(); + } +} + +export function isCached(fileName: string): boolean { + const entry = cache.find((e) => { return e.fileName === fileName; }); + const cached = entry !== undefined && entry.expiry > Date.now(); + if (cached) { log.debug(`cache HIT for ${fileName}`); } + else { log.debug(`cache MISS for ${fileName}`); } + return cached; +} + +export function addToCache(fileName: string): void { + cache.push({ + fileName: fileName, + expiry: Date.now() + cacheTtl + }); + rewriteCache(); +} + +// setTimeout(() => addToCache("jorry.tx"), 1000); + diff --git a/src/config.ts b/src/config.ts index 9106e6b..0ff5ee7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -12,6 +12,8 @@ const configSchema = z.object({ port: z.number().int().min(0).max(65535).or(z.string()) }), downloader: z.object({ + ffmpeg_path: z.string(), + ytdlp_path: z.string(), cache: z.object({ directory: z.string(), ttl: z.number().int().min(0) diff --git a/src/downloader/index.ts b/src/downloader/index.ts new file mode 100644 index 0000000..c62fcd1 --- /dev/null +++ b/src/downloader/index.ts @@ -0,0 +1,50 @@ +import { config } from "../config.js"; +import { spawn } from "node:child_process"; +import path from "node:path"; +import { addToCache, isCached } from "../cache.js"; + +// TODO: make this have a return type +export async function downloadSong(streamUrl: string, decryptionKey: string): Promise { + const baseOutputName = streamUrl.split("/").at(-1)?.split("?").at(0)?.split(".").splice(0, 1).join(".")?.trim(); + if (!baseOutputName) { throw "could not get base output name from stream url"; } + const encryptedName = baseOutputName + "_enc.mp4"; + const encryptedPath = path.join(config.downloader.cache.directory, encryptedName); + const decryptedName = baseOutputName + ".mp4"; + const decryptedPath = path.join(config.downloader.cache.directory, decryptedName); + + if ( // TODO: remove check for encrypted file/cache for encrypted? + isCached(encryptedName) && + isCached(decryptedName) + ) { return; } + + await new Promise((res, rej) => { + const child = spawn(config.downloader.ytdlp_path, [ + "--quiet", + "--no-warnings", + "--allow-unplayable-formats", + "--fixup", "never", + "--paths", config.downloader.cache.directory, + "--output", encryptedName, + streamUrl + ]).on("error", (err) => { rej(err); }); + child.stderr.on("data", (chunk) => { rej(chunk); }); + child.on("exit", () => { res(); }); + }); + + await new Promise((res, rej) => { + const child = spawn(config.downloader.ffmpeg_path, [ + "-loglevel", "error", + "-y", + "-decryption_key", decryptionKey, + "-i", encryptedPath, + "-c", "copy", + "-movflags", "+faststart", + decryptedPath + ]).on("error", (err) => { rej(err); }); + child.stderr.on("data", (chunk) => { rej(chunk); }); + child.on("exit", () => { res(); } ); + }); + + addToCache(encryptedName); + addToCache(decryptedName); +} diff --git a/src/downloader/keygen.ts b/src/downloader/keygen.ts index c631503..2cf006e 100644 --- a/src/downloader/keygen.ts +++ b/src/downloader/keygen.ts @@ -19,7 +19,7 @@ export async function getWidevineDecryptionKey(psshDataUri: string, trackId: str // for some reason, if gotten from a webplayback manifest, the pssh is in a completely different format // well, somewhat. it's just the raw data, we have to rebuild the pssh const rebuiltPssh = psshTools.widevine.encodePssh({ - contentId: "Hiiii", // lol?? i don't know what this is, random slop go!!!! + contentId: "meow", // this actually isn't even needed, but this library is stubborn dataOnly: false, keyIds: [Buffer.from(dataUriToBuffer(psshDataUri).buffer).toString("hex")] }); diff --git a/src/downloader/streamInfo.ts b/src/downloader/streamInfo.ts index bc00416..8cde95b 100644 --- a/src/downloader/streamInfo.ts +++ b/src/downloader/streamInfo.ts @@ -7,6 +7,7 @@ import { getWidevineDecryptionKey } from "./keygen.js"; import { widevine, playready, fairplay } from "../constants/keyFormats.js"; import { select } from "@inquirer/prompts"; import type { WebplaybackResponse } from "api/appleMusicApi.js"; +import { downloadSong } from "./index.js"; // ugliest type ever // this library is so bad @@ -27,24 +28,29 @@ type M3u8 = ReturnType; // OH. it doesn't seem to give the keys you want anyway LOLLLLLLL???? // i'm sure its used for *SOMETHING* so i'll keep it -export class StreamInfo { +export default class StreamInfo { public readonly trackId: string; + public readonly streamUrl: string; public readonly widevinePssh: string | undefined; public readonly playreadyPssh: string | undefined; public readonly fairplayKey: string | undefined; private constructor( trackId: string, + streamUrl: string, widevinePssh: string | undefined, playreadyPssh: string | undefined, fairplayKey: string | undefined ) { this.trackId = trackId; + this.streamUrl = streamUrl; this.widevinePssh = widevinePssh; this.playreadyPssh = playreadyPssh; this.fairplayKey = fairplayKey; } + // TODO: why can't we decrypt widevine ones with this? + // we get a valid key.. but it doesn't work :-( public static async fromTrackMetadata(trackMetadata: SongAttributes<["extendedAssetUrls"]>): Promise { log.warn("the track metadata method is experimental, and may not work or give correct values!"); log.warn("if there is a failure--use a codec that uses the webplayback method"); @@ -70,6 +76,7 @@ export class StreamInfo { return new StreamInfo( trackId, + m3u8Url, // TODO: make this keep in mind the CODEC, yt-dlp will shit itself if not supplied i think widevinePssh, playreadyPssh, fairplayKey @@ -95,6 +102,7 @@ export class StreamInfo { // afaik this ONLY has widevine return new StreamInfo( trackId, + m3u8Url, widevinePssh, undefined, undefined @@ -121,7 +129,6 @@ function getDrmInfos(m3u8Data: M3u8): DrmInfos { throw "m3u8 missing audio session key info!"; } - type AssetInfos = { [key: string]: { "AUDIO-SESSION-KEY-IDS": string[]; }; } function getAssetInfos(m3u8Data: M3u8): AssetInfos { // LOL??? THIS LIBRARY IS SO BAD @@ -176,7 +183,9 @@ const getPlayreadyPssh = (drmInfos: DrmInfos, drmIds: string[]): string | undefi const getFairplayKey = (drmInfos: DrmInfos, drmIds: string[]): string | undefined => getDrmData(drmInfos, drmIds, fairplay); // TODO: remove later, this is just for testing -const streamInfo1 = await StreamInfo.fromWebplayback(await appleMusicApi.getWebplayback("1615276490"), "32:ctrp64"); -const streamInfo2 = await StreamInfo.fromTrackMetadata((await appleMusicApi.getSong("1615276490")).data[0].attributes); -if (streamInfo1.widevinePssh !== undefined) { log.debug(await getWidevineDecryptionKey(streamInfo1.widevinePssh, streamInfo1.trackId)); } -if (streamInfo2.widevinePssh !== undefined) { log.debug(await getWidevineDecryptionKey(streamInfo2.widevinePssh, streamInfo2.trackId)); } +const streamInfo1 = await StreamInfo.fromWebplayback(await appleMusicApi.getWebplayback("1744965708"), "32:ctrp64"); +// const streamInfo2 = await StreamInfo.fromTrackMetadata((await appleMusicApi.getSong("1615276490")).data[0].attributes); +// if (streamInfo1.widevinePssh !== undefined) { log.debug(await getWidevineDecryptionKey(streamInfo1.widevinePssh, streamInfo1.trackId)); } +// if (streamInfo2.widevinePssh !== undefined) { log.debug(await getWidevineDecryptionKey(streamInfo2.widevinePssh, streamInfo2.trackId)); } + +if (streamInfo1.widevinePssh !== undefined) { await downloadSong(streamInfo1.streamUrl, await getWidevineDecryptionKey(streamInfo1.widevinePssh, streamInfo1.trackId)); }