reidlab/amdl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

axios->undici, reducing overhead and vuln

Browse source
This commit is contained in:
Reid 2025-10-13 00:11:39 -07:00
parent a3cefee49a
commit 3a179e0c69
Signed by: reidlab
GPG key ID: DAF5EAF6665839FD
12 changed files with 160 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

133
src/appleMusicApi/index.ts
View file

@ -1,4 +1,3 @@
import axios, { type AxiosInstance } from "axios";
import { ampApiUrl, appleMusicHomepageUrl, licenseApiUrl, webplaybackApiUrl } from "../constants/urls.js";
import type { GetAlbumResponse, GetPlaylistResponse, GetSongResponse, SearchResponse } from "./types/responses.js";
import type { AlbumAttributesExtensionTypes, AnyAttributesExtensionTypes, SongAttributesExtensionTypes } from "./types/extensions.js";
@ -6,10 +5,12 @@ import { getToken } from "./token.js";
import { config, env } from "../config.js";
import { HttpException } from "../web/index.js";
import type { RelationshipTypes } from "./types/relationships.js";
import { fetch, request } from "undici";
export default class AppleMusicApi {
private storefront: string;
private http: AxiosInstance;
private headers: Headers;
private params: URLSearchParams;
public constructor(
storefront: string,
@ -17,21 +18,58 @@ export default class AppleMusicApi {
mediaUserToken: string
) {
this.storefront = storefront;
this.http = axios.create({
baseURL: ampApiUrl
});
this.http.defaults.headers.common["Origin"] = appleMusicHomepageUrl;
this.http.defaults.headers.common["Media-User-Token"] = mediaUserToken;
this.headers = new Headers();
this.headers.set("Origin", appleMusicHomepageUrl);
this.headers.set("Media-User-Token", mediaUserToken);
this.headers.set("x-apple-music-user-token", mediaUserToken);
this.headers.set("x-apple-renewal", "true");
// yeah dude. awesome
// https://stackoverflow.com/a/54636780
this.http.defaults.params = {};
this.http.defaults.params["l"] = language;
this.params = new URLSearchParams();
this.params.set("l", language);
}
public async login(): Promise<void> {
this.http.defaults.headers.common["Authorization"] = `Bearer ${await getToken(appleMusicHomepageUrl)}`;
this.headers.set("Authorization", `Bearer ${await getToken(appleMusicHomepageUrl)}`);
}
// TODO: dedupe these functions
// TODO: also make their param/body/header stuff more modular
// please!!!!!
private async get<
T
> (link: string, params: Record<string, string | number | boolean> = {}): Promise<T> {
const url = new URL(link);
const urlParams = new URLSearchParams(this.params);
for (const entry of Object.entries(params)) { urlParams.set(entry[0], entry[1].toString()); }
url.search = urlParams.toString();
const response = await request(url, { headers: this.headers });
const json = await response.body.json();
return json as T;
}
// TODO: discover why it works when its fetch but not request
// what i mean by "works" is it doesn't return an error upstream that i can't replicate in cURL
// i'm so confused mannnn
private async post<
T
> (link: string, data: Record<string, string | number | boolean> = {}): Promise<T> {
const url = new URL(link);
const urlParams = new URLSearchParams(this.params);
url.search = urlParams.toString();
const headers = new Headers(this.headers);
headers.set("Content-Type", "application/json");
const response = await fetch(url, {
method: "POST",
body: JSON.stringify(data),
headers: headers
});
const json = await response.json();
return json as T;
}
async getAlbum<
@ -42,12 +80,10 @@ export default class AppleMusicApi {
extend: T = [] as unknown[] as T,
relationships: U = ["tracks"] as U
): Promise<GetAlbumResponse<T, U>> {
return (await this.http.get<GetAlbumResponse<T, U>>(`/v1/catalog/${this.storefront}/albums/${id}`, {
params: {
extend: extend.join(","),
include: relationships.join(",")
}
})).data;
return await this.get<GetAlbumResponse<T, U>>(`${ampApiUrl}/v1/catalog/${this.storefront}/albums/${id}`, {
extend: extend.join(","),
include: relationships.join(",")
});
}
// TODO: make it so you can get more than the first 100 tracks
@ -61,12 +97,10 @@ export default class AppleMusicApi {
extend: T = [] as never as T,
relationships: U = ["tracks"] as U
): Promise<GetPlaylistResponse<T, U>> {
return (await this.http.get<GetPlaylistResponse<T, U>>(`/v1/catalog/${this.storefront}/playlists/${id}`, {
params: {
extend: extend.join(","),
include: relationships.join(",")
}
})).data;
return await this.get<GetPlaylistResponse<T, U>>(`${ampApiUrl}/v1/catalog/${this.storefront}/playlists/${id}`, {
extend: extend.join(","),
include: relationships.join(",")
});
}
async getSong<
@ -80,12 +114,10 @@ export default class AppleMusicApi {
extend: T = ["extendedAssetUrls"] as T,
relationships: U = ["albums"] as U
): Promise<GetSongResponse<T, U>> {
return (await this.http.get<GetSongResponse<T, U>>(`/v1/catalog/${this.storefront}/songs/${id}`, {
params: {
extend: extend.join(","),
include: relationships.join(",")
}
})).data;
return await this.get<GetSongResponse<T, U>>(`${ampApiUrl}/v1/catalog/${this.storefront}/songs/${id}`, {
extend: extend.join(","),
include: relationships.join(",")
});
}
// TODO: add support for other types / abstract it for other types
@ -100,19 +132,17 @@ export default class AppleMusicApi {
extend: T = [] as unknown[] as T,
relationships: U = ["tracks"] as U
): Promise<SearchResponse<T, U>> {
return (await this.http.get(`/v1/catalog/${this.storefront}/search`, {
params: {
...this.addScopingParameters("albums", relationships, extend),
...{
term: term,
types: ["albums", "songs"].join(","), // adding "songs" makes search results have albums when searching song name
limit: limit,
offset: offset,
extend: extend.join(","),
include: relationships.join(",")
}
return await this.get(`${ampApiUrl}/v1/catalog/${this.storefront}/search`, {
...this.addScopingParameters("albums", relationships, extend),
...{
term: term,
types: ["albums", "songs"].join(","), // adding "songs" makes search results have albums when searching song name
limit: limit,
offset: offset,
extend: extend.join(","),
include: relationships.join(",")
}
})).data;
});
}
async getWebplayback(
@ -122,18 +152,18 @@ export default class AppleMusicApi {
// as we know, we can't have fun things with "WOA" urls
// https://files.catbox.moe/5oqolg.png (THE LINK WAS CENSORED?? TAKEN DOWN FROM CNN??)
// https://files.catbox.moe/wjxwzk.png
const res = await this.http.post(webplaybackApiUrl, {
const res = await this.post<WebplaybackResponse & { failureType?: string }>(webplaybackApiUrl, {
salableAdamId: trackId,
language: config.downloader.api.language
});
if (res.data?.failureType === "3077") {
if (res?.failureType === "3077") {
throw new HttpException(404, "track not found");
} else if (res.data?.failureType !== undefined) {
throw new HttpException(500, `upstream webplayback api error: ${res.data.failureType}`);
} else if (res?.failureType !== undefined) {
throw new HttpException(500, `upstream webplayback api error: ${res.failureType}`);
}
return res.data;
return res;
}
async getWidevineLicense(
@ -141,19 +171,14 @@ export default class AppleMusicApi {
trackUri: string,
challenge: string
): Promise<WidevineLicenseResponse> {
return (await this.http.post(licenseApiUrl, {
return (await this.post(licenseApiUrl, {
challenge: challenge,
"key-system": "com.widevine.alpha",
uri: trackUri,
adamId: trackId,
isLibrary: false,
"user-initiated": true
}, { headers: {
// do these do anything.
// i'm including them anyway,,
"x-apple-music-user-token": this.http.defaults.headers.common["Media-User-Token"],
"x-apple-renewal": true
}})).data;
}));
}
// helper function to automatically add scoping parameters

9
src/appleMusicApi/token.ts
View file

@ -1,3 +1,4 @@
import { request } from "undici";
import * as log from "../log.js";
// basically, i don't want to pay 100 dollars for a dev token to the official API
@ -5,8 +6,8 @@ import * as log from "../log.js";
// thanks to this guy complaining to apple for telling us this! https://developer.apple.com/forums/thread/702228
// apple says "any other method may be blocked at any time" (posted in mar 2022, most likely not happening)
export async function getToken(baseUrl: string): Promise<string> {
const indexResponse = await fetch(baseUrl);
const indexBody = await indexResponse.text();
const indexResponse = await request(baseUrl);
const indexBody = await indexResponse.body.text();
const jsRegex = /\/assets\/index-legacy-[^/]+\.js/;
const jsPath = indexBody.match(jsRegex)?.[0];
@ -15,8 +16,8 @@ export async function getToken(baseUrl: string): Promise<string> {
throw new Error("could not match for the index javascript file!");
}
const jsResponse = await fetch(baseUrl + jsPath);
const jsBody = await jsResponse.text();
const jsResponse = await request(baseUrl + jsPath);
const jsBody = await jsResponse.body.text();
// the token is actually a base64-encoded JWT
// `eyJh` === `{"a`, which is the beginning of a JWT (a is the start of alg)

12
src/downloader/index.ts
View file

@ -9,6 +9,7 @@ import { createDecipheriv } from "node:crypto";
import type { AlbumAttributes } from "../appleMusicApi/types/attributes.js";
import { pipeline } from "node:stream/promises";
import { createWriteStream } from "node:fs";
import { request } from "undici";
export async function downloadSongFile(streamUrl: string, decryptionKey: string, songCodec: RegularCodecType | WebplaybackCodecType, songResponse: GetSongResponse<[], ["albums"]>): Promise<string> {
let baseOutputName = streamUrl.match(/(?:.*\/)\s*(\S*?)[.?]/)?.[1];
@ -59,8 +60,8 @@ export async function downloadSongFile(streamUrl: string, decryptionKey: string,
// TODO: less mem alloc/access
// TODO: use actual atom scanning. what if the magic bytes appear in a sample
export async function fetchAndDecryptStreamSegment(segmentUrl: string, decryptionKey: string, fetchLength: number, offset: number): Promise<Uint8Array> {
const response = await fetch(segmentUrl, { headers: { "range": `bytes=${offset}-${offset + fetchLength - 1}` }});
const file = new Uint8Array(await response.arrayBuffer());
const response = await request(segmentUrl, { headers: { "range": `bytes=${offset}-${offset + fetchLength - 1}` }});
const file = new Uint8Array(await response.body.bytes());
// this translates to "moof"
const moof = new Uint8Array([0x6D, 0x6F, 0x6F, 0x66]);
@ -121,12 +122,9 @@ export async function downloadAlbumCover(albumAttributes: AlbumAttributes<[]>):
const imagePath = path.join(config.downloader.cache.directory, imageFileName);
if (await isFileCached(imageFileName) === false) {
const response = await fetch(url);
const response = await request(url);
if (!response.ok) { throw new Error(`failed to fetch artwork: ${response.status}`); }
if (!response.body) { throw new Error("no response body for artwork!"); }
await pipeline(response.body as ReadableStream, createWriteStream(imagePath));
await pipeline(response.body, createWriteStream(imagePath));
await addFileToCache(imageFileName);
}

14
src/downloader/streamInfo.ts
View file

@ -1,11 +1,11 @@
import * as log from "../log.js";
import type { SongAttributes } from "../appleMusicApi/types/attributes.js";
import hls, { Item } from "parse-hls";
import axios from "axios";
import { widevine, playready, fairplay } from "../constants/keyFormats.js";
import { songCodecRegex } from "../constants/codecs.js";
import type { WebplaybackResponse } from "appleMusicApi/index.js";
import type { RegularCodecType, WebplaybackCodecType } from "./codecType.js";
import { request } from "undici";
// why is this private
// i wish pain on the person who wrote this /j :smile:
@ -47,8 +47,8 @@ export default class StreamInfo {
log.warn("if there is a failure--use a codec that uses the webplayback method");
const m3u8Url = trackMetadata.extendedAssetUrls.enhancedHls;
const m3u8 = await axios.get(m3u8Url, { responseType: "text" });
const m3u8Parsed = hls.default.parse(m3u8.data);
const m3u8 = await request(m3u8Url);
const m3u8Parsed = hls.default.parse(await m3u8.body.text());
const drmInfos = getDrmInfos(m3u8Parsed);
const assetInfos = getAssetInfos(m3u8Parsed);
@ -59,8 +59,8 @@ export default class StreamInfo {
const drmIds = assetInfos[variantId]["AUDIO-SESSION-KEY-IDS"];
const correctM3u8Url = m3u8Url.substring(0, m3u8Url.lastIndexOf("/")) + "/" + playlist.uri;
const correctM3u8 = await axios.get(correctM3u8Url, { responseType: "text" });
const correctM3u8Parsed = hls.default.parse(correctM3u8.data);
const correctM3u8 = await request(correctM3u8Url);
const correctM3u8Parsed = hls.default.parse(await correctM3u8.body.text());
const primaryFileUrl = getPrimaryFileUrl(m3u8Parsed);
@ -97,8 +97,8 @@ export default class StreamInfo {
const trackId = song.songId;
const m3u8Url = asset.URL;
const m3u8 = await axios.get(m3u8Url, { responseType: "text" });
const m3u8Parsed = hls.default.parse(m3u8.data);
const m3u8 = await request(m3u8Url);
const m3u8Parsed = hls.default.parse(await m3u8.body.text());
const primaryFileUrl = getPrimaryFileUrl(m3u8Parsed);

42
src/index.ts
View file

@ -3,14 +3,33 @@ import process from "node:process";
import * as log from "./log.js";
import { appleMusicApi } from "./appleMusicApi/index.js";
import { app } from "./web/index.js";
// @ts-expect-error: cacheStores should exist--it does not
// TODO: file an issue on undici and remove this when fixed
// DONE: issue filed... https://github.com/nodejs/undici/issues/4614
// OKAY: it was "resolved". just have to wait for the next release
import { Agent, interceptors, setGlobalDispatcher, cacheStores } from "undici";
await appleMusicApi.login().catch((err) => {
log.error("failed to login to apple music api");
setGlobalDispatcher(new Agent().compose([
interceptors.responseError(),
interceptors.redirect(),
interceptors.decompress(),
// TODO: configurable cache sizes?
// these values are pretty nice for non-binary (lol) data
interceptors.cache({ store: new cacheStores.MemoryCacheStore({
maxSize: 50 * 1024 * 1024, // 5mb
maxCount: 1000,
maxEntrySize: 5 * 1024 // 5kb
})})
]));
try {
await appleMusicApi.login();
log.info("logged in to apple music api");
} catch (err) {
log.error("failed to login to apple music api!");
log.error(err);
process.exit(1);
}).finally(() => {
log.info("logged in to apple music api");
});
}
try {
const listener = app.listen(config.server.port, () => {
@ -24,18 +43,7 @@ try {
else { log.info(`hosting on http://localhost:${address.port}`); }
});
} catch (err) {
log.error("failed to start server");
log.error("failed to start server!");
log.error(err);
process.exit(1);
}
process.on("uncaughtException", (err) => {
log.error("uncaught exception!");
log.error(err);
process.exit(1);
});
process.on("unhandledRejection", (err) => {
log.error("unhandled rejection!");
log.error(err);
process.exit(1);
});

18
src/log.ts
View file

@ -89,3 +89,21 @@ export function debug(...message: unknown[]): void { log(Level.Debug, ...message
export function info(...message: unknown[]): void { log(Level.Info, ...message); }
export function warn(...message: unknown[]): void { log(Level.Warn, ...message); }
export function error(...message: unknown[]): void { log(Level.Error, ...message); }
// custom interceptors for uncaught exceptions and unhandled rejections
// additionally, also wrap warnings (but, make the experimental warning shut up)
process.on("uncaughtException", (err) => {
error("uncaught exception!");
error(err);
process.exit(1);
});
process.on("unhandledRejection", (err) => {
error("unhandled rejection!");
error(err);
process.exit(1);
});
process.removeAllListeners("warning");
process.on("warning", (err) => {
if (err.name === "ExperimentalWarning") { return; }
warn(err);
});

3
src/web/endpoints/back/download.ts
View file

@ -53,10 +53,11 @@ router.get(path, async (req, res, next) => {
}
const decryptionKey =
await getKeyFromCache(id, codecType.codecType) ||
await getKeyFromCache(id, codecType.codecType) ??
await getWidevineDecryptionKey(streamInfo.widevinePssh, streamInfo.trackId);
await addKeyToCache(id, codecType.codecType, decryptionKey);
// TODO: stream to user
const filePath = await downloadSongFile(streamInfo.streamUrl, decryptionKey, codecType.codecType, trackMetadata);
const fileExt = "." + filePath.split(".").at(-1) as string; // safe cast, filePath is always a valid path
const fileName = formatSongForFs(trackAttributes) + fileExt;

2
src/web/endpoints/back/downloadAlbum.ts
View file

@ -67,7 +67,7 @@ router.get(path, async (req, res, next) => {
}
const decryptionKey =
await getKeyFromCache(trackId, codecType.codecType) ||
await getKeyFromCache(trackId, codecType.codecType) ??
await getWidevineDecryptionKey(streamInfo.widevinePssh, streamInfo.trackId);
await addKeyToCache(trackId, codecType.codecType, decryptionKey);

4
src/web/endpoints/back/downloadSegment.ts
View file

@ -46,8 +46,6 @@ paths[path] = {
}
};
// TODO: cache the decryption key for a while, so we don't have to fetch it every time
// the way we could do that is store track id + codec mapped to a decryption key (in memory? for like how long? maybe have an expiry?)
router.get(path, async (req, res, next) => {
try {
const { id, originalMp4, codec } = (await validate(req, schema)).query;
@ -68,7 +66,7 @@ router.get(path, async (req, res, next) => {
}
const decryptionKey =
await getKeyFromCache(id, codecType.codecType) ||
await getKeyFromCache(id, codecType.codecType) ??
await getWidevineDecryptionKey(streamInfo.widevinePssh, streamInfo.trackId);
await addKeyToCache(id, codecType.codecType, decryptionKey);

11
src/web/index.ts
View file

@ -5,7 +5,7 @@ import formatDuration from "format-duration";
import { back, front } from "./endpoints/index.js";
import { ZodError } from "zod";
import { fromZodError } from "zod-validation-error";
import { AxiosError } from "axios";
import { errors as undiciErrors } from "undici";
import { env } from "../config.js";
import { createOpenApiDocument } from "./openApi.js";
@ -54,10 +54,11 @@ app.use((req, _res, next) => {
// ex. if the apple music api returns a 403, we want to return a 403
// this is so damn useful, i'm so glad i thought of this
app.use((err: AxiosError, _req: Request, _res: Response, next: NextFunction) => {
if (err instanceof AxiosError && err.response) {
const status = err.response.status;
const message = `upstream api error: ${err.response.status}`;
// TODO: make this only happen on AM api? doesn't make too much sense otherwise
app.use((err: undiciErrors.ResponseError, _req: Request, _res: Response, next: NextFunction) => {
if (err instanceof undiciErrors.ResponseError) {
const status = err.statusCode;
const message = `fetch error/upstream error: ${err.statusCode}. file an issue if unexpected/reoccuring`;
next(new HttpException(status, message));
} else {