replace Map-based file listings with plain Records, unify cover result, add path helpers
Mserver/src/index.ts
@@ -2,7 +2,6 @@ import path, { basename } from "node:path";
import staticPlugin from "@elysiajs/static";
import { randomUUIDv7 } from "bun";
import { type Context, Elysia, StatusMap, t } from "elysia";
import { stringifyMap } from "music-server-shared/mapconversion";
import {
AudioCodec,
type IsVideoResponse,
@@ -261,7 +260,11 @@ const app = setup
.get(
"/list/*",
async ({ params, set, query }) => {
const dirPath = path.join(musicRoot, decodePath(params["*"]));
const dirPath = resolveInRoot(params["*"]);
if (dirPath instanceof ServerError) {
set.status = dirPath.status;
return dirPath.error;
}
const fileList = await listFiles(dirPath, query.recursive || false);
if (fileList instanceof ServerError) {
set.status = fileList.status;
@@ -269,7 +272,7 @@ const app = setup
}
set.headers["Content-Type"] = "application/json";
set.status = "OK";
return stringifyMap(fileList);
return JSON.stringify(fileList);
},
{ query: t.Optional(t.Object({ recursive: t.Boolean() })) },
)
@@ -285,39 +288,24 @@ const app = setup
.get(
"/cover/*",
async ({ params, set, query }) => {
set.status = "Not Found";
const dirPath = resolveInRoot(params["*"]);
if (dirPath instanceof ServerError) {
set.status = dirPath.status;
return dirPath.error;
}
const fileScanResult = await findCover(dirPath);
const result = await findCover(dirPath);
if (!("bytes" in result)) {
set.status = result.info.status;
return result.info.error;
}
const setContentType = (type: string) => {
set.headers["Content-Type"] = type;
};
if ("path" in fileScanResult) {
set.status = "OK";
if (query.transcode)
return new Response(
await toAvif(
await Bun.file(fileScanResult.path).bytes(),
fileScanResult.info.mimeType,
setContentType,
),
);
set.headers["Content-Type"] = fileScanResult.info.mimeType;
//wrapping in new response discards the accept-range header, which we don't support here
return new Response(Bun.file(fileScanResult.path));
}
if ("content" in fileScanResult) {
set.status = "OK";
if (query.transcode)
return new Response(await toAvif(fileScanResult.content, fileScanResult.info.mimeType, setContentType));
set.headers["Content-Type"] = fileScanResult.info.mimeType;
return new Response(fileScanResult.content);
}
set.status = fileScanResult.info.status;
return fileScanResult.info.error;
set.status = "OK";
if (query.transcode)
return new Response(await toAvif(await result.bytes(), result.info.mimeType, setContentType));
set.headers["Content-Type"] = result.info.mimeType;
return new Response(await result.bytes());
},
{ query: t.Optional(t.Object({ transcode: t.Boolean() })) },
)
Mserver/src/utils.ts
@@ -30,9 +30,10 @@ export async function findAsync<T>(arr: Array<T>, asyncCallback: (value: T) => P
return arr[index];
}
type CoverResult =
| { info: PathInfo; path: string }
| { info: PathInfo; content: Uint8Array<ArrayBuffer> }
//success carries a bytes thunk so the handler doesn't care whether the cover came from a file on
//disk or from embedded metadata
export type CoverResult =
| { info: PathInfo; bytes: () => Promise<Uint8Array<ArrayBuffer>> }
| { info: ServerError };
//the mediainfo CLI JSON output emits every value as a string; broken text encodings arrive
@@ -79,9 +80,10 @@ export async function findCover(targetPath: string): Promise<CoverResult> {
const parsed = await parseFile(targetPath, { duration: false, skipCovers: false });
if (!parsed.common.picture || parsed.common.picture.length === 0) return findCover(path.dirname(targetPath));
const picture = parsed.common.picture[0];
return {
info: new PathInfo(parsed.common.picture[0].format),
content: parsed.common.picture[0].data as Uint8Array<ArrayBuffer>,
info: new PathInfo(picture.format),
bytes: () => Promise.resolve(picture.data as Uint8Array<ArrayBuffer>),
};
}
} catch (e) {
@@ -106,11 +108,12 @@ export async function findCover(targetPath: string): Promise<CoverResult> {
return { info: new ServerError(StatusMap["Not Found"], "Directory not found") };
}
if (coverFile) {
const info = (await getPathInfo(path.join(targetPath, coverFile))) as PathInfo | ServerError;
const coverPath = path.join(targetPath, coverFile);
const info = (await getPathInfo(coverPath)) as PathInfo | ServerError;
if (info instanceof ServerError) {
return { info: info };
}
return { path: path.join(targetPath, coverFile), info };
return { info, bytes: async () => (await Bun.file(coverPath).bytes()) as Uint8Array<ArrayBuffer> };
}
return { info: new ServerError(StatusMap["Not Found"], "Cover not found") };
}
@@ -259,11 +262,12 @@ export function matchesType(mimeType: string, types: string[]): boolean {
export function isBelow(basePath: string, targetPath: string) {
const relativePath = path.relative(basePath, targetPath);
return relativePath && !relativePath.startsWith("..");
//empty when both paths are equal, which counts as inside
return !relativePath.startsWith("..");
}
export async function listFiles(subPath: string, recursive: boolean): Promise<FileListingWithStatus | ServerError> {
const files: FileListingWithStatus = new Map();
const files: FileListingWithStatus = {};
try {
//process entries concurrently; the MediaInfo pool already bounds the expensive probing part
@@ -280,14 +284,14 @@ export async function listFiles(subPath: string, recursive: boolean): Promise<Fi
if (recursive) {
const subListing = await listFiles(itemPath, true);
if (subListing instanceof ServerError) throw subListing;
files.set(fileName, { files: subListing, status: "Scanned" });
files[fileName] = { files: subListing, status: "Scanned" };
} else {
files.set(fileName, { files: new Map(), status: "Unknown" });
files[fileName] = { files: {}, status: "Unknown" };
}
} else if (pathInfoResult instanceof PathInfo && matchesType(pathInfoResult.mimeType, mediaTypes)) {
try {
const probe = await probeFile(itemPath);
files.set(fileName, { metadata: probe });
files[fileName] = { metadata: probe };
} catch (probeError) {
console.error("probing error:", probeError);
}
Mwebclient/src/App.tsx
@@ -126,7 +126,7 @@ const App: Component = () => {
const [files, fetchFiles] = createResource<FlatFileListing, { dir: string; offline: boolean }>(
() => ({ dir: currentDir(), offline: isOffline() }),
async (args) => {
return Array.from(await listFiles(args.dir, args.offline, false));
return Object.entries(await listFiles(args.dir, args.offline, false));
},
{ initialValue: [] },
);
Mwebclient/src/components/FileBrowser.tsx
@@ -17,7 +17,7 @@ import { createStore, reconcile } from "solid-js/store";
import type { FlatFileListing } from "../App";
import { clearDirectoryCache, getCover, listFiles, storedFilesLive } from "../offline";
import type { AppOptions, PlaylistItem } from "../types";
import { formatFilename, generateUUID, joinPath, LocalStorageValues, toast } from "../utils";
import { basename, compareDirFirst, formatFilename, generateUUID, joinPath, LocalStorageValues, toast } from "../utils";
import { DownloadButton } from "./DownloadButton";
import { LoadingScreen } from "./LoadingScreen";
import { ClosedFolderSVG, DownSVG, FilterSVG, MusicSVG, RefreshSVG, UpSVG } from "./SVG";
@@ -84,29 +84,24 @@ export default function FileBrowser(props: FileBrowserProps) {
// Recursively convert file listing to PlaylistItem array
function listingToPlaylistItems(basePath: string, files: FileListing): PlaylistItem[] {
//keep sorting same as shown in the filebrowser
const sorted = Array.from(files.entries()).sort((a, b) => {
if ("metadata" in b[1] && !("metadata" in a[1])) return -1;
if ("metadata" in a[1] && !("metadata" in b[1])) return 1;
return (
"files" in a[1]
? a[0]
: formatFilename({ path: a[0], metadata: a[1].metadata }, props.options.filebrowserDisplay)
).localeCompare(
"files" in b[1]
? b[0]
: formatFilename({ path: b[0], metadata: b[1].metadata }, props.options.filebrowserDisplay),
undefined,
{ numeric: true },
);
});
//keep sorting same as shown in the filebrowser: directories first (by name), then files (by display name)
const displayNameOf = (name: string, item: Directory | MediaFile) =>
"metadata" in item ? formatFilename({ path: name, metadata: item.metadata }, props.options.filebrowserDisplay) : name;
const sorted = Object.entries(files).sort(([aName, aItem], [bName, bItem]) =>
compareDirFirst(
displayNameOf(aName, aItem),
!("metadata" in aItem),
displayNameOf(bName, bItem),
!("metadata" in bItem),
),
);
let result: PlaylistItem[] = [];
for (const [fileName, file] of sorted) {
const fullPath = joinPath(basePath, fileName);
if ("metadata" in file) {
// It's an audio file
result.push(fileToPlaylistItem(fullPath, file));
} else if (file.files && file.files.size > 0) {
} else if (Object.keys(file.files).length > 0) {
// It's a directory with files, recursively add its contents
result = [...result, ...listingToPlaylistItems(fullPath, file.files)];
}
@@ -239,10 +234,7 @@ export default function FileBrowser(props: FileBrowserProps) {
const [fileStore, setFileStore] = createStore<FileProxy[]>([]);
function fileSort(a: FileProxy, b: FileProxy): number {
//sort directories first
if ("metadata" in b.file && !("metadata" in a.file)) return -1;
if ("metadata" in a.file && !("metadata" in b.file)) return 1;
return a.displayName.localeCompare(b.displayName, undefined, { numeric: true });
return compareDirFirst(a.displayName, !("metadata" in a.file), b.displayName, !("metadata" in b.file));
}
createEffect(() => {
@@ -254,7 +246,7 @@ export default function FileBrowser(props: FileBrowserProps) {
const displayName =
"metadata" in value[1]
? formatFilename({ path: value[0], metadata: value[1].metadata }, props.options.filebrowserDisplay)
: value[0].split("/").toReversed()[0];
: basename(value[0]);
return {
key: fullPath + displayName,
fullPath,
Mwebclient/src/offline.ts
@@ -1,5 +1,4 @@
import Dexie, { liveQuery } from "dexie";
import { parseMap } from "music-server-shared/mapconversion";
import type {
Directory,
DirectoryWithStatus,
@@ -14,7 +13,7 @@ import { createStore } from "solid-js/store";
import type { FlatFileListing } from "./App";
import { AsyncSemaphore } from "./semaphore";
import { type AppOptions, type Playlist, StreamingMode } from "./types";
import { handleUnauthorized, joinPath, toast, unproxy } from "./utils";
import { basename, dirname, handleUnauthorized, joinPath, parentPaths, toast, unproxy } from "./utils";
interface StoredFile {
path: string;
@@ -74,12 +73,8 @@ function updateRecursiveStatus(path: string, loadChange: number, includeCurrent:
if (includeCurrent) updateProgressAmount(path);
// Update all parent directories
let currentPath = path;
while (true) {
const lastSlash = currentPath.lastIndexOf("/");
if (lastSlash === -1) break;
currentPath = currentPath.substring(0, lastSlash);
updateProgressAmount(currentPath);
for (const parent of parentPaths(path)) {
updateProgressAmount(parent);
}
}
@@ -113,11 +108,11 @@ export async function loadDataFromDB() {
//contains all locally synced files
const offlineTreeRoot: Directory = {
files: new Map(),
files: {},
};
//used for caching online file listings
const onlineTreeRoot: DirectoryWithStatus = {
files: new Map(),
files: {},
status: "Unknown",
};
@@ -125,12 +120,12 @@ const onlineTreeRoot: DirectoryWithStatus = {
function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
if (other.status === "Unknown") return;
target.status = other.status;
for (const [fileName, file] of other.files.entries()) {
const targetFile = target.files.get(fileName);
for (const [fileName, file] of Object.entries(other.files)) {
const targetFile = target.files[fileName];
if (targetFile && "files" in targetFile && "files" in file) {
mergeTrees(targetFile, file);
} else {
target.files.set(fileName, file);
target.files[fileName] = file;
}
}
}
@@ -144,19 +139,21 @@ function updateFileTree(path: string, file: DirectoryWithStatus, offline: false)
// Updates file tree structure with new data (offline or online)
function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
if (offline && path === "") throw new Error("Trying to update root in offline mode");
const basePath = path.split("/").slice(0, -1).join("/");
const fileName = path.split("/").toReversed()[0];
const basePath = dirname(path);
const fileName = basename(path);
const node = getDirInTree(basePath, true, offline);
//widen so the writes below don't have to satisfy both tree flavors at once
const nodeFiles: Record<string, DirectoryWithStatus | Directory | MediaFile> = node.files;
if (offline) {
node.files.set(fileName, file);
nodeFiles[fileName] = file;
} else {
if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
else {
let target = node.files.get(fileName) as DirectoryWithStatus;
if (!target) target = { files: new Map(), status: "Unknown" };
let target = nodeFiles[fileName] as DirectoryWithStatus;
if (!target) target = { files: {}, status: "Unknown" };
mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
node.files.set(fileName, target);
nodeFiles[fileName] = target;
}
}
}
@@ -175,13 +172,15 @@ function getDirInTree(path: string, create: boolean, offline: boolean): Director
const parts = path.split("/");
for (const part of parts) {
let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
//widen so reads and writes don't have to satisfy both tree flavors at once
const files: Record<string, DirectoryWithStatus | Directory | MediaFile> = current.files;
let next: DirectoryWithStatus | Directory | MediaFile | undefined = files[part];
if (!next || "metadata" in next) {
if (!create) return null;
//if mediafile already exists, we overwrite it with a directory anyway
if (offline) next = { files: new Map() };
else next = { files: new Map(), status: "Unknown" };
current.files.set(part, next);
if (offline) next = { files: {} };
else next = { files: {}, status: "Unknown" };
files[part] = next;
}
current = next;
}
@@ -190,14 +189,14 @@ function getDirInTree(path: string, create: boolean, offline: boolean): Director
function deleteInOfflineTree(path: string): void {
if (path === "") return;
const basePath = path.split("/").slice(0, -1).join("/");
const fileName = path.split("/").toReversed()[0];
const basePath = dirname(path);
const fileName = basename(path);
const node = getDirInTree(basePath, false, true);
if (!node) return;
node.files.delete(fileName);
if (node.files.size === 0) {
delete node.files[fileName];
if (Object.keys(node.files).length === 0) {
db.covers.delete(basePath); //remove stored cover for this dir if there are not files left
deleteInOfflineTree(basePath);
}
@@ -208,16 +207,16 @@ export const clearDirectoryCache = async (path: string) => {
if (!node) return;
// Save existing children before clearing
const preservedChildren = new Map(node.files);
const preservedChildren = { ...node.files };
node.files.clear();
node.files = {};
node.status = "Unknown";
// restore any preserved subtrees where the parent directory still exist
const newListing = await listFiles(path, false, false);
for (const [name, child] of preservedChildren.entries()) {
if ("files" in child && newListing.has(name)) {
node.files.set(name, child);
for (const [name, child] of Object.entries(preservedChildren)) {
if ("files" in child && newListing[name] !== undefined) {
node.files[name] = child;
}
}
};
@@ -238,12 +237,12 @@ async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileLis
return;
}
return parseMap(await response.text()) as FileListingWithStatus;
return (await response.json()) as FileListingWithStatus;
}
function listOfflineFiles(dir: string): FileListing {
const node = getDirInTree(dir, false, true);
if (!node?.files) return new Map();
if (!node?.files) return {};
return node.files;
}
@@ -252,7 +251,7 @@ function fullyKnown(dir: DirectoryWithStatus) {
if (dir.status === "Unknown") {
return false;
}
if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
if (Object.values(dir.files).some((value) => "files" in value && !fullyKnown(value))) {
return false;
}
return true;
@@ -260,25 +259,25 @@ function fullyKnown(dir: DirectoryWithStatus) {
//non-recursive listing should not include nested children
function pruneChildren(listing: FileListing) {
const pruned = new Map();
for (const [name, item] of listing.entries()) {
if ("metadata" in item) pruned.set(name, item);
else pruned.set(name, { files: new Map() });
const pruned: FileListing = {};
for (const [name, item] of Object.entries(listing)) {
if ("metadata" in item) pruned[name] = item;
else pruned[name] = { files: {} };
}
return pruned;
}
//used for combining the online and offline results, to be able to show files that only exist locally
function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
const result = new Map(listing1);
const result: FileListing = { ...listing1 };
for (const [key, value2] of listing2.entries()) {
const value1 = result.get(key);
for (const [key, value2] of Object.entries(listing2)) {
const value1 = result[key];
if (value1 && "files" in value1 && "files" in value2) {
result.set(key, { files: combineFileListings(value1.files, value2.files) });
result[key] = { files: combineFileListings(value1.files, value2.files) };
} else {
result.set(key, value2);
result[key] = value2;
}
}
@@ -301,7 +300,7 @@ export async function listFiles(dir: string, offline: boolean, recursive: boolea
return recursive ? combined : pruneChildren(combined);
}
const files = await listOnlineFiles(dir, recursive);
if (!files) return new Map();
if (!files) return {};
updateFileTree(dir, { files, status: "Scanned" }, false);
const combined = combineFileListings(files, listOfflineFiles(dir));
return recursive ? combined : pruneChildren(combined);
@@ -434,7 +433,7 @@ export async function deletePlaylist(name: string) {
function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
const paths: string[] = [];
for (const [name, item] of listing.entries()) {
for (const [name, item] of Object.entries(listing)) {
const currentPath = joinPath(basePath, name);
if ("metadata" in item) {
@@ -470,11 +469,7 @@ export async function deleteFile(
if ("files" in item) {
coversToDelete.add(fullPath);
for (const filePath of toDelete) {
let dir = filePath;
while (true) {
const lastSlash = dir.lastIndexOf("/");
if (lastSlash === -1) break;
dir = dir.substring(0, lastSlash);
for (const dir of parentPaths(filePath)) {
//stop at the deleted root, parents of it may still hold other synced files
if (dir.length <= fullPath.length) break;
coversToDelete.add(dir);
@@ -581,11 +576,8 @@ async function downloadFile(
fetchFiles.refetch();
//iterate through all parent directories and download their covers to database
const parts = path.split("/");
while (true) {
getCover(parts.join("/"), options, true, false).then((cover) => URL.revokeObjectURL(cover || ""));
parts.pop();
if (parts.length === 0) break;
for (const dir of [path, ...parentPaths(path)]) {
getCover(dir, options, true, false).then((cover) => URL.revokeObjectURL(cover || ""));
}
} finally {
downloadSemaphore.release();
@@ -622,7 +614,7 @@ export async function download(
const promises: Promise<unknown>[] = [];
const listing = await listFiles(fullPath, false, false);
batch(() => {
for (const [childName, childItem] of listing) {
for (const [childName, childItem] of Object.entries(listing)) {
promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload));
}
});
Mwebclient/src/utils.ts
@@ -1,4 +1,3 @@
import type { FileListing, MediaFile } from "music-server-shared/types";
import Toastify from "toastify-js";
import { FileDisplay, type PlaylistItem } from "./types";
@@ -11,6 +10,32 @@ export function joinPath(...paths: string[]): string {
return paths.join("/").replace(/\/+/g, "/").replace(/^\//, "").replace(/\/$/, "");
}
//dirname("a/b/c") = "a/b", dirname("a") = ""
export function dirname(path: string): string {
return path.split("/").slice(0, -1).join("/");
}
export function basename(path: string): string {
return path.split("/").toReversed()[0];
}
//yields each ancestor directory, immediate parent first: parentPaths("a/b/c") yields "a/b", "a"
export function* parentPaths(path: string): Generator<string> {
let current = path;
while (true) {
const lastSlash = current.lastIndexOf("/");
if (lastSlash === -1) return;
current = current.substring(0, lastSlash);
yield current;
}
}
//directories sort before files, names compare naturally ("track 2" before "track 10")
export function compareDirFirst(aName: string, aIsDir: boolean, bName: string, bIsDir: boolean): number {
if (aIsDir !== bIsDir) return aIsDir ? -1 : 1;
return aName.localeCompare(bName, undefined, { numeric: true });
}
export function formatTime(seconds: number): string {
const mins = Math.floor(seconds / 60);
const secs = Math.floor(seconds % 60);
@@ -94,20 +119,6 @@ export function handleUnauthorized(response?: Response | { status?: number } | v
return true;
}
export type FileListingRecord = Record<string, MediaFile | { files: FileListingRecord }>;
export function fileListingToObject(listing: FileListing): FileListingRecord {
const result: FileListingRecord = {};
for (const [name, entry] of listing.entries()) {
if ("files" in entry) {
result[name] = { files: fileListingToObject(entry.files) };
} else {
result[name] = entry;
}
}
return result;
}
export function formatBytes(bytes: number): string {
if (bytes === 0) return "Queued...";
@@ -127,7 +138,7 @@ function getItemName(item: Pick<PlaylistItem, "path">): string;
function getItemName(item: Pick<PlaylistItem, "path"> | null): string | undefined;
function getItemName(item: Pick<PlaylistItem, "path"> | null) {
if (!item) return undefined;
return item.path.split("/").toReversed()[0];
return basename(item.path);
}
export function formatFilename(file: Omit<PlaylistItem, "id">, format: FileDisplay): string {