import type { MediaFile, Directory, FileListing, FileListingWithStatus, DirectoryWithStatus, } from "music-server-shared/types"; import { parseMap } from "music-server-shared/mapconversion"; import type { AppOptions, Playlist, PlaylistItem } from "./types"; import { joinPath, toast } from "./utils"; import type { Accessor, ResourceActions } from "solid-js"; import { createStore } from "solid-js/store"; import { AsyncSemaphore } from "./semaphore"; interface Progress { bytes: number; } export type DownloadStatus = Progress | "DirectoryProgress" | "SyncedFile" | undefined; export const [downloadStatusMap, setDownloadStatusMap] = createStore>({}); export async function loadDataFromDB() { try { const db = await openDB(); const files = await db.listFiles(FileStore); for (const file of files) { const parts = file.path.split("/"); let current = ""; for (let index = 0; index < parts.length; index++) { current = joinPath(current, parts[index]); setDownloadStatusMap(current, "SyncedFile"); } updateFileTree(file.path, { metadata: file.metadata }, true); } } catch (e) { toast("Loading offline files failed", "error", e); console.error(e); } } const offlineTreeRoot: Directory = { files: new Map(), }; const onlineTreeRoot: DirectoryWithStatus = { files: new Map(), status: "Unknown", }; function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) { if (other.status === "Unknown") return; target.status = other.status; for (const [fileName, file] of other.files.entries()) { const targetFile = target.files.get(fileName); if (targetFile && "files" in targetFile && "files" in file) { mergeTrees(targetFile, file); } else { target.files.set(fileName, file); } } } //TODO: improve typing to avoid "as" //overloads for typecheking function updateFileTree(path: string, file: MediaFile, offline: true): void; function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void; function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean): void { if (offline && path === "") throw new Error("Trying to update root in offline mode"); const basePath = path.split("/").slice(0, -1).join("/"); const fileName = path.split("/").toReversed()[0]; const node = getDirInTree(basePath, true, offline); if (offline) { node.files.set(fileName, file); } else { if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus); else { let target = node.files.get(fileName) as DirectoryWithStatus; if (!target) target = { files: new Map(), status: "Unknown" }; mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus); node.files.set(fileName, target); } } } //overloads for typecheking function getDirInTree(path: string, create: true, offline: true): Directory; function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus; function getDirInTree(path: string, create: false, offline: true): Directory | null; function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null; function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory; function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null; // get a DirectoryNode from the directory tree, and optionally create it if it doesn't exist yet function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null { let current = offline ? offlineTreeRoot : onlineTreeRoot; if (path === "") return current; const parts = path.split("/"); for (const part of parts) { let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part); if (!next || "metadata" in next) { if (!create) return null; //if mediafile already exists, we overwrite it with a directory anyway if (offline) next = { files: new Map() }; else next = { files: new Map(), status: "Unknown" }; current.files.set(part, next); } current = next; } return current; } function deleteOfflinePath(path: string): void { const basePath = path.split("/").slice(0, -1).join("/"); const fileName = path.split("/").toReversed()[0]; const node = getDirInTree(basePath, false, true); if (!node) return; node.files.delete(fileName); } export const clearDirectoryCache = (path: string) => { const node = getDirInTree(path, false, false); if (!node) return; node.files.clear(); node.status = "Unknown"; }; async function listOnlineFiles(dir: string, recursive: boolean): Promise { const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {}); if (response?.status === 404) { toast(`Directory not found: ${dir}`, "error"); return; } if (!response || !response.ok) { toast( `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`, "error", ); return; } return parseMap(await response.text()) as FileListingWithStatus; } function listOfflineFiles(dir: string): FileListing { const node = getDirInTree(dir, false, true); if (!node || !node.files) return new Map(); return node.files; } function fullyKnown(dir: DirectoryWithStatus) { if (dir.status === "Unknown") { return false; } if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) { return false; } return true; } //non-recursive listing should not include nested children function pruneChildren(listing: FileListing) { const cloned = structuredClone(listing); for (const [, item] of cloned.entries()) { if ("files" in item) item.files.clear(); } return cloned; } function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing { const result = new Map(listing1); for (const [key, value2] of listing2.entries()) { const value1 = result.get(key); if (value1 && "files" in value1 && "files" in value2) { result.set(key, { files: combineFileListings(value1.files, value2.files) }); } else { result.set(key, value2); } } return result; } export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise { if (offline) { const files = listOfflineFiles(dir); return recursive ? files : pruneChildren(files); } const cached = getDirInTree(dir, false, false); if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) { const combined = combineFileListings(cached.files, listOfflineFiles(dir)); return recursive ? combined : pruneChildren(combined); } const files = await listOnlineFiles(dir, recursive); if (!files) return new Map(); updateFileTree(dir, { files, status: "Scanned" }, false); const combined = combineFileListings(files, listOfflineFiles(dir)); return recursive ? combined : pruneChildren(combined); } export async function getCover(dir: string, offline: boolean) { const db = await openDB(); const dbResult = await db.get(CoverStore, dir); if (dbResult !== undefined) { const cached = dbResult as StoredCover; return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data])); } if (offline) return undefined; const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {}); if (!response || response.status === 404) { db.put(CoverStore, { path: dir, data: null }); return undefined; } const blob = await response.blob(); db.put(CoverStore, { path: dir, data: blob }); return URL.createObjectURL(blob); } export const clearCoverCache = async (path: string) => { const db = await openDB(); return db.delete(CoverStore, path); }; export async function getSrc(path: string, options: AppOptions, offline: boolean) { const db = await openDB(); const dbResult = (await db.get(FileStore, path)) as StoredFile; if (dbResult) return URL.createObjectURL(dbResult.data); if (offline) { toast("This file is not cached, leave offline mode to play it", "error"); return undefined; } if (options.enableTranscoding) { return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`; } return joinPath("/download", path); } export async function loadPlaylists() { const db = await openDB(); return await db.listPlaylists(PlaylistStore); } export async function savePlaylist(playlist: Playlist, overwrite: boolean) { const db = await openDB(); playlist.items = playlist.items.map((item) => { //some less space in the db by omitting ids const { id, ...itemWithoutId } = item as PlaylistItem; return itemWithoutId; }); if (overwrite) await db.put(PlaylistStore, playlist); else await db.add(PlaylistStore, playlist); } export async function deletePlaylist(name: string) { const db = await openDB(); await db.delete(PlaylistStore, name); } export async function deleteFile( fileName: string, item: MediaFile | Directory, basePath: string, fetchFiles: ResourceActions, ) { const fullPath = joinPath(basePath, fileName); if (!downloadStatusMap[fullPath]) return; if (!("metadata" in item)) { const newBasePath = joinPath(basePath, fileName); const result = listOfflineFiles(newBasePath); for (const [childName, childItem] of result) { await deleteFile(childName, childItem, newBasePath, fetchFiles); } } deleteOfflinePath(fullPath); const db = await openDB(); await db.delete(FileStore, fullPath); setDownloadStatusMap(fullPath, undefined); fetchFiles.refetch(); } const downloadSemaphore = new AsyncSemaphore(1); async function downloadFile( path: string, currentDir: Accessor, options: AppOptions, item: MediaFile, fetchFiles: ResourceActions, ): Promise { const initialProgress = downloadStatusMap[path]; if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress try { setDownloadStatusMap(path, { bytes: 0 }); const db = await openDB(); let mimeType: string | undefined; downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads); let blob: Blob; try { await downloadSemaphore.acquire(); let res: Response; if (options.enableTranscoding) { res = await fetch( `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`, ); } else { res = await fetch(joinPath("/download", path)); } //let the browser do the actual download const blobPromise = res.clone().blob(); if (!res.ok || !res.body) throw res.statusText; //we just iterate over the chunks to get the current progress const reader = res.body.getReader(); mimeType = res.headers.get("content-type") ?? undefined; let received = 0; let lastUpdate = 0; while (true) { const { done, value } = await reader.read(); if (done) break; received += value.length; const now = Date.now(); if (now - lastUpdate < 250) continue; lastUpdate = now; setDownloadStatusMap(path, { bytes: received }); } blob = await blobPromise; } finally { downloadSemaphore.release(); } await db.put(FileStore, { path, data: blob, metadata: item.metadata, } as StoredFile); updateFileTree(path, item, true); setDownloadStatusMap(path, "SyncedFile"); const basePath = path.split("/").slice(0, -1).join("/"); const fileName = path.split("/").toReversed()[0]; if (currentDir() === basePath) { fetchFiles.mutate((prev) => { prev.set(fileName, item); //new map to force rerender return new Map(prev); }); } } catch (e) { toast(`Downloading ${path} failed`, "error", e); setDownloadStatusMap(path, initialProgress === "SyncedFile" ? "SyncedFile" : undefined); } } export async function download( fileName: string, item: MediaFile | Directory, basePath: string, currentDir: Accessor, options: AppOptions, fetchFiles: ResourceActions, ) { const fullPath = joinPath(basePath, fileName); if ("metadata" in item) { await downloadFile(fullPath, currentDir, options, item, fetchFiles); } else { const initialProgress = downloadStatusMap[fullPath]; setDownloadStatusMap(fullPath, "DirectoryProgress"); await getCover(fullPath, false); try { const promises: Promise[] = []; for (const [childName, childItem] of await listFiles(fullPath, false, false)) { promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles)); } await Promise.allSettled(promises); setDownloadStatusMap(fullPath, "SyncedFile"); } catch { setDownloadStatusMap(fullPath, initialProgress === "SyncedFile" ? "SyncedFile" : undefined); } } } export async function isFileSynced(path: string, status: DownloadStatus | undefined): Promise { if (status === "SyncedFile") return true; //if in progress, it might already exist anyway and just is being re-synced right now if (typeof status === "object" && "bytes" in status) { const db = await openDB(); return await db.exist(FileStore, path); } return false; } interface StoredFile { path: string; data: Blob; metadata: { duration: number; }; } interface StoredCover { path: string; data: Blob | null; } //separate blobs and metadata, so we don't have to load all file contents at startup, which would consume huge amounts of ram, and be very slow const FileStore = "fileblobs"; const CoverStore = "covers"; const PlaylistStore = "playlists"; async function openDB() { return new Promise((resolve, reject) => { const req = indexedDB.open("music-files"); req.onerror = () => reject(req.error); req.onupgradeneeded = () => { const db = req.result; if (!db.objectStoreNames.contains(FileStore)) db.createObjectStore(FileStore, { keyPath: "path" }); if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" }); if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" }); }; req.onsuccess = () => resolve(wrapDB(req.result)); }); } interface DatabaseWrapper { get(store: typeof CoverStore | typeof FileStore, key: string): Promise; put(store: typeof FileStore, value: StoredFile): Promise; put(store: typeof CoverStore, value: StoredCover): Promise; put(store: typeof PlaylistStore, value: Playlist): Promise; add(store: typeof PlaylistStore, value: Playlist): Promise; delete(store: typeof FileStore | typeof CoverStore | typeof PlaylistStore, key: string): Promise; clear(store: string): Promise; listFiles(store: typeof FileStore): Promise; listPlaylists(store: typeof PlaylistStore): Promise; exist(store: typeof FileStore, key: string): Promise; } function wrapDB(db: IDBDatabase): DatabaseWrapper { return { get: (store: string, key: string) => new Promise((res, rej) => { const tx = db.transaction(store, "readonly"); const req = tx.objectStore(store).get(key); req.onsuccess = () => res(req.result); req.onerror = () => rej(req.error); }), put: (store: string, value: unknown) => new Promise((res, rej) => { const tx = db.transaction(store, "readwrite"); const req: IDBRequest = tx.objectStore(store).put(value); req.onsuccess = () => res(); req.onerror = () => rej(req.error); }), add: (store: string, value: unknown) => new Promise((res, rej) => { const tx = db.transaction(store, "readwrite"); const req: IDBRequest = tx.objectStore(store).add(value); req.onsuccess = () => res(); req.onerror = () => rej(req.error); }), delete: (store: string, key: string) => new Promise((res, rej) => { const tx = db.transaction(store, "readwrite"); const req = tx.objectStore(store).delete(key); req.onsuccess = () => res(); req.onerror = () => rej(req.error); }), clear: (store: string) => new Promise((res, rej) => { const tx = db.transaction(store, "readwrite"); const req = tx.objectStore(store).clear(); req.onsuccess = () => res(); req.onerror = () => rej(req.error); }), listFiles: (store: typeof FileStore) => new Promise((res, rej) => { const tx = db.transaction(store, "readonly"); const req = tx.objectStore(store).getAll(); req.onsuccess = () => res(req.result); req.onerror = () => rej(req.error); }), listPlaylists: (store: typeof PlaylistStore) => new Promise((res, rej) => { const tx = db.transaction(store, "readonly"); const req = tx.objectStore(store).getAll(); req.onsuccess = () => res(req.result); req.onerror = () => rej(req.error); }), exist: (store: typeof FileStore, key: string) => new Promise((res, rej) => { const tx = db.transaction(store, "readonly"); const req = tx.objectStore(store).count(key); req.onsuccess = () => res(req.result !== 0); req.onerror = () => rej(req.error); }), }; }