import Dexie, { liveQuery } from "dexie"; import { AsyncSemaphore } from "music-server-shared/semaphore"; import type { Chapter, Directory, DirectoryWithStatus, FileListing, FileListingWithStatus, MediaFile, Metadata, SubtitleTrack, VideoExtras, } from "music-server-shared/types"; import { audioCodecHasBitrate, encodePath } from "music-server-shared/utils"; import { batch, from, type ResourceActions, untrack } from "solid-js"; import { createStore } from "solid-js/store"; import type { FlatFileListing } from "./App"; import { type AppOptions, type LoadedVideoExtras, type Playlist, StreamingMode } from "./types"; import { basename, dirname, handleUnauthorized, joinPath, parentPaths, toast, unproxy } from "./utils"; interface StoredSubtitle extends SubtitleTrack { data: Blob; } interface StoredVideoExtras { chapters: Chapter[]; subtitles: StoredSubtitle[]; } interface StoredFile { path: string; data: Blob; metadata: Metadata; videoExtras?: StoredVideoExtras; } interface StoredCover { path: string; data: Blob | null; } interface Database extends Dexie { files: Dexie.Table; covers: Dexie.Table; playlists: Dexie.Table; } const db = new Dexie("music-files") as Database; db.version(1).stores({ files: "path", covers: "path", playlists: "name", }); export type Progress = | { status: number; abort: () => void } | { status: "Aborted"; retry: () => void } | { status: "Failed"; retry: () => void; reason: string }; interface LoadingChildren { amount: number; } export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined; export const [downloadStatusMap, setDownloadStatusMap] = createStore>({}); //creates entry if it doesn't exist, so reactivity works export function getDownloadStatus(path: string): DownloadStatus { untrack(() => { if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined); }); return downloadStatusMap[path]; } function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) { function updateProgressAmount(path: string) { const currentStatus = downloadStatusMap[path]; if (typeof currentStatus !== "object" || !("amount" in currentStatus)) { setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined); return; } const newAmount = currentStatus.amount + loadChange; setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined); } if (includeCurrent) updateProgressAmount(path); // Update all parent directories for (const parent of parentPaths(path)) { updateProgressAmount(parent); } } function startProgress(path: string, status: DownloadStatus) { updateRecursiveStatus(path, 1, false); setDownloadStatusMap(path, status); } function stopProgress(path: string, status: DownloadStatus) { updateRecursiveStatus(path, -1, false); setDownloadStatusMap(path, status); } //initialize filetrees and download statuses from indexedDB export async function loadDataFromDB() { try { const files = await db.files.toArray(); for (const file of files) { setDownloadStatusMap(file.path, "Synced"); updateFileTree(file.path, { metadata: file.metadata }, true); } const covers = await db.covers.toArray(); for (const cover of covers) { coverCache.set(cover.path, { data: cover.data }); } } catch (e) { toast("Loading offline files failed", "error", e); console.error(e); } } //contains all locally synced files const offlineTreeRoot: Directory = { files: {}, }; //used for caching online file listings const onlineTreeRoot: DirectoryWithStatus = { files: {}, status: "Unknown", }; //merges "Scanned" directories from "other" to "target" function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) { if (other.status === "Unknown") return; target.status = other.status; for (const [fileName, file] of Object.entries(other.files)) { const targetFile = target.files[fileName]; if (targetFile && "files" in targetFile && "files" in file) { mergeTrees(targetFile, file); } else { target.files[fileName] = file; } } } //TODO: improve typing to avoid "as" //overloads for typecheking //offline tree only stores files, directories are created ad-hoc function updateFileTree(path: string, file: MediaFile, offline: true): void; //online tree only stores cached directory listings function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void; // Updates file tree structure with new data (offline or online) function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) { if (offline && path === "") throw new Error("Trying to update root in offline mode"); const basePath = dirname(path); const fileName = basename(path); const node = getDirInTree(basePath, true, offline); //widen so the writes below don't have to satisfy both tree flavors at once const nodeFiles: Record = node.files; if (offline) { nodeFiles[fileName] = file; } else { if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus); else { let target = nodeFiles[fileName] as DirectoryWithStatus; if (!target) target = { files: {}, status: "Unknown" }; mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus); nodeFiles[fileName] = target; } } } //overloads for typechecking function getDirInTree(path: string, create: true, offline: true): Directory; function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus; function getDirInTree(path: string, create: false, offline: true): Directory | null; function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null; function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory; function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null; // get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null { let current = offline ? offlineTreeRoot : onlineTreeRoot; if (path === "") return current; const parts = path.split("/"); for (const part of parts) { //widen so reads and writes don't have to satisfy both tree flavors at once const files: Record = current.files; let next: DirectoryWithStatus | Directory | MediaFile | undefined = files[part]; if (!next || "metadata" in next) { if (!create) return null; //if mediafile already exists, we overwrite it with a directory anyway if (offline) next = { files: {} }; else next = { files: {}, status: "Unknown" }; files[part] = next; } current = next; } return current; } function deleteInOfflineTree(path: string): void { if (path === "") return; const basePath = dirname(path); const fileName = basename(path); const node = getDirInTree(basePath, false, true); if (!node) return; delete node.files[fileName]; if (Object.keys(node.files).length === 0) { db.covers.delete(basePath); //remove stored cover for this dir if there are not files left deleteInOfflineTree(basePath); } } export const clearDirectoryCache = async (path: string) => { const node = getDirInTree(path, false, false); if (!node) return; // Save existing children before clearing const preservedChildren = { ...node.files }; node.files = {}; node.status = "Unknown"; // restore any preserved subtrees where the parent directory still exist const newListing = await listFiles(path, false, false); for (const [name, child] of Object.entries(preservedChildren)) { if ("files" in child && newListing[name] !== undefined) { node.files[name] = child; } } }; async function listOnlineFiles(dir: string, recursive: boolean): Promise { const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`, { credentials: "same-origin" }).catch( () => {}, ); if (handleUnauthorized(response)) return; if (response?.status === 404) { toast(`Directory not found: ${dir}`, "error"); return; } if (!response?.ok) { toast( `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`, "error", ); return; } return (await response.json()) as FileListingWithStatus; } function listOfflineFiles(dir: string): FileListing { const node = getDirInTree(dir, false, true); if (!node?.files) return {}; return node.files; } //checks if this Directory is completely scanned, i.e. suitable for a recursive listing result function fullyKnown(dir: DirectoryWithStatus) { if (dir.status === "Unknown") { return false; } if (Object.values(dir.files).some((value) => "files" in value && !fullyKnown(value))) { return false; } return true; } //non-recursive listing should not include nested children function pruneChildren(listing: FileListing) { const pruned: FileListing = {}; for (const [name, item] of Object.entries(listing)) { if ("metadata" in item) pruned[name] = item; else pruned[name] = { files: {} }; } return pruned; } //used for combining the online and offline results, to be able to show files that only exist locally function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing { const result: FileListing = { ...listing1 }; for (const [key, value2] of Object.entries(listing2)) { const value1 = result[key]; if (value1 && "files" in value1 && "files" in value2) { result[key] = { files: combineFileListings(value1.files, value2.files) }; } else { result[key] = value2; } } return result; } // File listing logic with hybrid online/offline support // Combines: // 1. Direct IndexedDB lookups for offline mode // 2. Cached online listings // 3. Merged online+offline results export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise { if (offline) { const files = listOfflineFiles(dir); return recursive ? files : pruneChildren(files); } const cached = getDirInTree(dir, false, false); if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) { const combined = combineFileListings(cached.files, listOfflineFiles(dir)); return recursive ? combined : pruneChildren(combined); } const files = await listOnlineFiles(dir, recursive); if (!files) return {}; updateFileTree(dir, { files, status: "Scanned" }, false); const combined = combineFileListings(files, listOfflineFiles(dir)); return recursive ? combined : pruneChildren(combined); } const coverCache = new Map(); //get cover art object-url for a file or directory with cache-first strategy export async function getCover( dir: string, options: AppOptions, saveToDb: boolean, offline: boolean, forceRefresh = false, ) { try { const cached = coverCache.get(dir); if (!forceRefresh && cached !== undefined) { if (saveToDb) db.covers.add({ path: dir, data: cached.data }).catch(() => {}); return cached.data === null ? undefined : URL.createObjectURL(cached.data); } if (offline) return undefined; const response = await fetch(`/cover/${encodePath(dir)}${options.transcodeCovers ? "?transcode=true" : ""}`, { method: "GET", credentials: "same-origin", }).catch(() => {}); if (handleUnauthorized(response)) return undefined; if (!response || response.status === 404) { coverCache.set(dir, { data: null }); if (saveToDb) db.covers.add({ path: dir, data: null }).catch(() => {}); //only update a row that already exists, otherwise merely browsing a directory would //persist its cover - note the await, without it the promise is always truthy if (!saveToDb && forceRefresh && (await db.covers.get(dir)) !== undefined) db.covers.put({ path: dir, data: null }).catch(() => {}); return undefined; } if (!response.ok) return undefined; const blob = await response.blob(); coverCache.set(dir, { data: blob }); if (saveToDb) db.covers.add({ path: dir, data: blob }).catch(() => {}); //see above: the await is what makes this an "already stored" check if (!saveToDb && forceRefresh && (await db.covers.get(dir)) !== undefined) db.covers.put({ path: dir, data: blob }).catch(() => {}); return URL.createObjectURL(blob); } catch { return undefined; } } const emptyVideoExtras = (): LoadedVideoExtras => ({ subtitleTracks: [], chapters: [], subtitleSources: {} }); function subtitleUrl(path: string, streamIndex: number): string { return `/subtitles/${encodePath(path)}?track=${streamIndex}`; } function videoInfoUrl(path: string): string { return `/video-info/${encodePath(path)}`; } function checkVideoExtrasResponse(response: Response | undefined, path: string): asserts response is Response { if (handleUnauthorized(response)) throw new Error("Unauthorized while fetching video extras"); if (!response?.ok) throw new Error(`Fetching video extras for ${path} failed: ${response?.statusText || "server unavailable"}`); } export async function loadVideoExtras(path: string, offline: boolean): Promise { const stored = await db.files.get(path); if (stored?.videoExtras) { const subtitleSources: Record = {}; const subtitleTracks: SubtitleTrack[] = []; for (const subtitle of stored.videoExtras.subtitles) { const { data, ...track } = subtitle; subtitleTracks.push(track); subtitleSources[track.streamIndex] = URL.createObjectURL(data); } return { subtitleTracks, chapters: stored.videoExtras.chapters, subtitleSources }; } if (offline) return emptyVideoExtras(); const response = await fetch(videoInfoUrl(path), { credentials: "same-origin" }).catch(() => undefined); checkVideoExtrasResponse(response, path); const extras = (await response.json()) as VideoExtras; const subtitleSources: Record = {}; for (const track of extras.subtitleTracks) subtitleSources[track.streamIndex] = subtitleUrl(path, track.streamIndex); return { ...extras, subtitleSources }; } export function revokeVideoExtras(extras: LoadedVideoExtras | undefined): void { if (!extras) return; for (const source of Object.values(extras.subtitleSources)) { if (source.startsWith("blob:")) URL.revokeObjectURL(source); } } //Unlike normal playback metadata, sync treats extras as part of the file request: a failed chapter probe //or a failed subtitle conversion rejects before the media blob is committed to IndexedDB. async function downloadVideoExtras(path: string, signal: AbortSignal): Promise { const response = await fetch(videoInfoUrl(path), { credentials: "same-origin", signal }).catch((error) => { throw error; }); checkVideoExtrasResponse(response, path); const extras = (await response.json()) as VideoExtras; const subtitles = await Promise.all( extras.subtitleTracks.map(async (track) => { const subtitleResponse = await fetch(subtitleUrl(path, track.streamIndex), { credentials: "same-origin", signal, }).catch((error) => { throw error; }); if (handleUnauthorized(subtitleResponse) || !subtitleResponse.ok) { throw new Error( `Fetching subtitle track ${track.streamIndex} for ${path} failed: ${subtitleResponse?.statusText || "server unavailable"}`, ); } return { ...track, data: await subtitleResponse.blob() }; }), ); return { chapters: extras.chapters, subtitles }; } //returns an object url to the blob if synced, otherwise the server url //with forceUrl it always resolves, without it returns undefined for an uncached file in offline mode export async function getSrc( path: string, metadata: Metadata, options: AppOptions, offline: boolean, seekTo: number, forceUrl = false, ): Promise<{ src: string; isVideo?: boolean; transcodeId?: string } | undefined> { if (!forceUrl) { const dbResult = await db.files.get(path); if (dbResult) return { src: URL.createObjectURL(dbResult.data), isVideo: undefined }; if (offline) { toast("This file is not cached, leave offline mode to play it", "error"); return undefined; } } if (options.enableTranscoding) { const isVideo = metadata.videoCodec !== undefined; const settings = isVideo ? options.videoSettings : options.audioSettings; const disableChunkedTranscoding = settings.streamingMode === StreamingMode.buffered; const audioBitrate = audioCodecHasBitrate(settings.audioCodec) ? settings.audioBitrate : undefined; const transcodeId = crypto.randomUUID(); const query = new URLSearchParams({ disableChunkedTranscoding: String(disableChunkedTranscoding), container: settings.container, audioCodec: settings.audioCodec, audioLanguagePreference: options.audioLanguagePreference, transcodeId, }); if (isVideo) { query.set("videoCodec", options.videoSettings.videoCodec); query.set("videoBitrate", options.videoSettings.videoBitrate.toString()); query.set("videoEncodingSetting", options.videoSettings.videoEncodingSetting); } if (audioBitrate !== undefined) query.set("audioBitrate", audioBitrate.toString()); if (seekTo !== 0) query.set("seekTo", seekTo.toString()); return { src: `${joinPath("transcode", encodePath(path))}?${query}`, isVideo: isVideo, transcodeId, }; } return { src: joinPath("download", encodePath(path)), isVideo: undefined }; } export async function loadPlaylists() { return await db.playlists.toArray(); } //save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false export async function savePlaylist(playlist: Playlist, overwrite: boolean) { playlist.items = playlist.items.map((item) => { //some less space in the db by omitting ids return { path: item.path, metadata: unproxy(item.metadata) }; }); if (overwrite) { await db.playlists.put(playlist); } else { await db.playlists.add(playlist); } } export async function deletePlaylist(name: string) { await db.playlists.delete(name); } function collectMediaFilePaths(listing: FileListing, basePath: string): string[] { const paths: string[] = []; for (const [name, item] of Object.entries(listing)) { const currentPath = joinPath(basePath, name); if ("metadata" in item) { paths.push(currentPath); } else if ("files" in item) { paths.push(...collectMediaFilePaths(item.files, currentPath)); } } return paths; } //deletes file or directory(+recursive contents) from indexedDB and updates their download statuses export async function deleteFile( fullPath: string, item: MediaFile | Directory, fetchFiles: ResourceActions, ) { try { if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return; startProgress(fullPath, { amount: 1 }); let toDelete: string[]; if ("files" in item) { toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath); } else { toDelete = [fullPath]; } //covers are keyed by file path (embedded art) and by directory path (directory art), so deleting a //directory also has to drop its own cover row plus every nested one, or they stay orphaned forever const coversToDelete = new Set(toDelete); if ("files" in item) { coversToDelete.add(fullPath); for (const filePath of toDelete) { for (const dir of parentPaths(filePath)) { //stop at the deleted root itself, parents of it may still hold other synced files if (dir === fullPath) break; coversToDelete.add(dir); } } } await db.files.bulkDelete(toDelete); //delete file from database await db.covers.bulkDelete([...coversToDelete]); //delete embedded/directory covers from database //also drop them from the session cache, otherwise a later getCover(saveToDb) would re-add them for (const coverPath of coversToDelete) coverCache.delete(coverPath); deleteInOfflineTree(fullPath); //delete node from tree batch(() => { for (const path of toDelete) { setDownloadStatusMap(path, undefined); // set all children to unsynced } }); fetchFiles.refetch(); } finally { stopProgress(fullPath, undefined); } } const downloadSemaphore = new AsyncSemaphore(1); //Download file to indexedDB and updates the offline file tree //concurrency control via the downloadSemaphore async function downloadFile( path: string, options: AppOptions, item: MediaFile, fetchFiles: ResourceActions, redownload: boolean, ): Promise { const retry = () => downloadFile(path, options, item, fetchFiles, redownload); // file already downloaded or download already in progress const initialProgress = downloadStatusMap[path]; if ( (!redownload && initialProgress === "Synced") || //an in-progress download carries a numeric byte count, without this check clicking a //downloading item would start a second fetch racing the first one on db.files.put (typeof initialProgress === "object" && "status" in initialProgress && typeof initialProgress.status === "number") ) return; const controller = new AbortController(); const abortDownload = () => { controller.abort(); stopProgress( path, // if it was already synced, restore it initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry }, ); }; startProgress(path, { status: 0, abort: abortDownload }); try { //options might change at any point, update the semaphore to the current value downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads); //make sure to only allow the set amount of parallel downloads. passing the signal makes an abort //while still queued reject here instead of entering the section below, so the release() in its //finally can't hand out a permit that was never acquired (which would leak a concurrency slot) await downloadSemaphore.acquire(controller.signal); try { const source = await getSrc(path, item.metadata, options, false, 0, true); if (!source) throw new Error("Unable to create a download URL"); const res = await fetch(source.src, { signal: controller.signal, credentials: "same-origin", }); handleUnauthorized(res); if (!res.ok || !res.body) throw res.statusText; //read the body ourselves to track progress, and assemble the blob from the same chunks - //res.clone().blob() would buffer the whole body a second time and reject unobserved on a mid-body error const reader = res.body.getReader(); //a fetch body never yields SharedArrayBuffer-backed views, so narrowing to ArrayBuffer is safe const chunks: Uint8Array[] = []; let received = 0; let lastUpdate = 0; while (true) { const { done, value } = await reader.read(); if (done) break; chunks.push(value as Uint8Array); received += value.length; //debouncing logic to not update UI too fast const now = Date.now(); if (now - lastUpdate < 250) continue; lastUpdate = now; setDownloadStatusMap(path, { status: received, abort: abortDownload }); } const blob = new Blob(chunks, { type: res.headers.get("content-type") ?? undefined }); const isVideo = item.metadata.videoCodec !== undefined; //A video sync is transactional from the user's perspective: subtitle conversion and chapter //extraction complete before the file row becomes Synced. A failed extra therefore leaves a //previous redownload untouched and never exposes a half-synced video offline. const videoExtras = isVideo ? await downloadVideoExtras(path, controller.signal) : undefined; if (controller.signal.aborted) return; const storedFile: StoredFile = { path, data: blob, metadata: unproxy(item.metadata), ...(videoExtras ? { videoExtras } : {}), }; await db.files.put(storedFile); updateFileTree(path, item, true); stopProgress(path, "Synced"); fetchFiles.refetch(); //iterate through all parent directories and download their covers to database for (const dir of [path, ...parentPaths(path)]) { getCover(dir, options, true, false).then((cover) => URL.revokeObjectURL(cover || "")); } } finally { downloadSemaphore.release(); } } catch (e) { if (e instanceof Error && e.name === "AbortError") return; stopProgress( path, // if it was already synced, restore it initialProgress === "Synced" ? "Synced" : { status: "Failed", reason: e ? e.toString() : "Unknown error", retry, }, ); } } export async function download( fullPath: string, item: MediaFile | Directory, options: AppOptions, fetchFiles: ResourceActions, redownload = false, ) { if ("metadata" in item) { await downloadFile(fullPath, options, item, fetchFiles, redownload || false); } else { updateRecursiveStatus(fullPath, 1, true); try { //start downloads for all children to allow them to run in parallel, then wait for the results const promises: Promise[] = []; const listing = await listFiles(fullPath, false, false); batch(() => { for (const [childName, childItem] of Object.entries(listing)) { promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload)); } }); await Promise.allSettled(promises); } finally { updateRecursiveStatus(fullPath, -1, true); } } } //aborts all in-progress downloads and deletes every synced file and cover from indexedDB. export async function clearAllDownloads(fetchFiles: ResourceActions) { //abort first, so in-flight fetches reject with AbortError instead of writing to a cleared //database. a download already past its fetch (sitting at db.files.put) cannot be aborted this //way and may re-insert one row after the clear - harmless, and gone on the next clear/reload for (const status of Object.values(downloadStatusMap)) { if (status && typeof status === "object" && "status" in status && typeof status.status === "number") { status.abort(); } } await db.files.clear(); await db.covers.clear(); coverCache.clear(); offlineTreeRoot.files = {}; batch(() => { for (const path of Object.keys(downloadStatusMap)) { setDownloadStatusMap(path, undefined); } }); fetchFiles.refetch(); } //deletes every local trace of the app export async function clearLocalData(): Promise { localStorage.clear(); db.close(); if ("serviceWorker" in navigator) { const registrations = await navigator.serviceWorker.getRegistrations().catch(() => []); await Promise.all(registrations.map((registration) => registration.unregister().catch(() => false))); } if ("caches" in window) { const cacheNames = await caches.keys().catch(() => []); //awaited, otherwise the reload races the cleanup and can leave caches behind await Promise.all(cacheNames.map((cacheName) => caches.delete(cacheName).catch(() => false))); } //onsuccess alone can hang forever: another open tab blocks the delete, and errors never resolve return await new Promise((resolve) => { const request = indexedDB.deleteDatabase("music-files"); request.onsuccess = () => resolve(undefined); request.onerror = () => resolve("Deleting the local database failed, close all other tabs and try again."); request.onblocked = () => resolve("Another tab still has this app open, close it and try again to delete the local database."); }); } export const storedFilesLive = from( liveQuery(async () => await db.files.toCollection().keys()), [], ); export const storageStatsLive = from( liveQuery(async () => { let fileBytes = 0; await db.files.each((file) => { fileBytes += file.data.size; for (const subtitle of file.videoExtras?.subtitles || []) fileBytes += subtitle.data.size; }); return { fileCount: await db.files.count(), fileBytes, coverCount: await db.covers.count(), playlistCount: await db.playlists.count(), }; }), { fileCount: 0, fileBytes: 0, coverCount: 0, playlistCount: 0 }, );