pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import Dexie, { liveQuery } from "dexie";
2import type {
3 Directory,
4 DirectoryWithStatus,
5 FileListing,
6 FileListingWithStatus,
7 MediaFile,
8 Metadata,
9} from "music-server-shared/types";
10import { audioCodecHasBitrate, encodePath } from "music-server-shared/utils";
11import { batch, from, type ResourceActions, untrack } from "solid-js";
12import { createStore } from "solid-js/store";
13import type { FlatFileListing } from "./App";
14import { AsyncSemaphore } from "./semaphore";
15import { type AppOptions, type Playlist, StreamingMode } from "./types";
16import { basename, dirname, handleUnauthorized, joinPath, parentPaths, toast, unproxy } from "./utils";
17
18interface StoredFile {
19 path: string;
20 data: Blob;
21 metadata: Metadata;
22}
23
24interface StoredCover {
25 path: string;
26 data: Blob | null;
27}
28
29interface Database extends Dexie {
30 files: Dexie.Table<StoredFile, string>;
31 covers: Dexie.Table<StoredCover, string>;
32 playlists: Dexie.Table<Playlist, string>;
33}
34
35const db = new Dexie("music-files") as Database;
36db.version(1).stores({
37 files: "path",
38 covers: "path",
39 playlists: "name",
40});
41
42export type Progress =
43 | { status: number; abort: () => void }
44 | { status: "Aborted"; retry: () => void }
45 | { status: "Failed"; retry: () => void; reason: string };
46
47interface LoadingChildren {
48 amount: number;
49}
50
51export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
52export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
53
54//creates entry if it doesn't exist, so reactivity works
55export function getDownloadStatus(path: string): DownloadStatus {
56 untrack(() => {
57 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
58 });
59 return downloadStatusMap[path];
60}
61
62function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
63 function updateProgressAmount(path: string) {
64 const currentStatus = downloadStatusMap[path];
65 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
66 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
67 return;
68 }
69
70 const newAmount = currentStatus.amount + loadChange;
71 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
72 }
73
74 if (includeCurrent) updateProgressAmount(path);
75 // Update all parent directories
76 for (const parent of parentPaths(path)) {
77 updateProgressAmount(parent);
78 }
79}
80
81function startProgress(path: string, status: DownloadStatus) {
82 updateRecursiveStatus(path, 1, false);
83 setDownloadStatusMap(path, status);
84}
85
86function stopProgress(path: string, status: DownloadStatus) {
87 updateRecursiveStatus(path, -1, false);
88 setDownloadStatusMap(path, status);
89}
90
91//initialize filetrees and download statuses from indexedDB
92export async function loadDataFromDB() {
93 try {
94 const files = await db.files.toArray();
95 for (const file of files) {
96 setDownloadStatusMap(file.path, "Synced");
97 updateFileTree(file.path, { metadata: file.metadata }, true);
98 }
99 const covers = await db.covers.toArray();
100 for (const cover of covers) {
101 coverCache.set(cover.path, { data: cover.data });
102 }
103 } catch (e) {
104 toast("Loading offline files failed", "error", e);
105 console.error(e);
106 }
107}
108
109//contains all locally synced files
110const offlineTreeRoot: Directory = {
111 files: {},
112};
113//used for caching online file listings
114const onlineTreeRoot: DirectoryWithStatus = {
115 files: {},
116 status: "Unknown",
117};
118
119//merges "Scanned" directories from "other" to "target"
120function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
121 if (other.status === "Unknown") return;
122 target.status = other.status;
123 for (const [fileName, file] of Object.entries(other.files)) {
124 const targetFile = target.files[fileName];
125 if (targetFile && "files" in targetFile && "files" in file) {
126 mergeTrees(targetFile, file);
127 } else {
128 target.files[fileName] = file;
129 }
130 }
131}
132
133//TODO: improve typing to avoid "as"
134//overloads for typecheking
135//offline tree only stores files, directories are created ad-hoc
136function updateFileTree(path: string, file: MediaFile, offline: true): void;
137//online tree only stores cached directory listings
138function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
139// Updates file tree structure with new data (offline or online)
140function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
141 if (offline && path === "") throw new Error("Trying to update root in offline mode");
142 const basePath = dirname(path);
143 const fileName = basename(path);
144 const node = getDirInTree(basePath, true, offline);
145 //widen so the writes below don't have to satisfy both tree flavors at once
146 const nodeFiles: Record<string, DirectoryWithStatus | Directory | MediaFile> = node.files;
147
148 if (offline) {
149 nodeFiles[fileName] = file;
150 } else {
151 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
152 else {
153 let target = nodeFiles[fileName] as DirectoryWithStatus;
154 if (!target) target = { files: {}, status: "Unknown" };
155 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
156 nodeFiles[fileName] = target;
157 }
158 }
159}
160
161//overloads for typechecking
162function getDirInTree(path: string, create: true, offline: true): Directory;
163function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
164function getDirInTree(path: string, create: false, offline: true): Directory | null;
165function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
166function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
167function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
168// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
169function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
170 let current = offline ? offlineTreeRoot : onlineTreeRoot;
171 if (path === "") return current;
172 const parts = path.split("/");
173
174 for (const part of parts) {
175 //widen so reads and writes don't have to satisfy both tree flavors at once
176 const files: Record<string, DirectoryWithStatus | Directory | MediaFile> = current.files;
177 let next: DirectoryWithStatus | Directory | MediaFile | undefined = files[part];
178 if (!next || "metadata" in next) {
179 if (!create) return null;
180 //if mediafile already exists, we overwrite it with a directory anyway
181 if (offline) next = { files: {} };
182 else next = { files: {}, status: "Unknown" };
183 files[part] = next;
184 }
185 current = next;
186 }
187 return current;
188}
189
190function deleteInOfflineTree(path: string): void {
191 if (path === "") return;
192 const basePath = dirname(path);
193 const fileName = basename(path);
194
195 const node = getDirInTree(basePath, false, true);
196 if (!node) return;
197
198 delete node.files[fileName];
199 if (Object.keys(node.files).length === 0) {
200 db.covers.delete(basePath); //remove stored cover for this dir if there are not files left
201 deleteInOfflineTree(basePath);
202 }
203}
204
205export const clearDirectoryCache = async (path: string) => {
206 const node = getDirInTree(path, false, false);
207 if (!node) return;
208
209 // Save existing children before clearing
210 const preservedChildren = { ...node.files };
211
212 node.files = {};
213 node.status = "Unknown";
214
215 // restore any preserved subtrees where the parent directory still exist
216 const newListing = await listFiles(path, false, false);
217 for (const [name, child] of Object.entries(preservedChildren)) {
218 if ("files" in child && newListing[name] !== undefined) {
219 node.files[name] = child;
220 }
221 }
222};
223async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
224 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`, { credentials: "same-origin" }).catch(
225 () => {},
226 );
227 if (handleUnauthorized(response)) return;
228 if (response?.status === 404) {
229 toast(`Directory not found: ${dir}`, "error");
230 return;
231 }
232 if (!response?.ok) {
233 toast(
234 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
235 "error",
236 );
237 return;
238 }
239
240 return (await response.json()) as FileListingWithStatus;
241}
242
243function listOfflineFiles(dir: string): FileListing {
244 const node = getDirInTree(dir, false, true);
245 if (!node?.files) return {};
246 return node.files;
247}
248
249//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
250function fullyKnown(dir: DirectoryWithStatus) {
251 if (dir.status === "Unknown") {
252 return false;
253 }
254 if (Object.values(dir.files).some((value) => "files" in value && !fullyKnown(value))) {
255 return false;
256 }
257 return true;
258}
259
260//non-recursive listing should not include nested children
261function pruneChildren(listing: FileListing) {
262 const pruned: FileListing = {};
263 for (const [name, item] of Object.entries(listing)) {
264 if ("metadata" in item) pruned[name] = item;
265 else pruned[name] = { files: {} };
266 }
267 return pruned;
268}
269
270//used for combining the online and offline results, to be able to show files that only exist locally
271function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
272 const result: FileListing = { ...listing1 };
273
274 for (const [key, value2] of Object.entries(listing2)) {
275 const value1 = result[key];
276
277 if (value1 && "files" in value1 && "files" in value2) {
278 result[key] = { files: combineFileListings(value1.files, value2.files) };
279 } else {
280 result[key] = value2;
281 }
282 }
283
284 return result;
285}
286
287// File listing logic with hybrid online/offline support
288// Combines:
289// 1. Direct IndexedDB lookups for offline mode
290// 2. Cached online listings
291// 3. Merged online+offline results
292export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
293 if (offline) {
294 const files = listOfflineFiles(dir);
295 return recursive ? files : pruneChildren(files);
296 }
297 const cached = getDirInTree(dir, false, false);
298 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
299 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
300 return recursive ? combined : pruneChildren(combined);
301 }
302 const files = await listOnlineFiles(dir, recursive);
303 if (!files) return {};
304 updateFileTree(dir, { files, status: "Scanned" }, false);
305 const combined = combineFileListings(files, listOfflineFiles(dir));
306 return recursive ? combined : pruneChildren(combined);
307}
308
309const coverCache = new Map<string, { data: Blob | null }>();
310
311//get cover art object-url for a file or directory with cache-first strategy
312export async function getCover(
313 dir: string,
314 options: AppOptions,
315 saveToDb: boolean,
316 offline: boolean,
317 forceRefresh = false,
318) {
319 try {
320 const cached = coverCache.get(dir);
321 if (!forceRefresh && cached !== undefined) {
322 if (saveToDb) db.covers.add({ path: dir, data: cached.data }).catch(() => {});
323 return cached.data === null ? undefined : URL.createObjectURL(cached.data);
324 }
325
326 if (offline) return undefined;
327 const response = await fetch(`/cover/${encodePath(dir)}${options.transcodeCovers ? "?transcode=true" : ""}`, {
328 method: "GET",
329 credentials: "same-origin",
330 }).catch(() => {});
331 if (handleUnauthorized(response)) return undefined;
332 if (!response || response.status === 404) {
333 coverCache.set(dir, { data: null });
334 if (saveToDb) db.covers.add({ path: dir, data: null }).catch(() => {});
335 //only update a row that already exists, otherwise merely browsing a directory would
336 //persist its cover - note the await, without it the promise is always truthy
337 else if (forceRefresh && (await db.covers.get(dir)) !== undefined)
338 db.covers.put({ path: dir, data: null }).catch(() => {});
339 return undefined;
340 }
341 const blob = await response.blob();
342 coverCache.set(dir, { data: blob });
343 if (saveToDb) db.covers.add({ path: dir, data: blob }).catch(() => {});
344 //see above: the await is what makes this an "already stored" check
345 else if (forceRefresh && (await db.covers.get(dir)) !== undefined)
346 db.covers.put({ path: dir, data: blob }).catch(() => {});
347 return URL.createObjectURL(blob);
348 } catch {
349 return undefined;
350 }
351}
352
353//returns an object url to the blob if synced, otherwise the server url
354//with forceUrl it always resolves, without it returns undefined for an uncached file in offline mode
355export async function getSrc(
356 path: string,
357 metadata: Metadata,
358 options: AppOptions,
359 offline: boolean,
360 seekTo: number,
361 forceUrl: true,
362): Promise<{ src: string; isVideo?: boolean }>;
363export async function getSrc(
364 path: string,
365 metadata: Metadata,
366 options: AppOptions,
367 offline: boolean,
368 seekTo: number,
369): Promise<{ src: string; isVideo?: boolean } | undefined>;
370export async function getSrc(
371 path: string,
372 metadata: Metadata,
373 options: AppOptions,
374 offline: boolean,
375 seekTo: number,
376 forceUrl?: boolean,
377) {
378 if (!forceUrl) {
379 const dbResult = await db.files.get(path);
380 if (dbResult) return { src: URL.createObjectURL(dbResult.data), isVideo: undefined };
381
382 if (offline) {
383 toast("This file is not cached, leave offline mode to play it", "error");
384 return undefined;
385 }
386 }
387 if (options.enableTranscoding) {
388 const isVideo = metadata.videoCodec !== undefined;
389 const settings = isVideo ? options.videoSettings : options.audioSettings;
390 const disableChunkedTranscoding = settings.streamingMode === StreamingMode.buffered;
391 const audioBitrate = audioCodecHasBitrate(settings.audioCodec) ? settings.audioBitrate : undefined;
392
393 return {
394 src:
395 `${joinPath("transcode", encodePath(path))}` +
396 `?disableChunkedTranscoding=${disableChunkedTranscoding}` +
397 `&container=${settings.container}` +
398 `${isVideo ? `&videoCodec=${options.videoSettings.videoCodec}` : ""}` +
399 `${isVideo ? `&videoBitrate=${options.videoSettings.videoBitrate}` : ""}` +
400 `${isVideo ? `&videoEncodingSetting=${options.videoSettings.videoEncodingSetting}` : ""}` +
401 `&audioCodec=${settings.audioCodec}` +
402 `${audioBitrate === undefined ? "" : `&audioBitrate=${audioBitrate}`}` +
403 `${seekTo === 0 ? "" : `&seekTo=${seekTo}`}` +
404 `&languages=${options.languages}`,
405 isVideo: isVideo,
406 };
407 }
408 return { src: joinPath("download", encodePath(path)), isVideo: undefined };
409}
410
411export async function loadPlaylists() {
412 return await db.playlists.toArray();
413}
414
415//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
416export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
417 playlist.items = playlist.items.map((item) => {
418 //some less space in the db by omitting ids
419 return { path: item.path, metadata: unproxy(item.metadata) };
420 });
421
422 if (overwrite) {
423 await db.playlists.put(playlist);
424 } else {
425 await db.playlists.add(playlist);
426 }
427}
428
429export async function deletePlaylist(name: string) {
430 await db.playlists.delete(name);
431}
432
433function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
434 const paths: string[] = [];
435
436 for (const [name, item] of Object.entries(listing)) {
437 const currentPath = joinPath(basePath, name);
438
439 if ("metadata" in item) {
440 paths.push(currentPath);
441 } else if ("files" in item) {
442 paths.push(...collectMediaFilePaths(item.files, currentPath));
443 }
444 }
445
446 return paths;
447}
448
449//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
450export async function deleteFile(
451 fullPath: string,
452 item: MediaFile | Directory,
453 fetchFiles: ResourceActions<FlatFileListing>,
454) {
455 try {
456 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
457 startProgress(fullPath, { amount: 1 });
458
459 let toDelete: string[];
460 if ("files" in item) {
461 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
462 } else {
463 toDelete = [fullPath];
464 }
465
466 //covers are keyed by file path (embedded art) and by directory path (directory art), so deleting a
467 //directory also has to drop its own cover row plus every nested one, or they stay orphaned forever
468 const coversToDelete = new Set(toDelete);
469 if ("files" in item) {
470 coversToDelete.add(fullPath);
471 for (const filePath of toDelete) {
472 for (const dir of parentPaths(filePath)) {
473 //stop at the deleted root, parents of it may still hold other synced files
474 if (dir.length <= fullPath.length) break;
475 coversToDelete.add(dir);
476 }
477 }
478 }
479
480 await db.files.bulkDelete(toDelete); //delete file from database
481 await db.covers.bulkDelete([...coversToDelete]); //delete embedded/directory covers from database
482 //also drop them from the session cache, otherwise a later getCover(saveToDb) would re-add them
483 for (const coverPath of coversToDelete) coverCache.delete(coverPath);
484 deleteInOfflineTree(fullPath); //delete node from tree
485 batch(() => {
486 for (const path of toDelete) {
487 setDownloadStatusMap(path, undefined); // set all children to unsynced
488 }
489 });
490 fetchFiles.refetch();
491 } finally {
492 stopProgress(fullPath, undefined);
493 }
494}
495
496const downloadSemaphore = new AsyncSemaphore(1);
497
498//Download file to indexedDB and updates the offline file tree
499//concurrency control via the downloadSemaphore
500async function downloadFile(
501 path: string,
502 options: AppOptions,
503 item: MediaFile,
504 fetchFiles: ResourceActions<FlatFileListing>,
505 redownload: boolean,
506): Promise<void> {
507 const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
508
509 // file already downloaded or download already in progress
510 const initialProgress = downloadStatusMap[path];
511 if (
512 (!redownload && initialProgress === "Synced") ||
513 //an in-progress download carries a numeric byte count, without this check clicking a
514 //downloading item would start a second fetch racing the first one on db.files.put
515 (typeof initialProgress === "object" && "status" in initialProgress && typeof initialProgress.status === "number")
516 )
517 return;
518
519 const controller = new AbortController();
520 const abortDownload = () => {
521 controller.abort();
522 stopProgress(
523 path,
524 // if it was already synced, restore it
525 initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
526 );
527 };
528 startProgress(path, { status: 0, abort: abortDownload });
529
530 try {
531 //options might change at any point, update the semaphore to the current value
532 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
533
534 //make sure to only allow the set amount of parallel downloads. passing the signal makes an abort
535 //while still queued reject here instead of entering the section below, so the release() in its
536 //finally can't hand out a permit that was never acquired (which would leak a concurrency slot)
537 await downloadSemaphore.acquire(controller.signal);
538 try {
539 const res = await fetch((await getSrc(path, item.metadata, options, false, 0, true)).src, {
540 signal: controller.signal,
541 credentials: "same-origin",
542 });
543
544 handleUnauthorized(res);
545 if (!res.ok || !res.body) throw res.statusText;
546 //read the body ourselves to track progress, and assemble the blob from the same chunks -
547 //res.clone().blob() would buffer the whole body a second time and reject unobserved on a mid-body error
548 const reader = res.body.getReader();
549 //a fetch body never yields SharedArrayBuffer-backed views, so narrowing to ArrayBuffer is safe
550 const chunks: Uint8Array<ArrayBuffer>[] = [];
551 let received = 0;
552
553 let lastUpdate = 0;
554 while (true) {
555 const { done, value } = await reader.read();
556 if (done) break;
557 chunks.push(value as Uint8Array<ArrayBuffer>);
558 received += value.length;
559
560 //debouncing logic to not update UI too fast
561 const now = Date.now();
562 if (now - lastUpdate < 250) continue;
563 lastUpdate = now;
564 setDownloadStatusMap(path, { status: received, abort: abortDownload });
565 }
566 const blob = new Blob(chunks, { type: res.headers.get("content-type") ?? undefined });
567
568 await db.files.put({
569 path,
570 data: blob,
571 metadata: unproxy(item.metadata),
572 });
573
574 updateFileTree(path, item, true);
575 stopProgress(path, "Synced");
576 fetchFiles.refetch();
577
578 //iterate through all parent directories and download their covers to database
579 for (const dir of [path, ...parentPaths(path)]) {
580 getCover(dir, options, true, false).then((cover) => URL.revokeObjectURL(cover || ""));
581 }
582 } finally {
583 downloadSemaphore.release();
584 }
585 } catch (e) {
586 if (e instanceof Error && e.name === "AbortError") return;
587 stopProgress(
588 path,
589 // if it was already synced, restore it
590 initialProgress === "Synced"
591 ? "Synced"
592 : {
593 status: "Failed",
594 reason: e ? e.toString() : "Unknown error",
595 retry,
596 },
597 );
598 }
599}
600
601export async function download(
602 fullPath: string,
603 item: MediaFile | Directory,
604 options: AppOptions,
605 fetchFiles: ResourceActions<FlatFileListing>,
606 redownload = false,
607) {
608 if ("metadata" in item) {
609 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
610 } else {
611 updateRecursiveStatus(fullPath, 1, true);
612 try {
613 //start downloads for all children to allow them to run in parallel, then wait for the results
614 const promises: Promise<unknown>[] = [];
615 const listing = await listFiles(fullPath, false, false);
616 batch(() => {
617 for (const [childName, childItem] of Object.entries(listing)) {
618 promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload));
619 }
620 });
621 await Promise.allSettled(promises);
622 } finally {
623 updateRecursiveStatus(fullPath, -1, true);
624 }
625 }
626}
627
628export const storedFilesLive = from(
629 liveQuery(async () => await db.files.toCollection().keys()),
630 [],
631);
632