pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import Dexie, { liveQuery } from "dexie";
2import { parseMap } from "music-server-shared/mapconversion";
3import type {
4 Directory,
5 DirectoryWithStatus,
6 FileListing,
7 FileListingWithStatus,
8 MediaFile,
9 Metadata,
10} from "music-server-shared/types";
11import { audioCodecHasBitrate, encodePath } from "music-server-shared/utils";
12import { batch, from, type ResourceActions, untrack } from "solid-js";
13import { createStore } from "solid-js/store";
14import type { FlatFileListing } from "./App";
15import { AsyncSemaphore } from "./semaphore";
16import { type AppOptions, type Playlist, StreamingMode } from "./types";
17import { handleUnauthorized, joinPath, toast, unproxy } from "./utils";
18
19interface StoredFile {
20 path: string;
21 data: Blob;
22 metadata: Metadata;
23}
24
25interface StoredCover {
26 path: string;
27 data: Blob | null;
28}
29
30interface Database extends Dexie {
31 files: Dexie.Table<StoredFile, string>;
32 covers: Dexie.Table<StoredCover, string>;
33 playlists: Dexie.Table<Playlist, string>;
34}
35
36const db = new Dexie("music-files") as Database;
37db.version(1).stores({
38 files: "path",
39 covers: "path",
40 playlists: "name",
41});
42
43export type Progress =
44 | { status: number; abort: () => void }
45 | { status: "Aborted"; retry: () => void }
46 | { status: "Failed"; retry: () => void; reason: string };
47
48interface LoadingChildren {
49 amount: number;
50}
51
52export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
53export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
54
55//creates entry if it doesn't exist, so reactivity works
56export function getDownloadStatus(path: string): DownloadStatus {
57 untrack(() => {
58 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
59 });
60 return downloadStatusMap[path];
61}
62
63function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
64 function updateProgressAmount(path: string) {
65 const currentStatus = downloadStatusMap[path];
66 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
67 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
68 return;
69 }
70
71 const newAmount = currentStatus.amount + loadChange;
72 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
73 }
74
75 if (includeCurrent) updateProgressAmount(path);
76 // Update all parent directories
77 let currentPath = path;
78 while (true) {
79 const lastSlash = currentPath.lastIndexOf("/");
80 if (lastSlash === -1) break;
81 currentPath = currentPath.substring(0, lastSlash);
82 updateProgressAmount(currentPath);
83 }
84}
85
86function startProgress(path: string, status: DownloadStatus) {
87 updateRecursiveStatus(path, 1, false);
88 setDownloadStatusMap(path, status);
89}
90
91function stopProgress(path: string, status: DownloadStatus) {
92 updateRecursiveStatus(path, -1, false);
93 setDownloadStatusMap(path, status);
94}
95
96//initialize filetrees and download statuses from indexedDB
97export async function loadDataFromDB() {
98 try {
99 const files = await db.files.toArray();
100 for (const file of files) {
101 setDownloadStatusMap(file.path, "Synced");
102 updateFileTree(file.path, { metadata: file.metadata }, true);
103 }
104 const covers = await db.covers.toArray();
105 for (const cover of covers) {
106 coverCache.set(cover.path, { data: cover.data });
107 }
108 } catch (e) {
109 toast("Loading offline files failed", "error", e);
110 console.error(e);
111 }
112}
113
114//contains all locally synced files
115const offlineTreeRoot: Directory = {
116 files: new Map(),
117};
118//used for caching online file listings
119const onlineTreeRoot: DirectoryWithStatus = {
120 files: new Map(),
121 status: "Unknown",
122};
123
124//merges "Scanned" directories from "other" to "target"
125function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
126 if (other.status === "Unknown") return;
127 target.status = other.status;
128 for (const [fileName, file] of other.files.entries()) {
129 const targetFile = target.files.get(fileName);
130 if (targetFile && "files" in targetFile && "files" in file) {
131 mergeTrees(targetFile, file);
132 } else {
133 target.files.set(fileName, file);
134 }
135 }
136}
137
138//TODO: improve typing to avoid "as"
139//overloads for typecheking
140//offline tree only stores files, directories are created ad-hoc
141function updateFileTree(path: string, file: MediaFile, offline: true): void;
142//online tree only stores cached directory listings
143function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
144// Updates file tree structure with new data (offline or online)
145function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
146 if (offline && path === "") throw new Error("Trying to update root in offline mode");
147 const basePath = path.split("/").slice(0, -1).join("/");
148 const fileName = path.split("/").toReversed()[0];
149 const node = getDirInTree(basePath, true, offline);
150
151 if (offline) {
152 node.files.set(fileName, file);
153 } else {
154 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
155 else {
156 let target = node.files.get(fileName) as DirectoryWithStatus;
157 if (!target) target = { files: new Map(), status: "Unknown" };
158 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
159 node.files.set(fileName, target);
160 }
161 }
162}
163
164//overloads for typechecking
165function getDirInTree(path: string, create: true, offline: true): Directory;
166function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
167function getDirInTree(path: string, create: false, offline: true): Directory | null;
168function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
169function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
170function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
171// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
172function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
173 let current = offline ? offlineTreeRoot : onlineTreeRoot;
174 if (path === "") return current;
175 const parts = path.split("/");
176
177 for (const part of parts) {
178 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
179 if (!next || "metadata" in next) {
180 if (!create) return null;
181 //if mediafile already exists, we overwrite it with a directory anyway
182 if (offline) next = { files: new Map() };
183 else next = { files: new Map(), status: "Unknown" };
184 current.files.set(part, next);
185 }
186 current = next;
187 }
188 return current;
189}
190
191function deleteInOfflineTree(path: string): void {
192 if (path === "") return;
193 const basePath = path.split("/").slice(0, -1).join("/");
194 const fileName = path.split("/").toReversed()[0];
195
196 const node = getDirInTree(basePath, false, true);
197 if (!node) return;
198
199 node.files.delete(fileName);
200 if (node.files.size === 0) {
201 db.covers.delete(basePath); //remove stored cover for this dir if there are not files left
202 deleteInOfflineTree(basePath);
203 }
204}
205
206export const clearDirectoryCache = async (path: string) => {
207 const node = getDirInTree(path, false, false);
208 if (!node) return;
209
210 // Save existing children before clearing
211 const preservedChildren = new Map(node.files);
212
213 node.files.clear();
214 node.status = "Unknown";
215
216 // restore any preserved subtrees where the parent directory still exist
217 const newListing = await listFiles(path, false, false);
218 for (const [name, child] of preservedChildren.entries()) {
219 if ("files" in child && newListing.has(name)) {
220 node.files.set(name, child);
221 }
222 }
223};
224async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
225 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`, { credentials: "same-origin" }).catch(
226 () => {},
227 );
228 if (handleUnauthorized(response)) return;
229 if (response?.status === 404) {
230 toast(`Directory not found: ${dir}`, "error");
231 return;
232 }
233 if (!response?.ok) {
234 toast(
235 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
236 "error",
237 );
238 return;
239 }
240
241 return parseMap(await response.text()) as FileListingWithStatus;
242}
243
244function listOfflineFiles(dir: string): FileListing {
245 const node = getDirInTree(dir, false, true);
246 if (!node?.files) return new Map();
247 return node.files;
248}
249
250//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
251function fullyKnown(dir: DirectoryWithStatus) {
252 if (dir.status === "Unknown") {
253 return false;
254 }
255 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
256 return false;
257 }
258 return true;
259}
260
261//non-recursive listing should not include nested children
262function pruneChildren(listing: FileListing) {
263 const pruned = new Map();
264 for (const [name, item] of listing.entries()) {
265 if ("metadata" in item) pruned.set(name, item);
266 else pruned.set(name, { files: new Map() });
267 }
268 return pruned;
269}
270
271//used for combining the online and offline results, to be able to show files that only exist locally
272function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
273 const result = new Map(listing1);
274
275 for (const [key, value2] of listing2.entries()) {
276 const value1 = result.get(key);
277
278 if (value1 && "files" in value1 && "files" in value2) {
279 result.set(key, { files: combineFileListings(value1.files, value2.files) });
280 } else {
281 result.set(key, value2);
282 }
283 }
284
285 return result;
286}
287
288// File listing logic with hybrid online/offline support
289// Combines:
290// 1. Direct IndexedDB lookups for offline mode
291// 2. Cached online listings
292// 3. Merged online+offline results
293export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
294 if (offline) {
295 const files = listOfflineFiles(dir);
296 return recursive ? files : pruneChildren(files);
297 }
298 const cached = getDirInTree(dir, false, false);
299 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
300 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
301 return recursive ? combined : pruneChildren(combined);
302 }
303 const files = await listOnlineFiles(dir, recursive);
304 if (!files) return new Map();
305 updateFileTree(dir, { files, status: "Scanned" }, false);
306 const combined = combineFileListings(files, listOfflineFiles(dir));
307 return recursive ? combined : pruneChildren(combined);
308}
309
310const coverCache = new Map<string, { data: Blob | null }>();
311
312//get cover art object-url for a file or directory with cache-first strategy
313export async function getCover(
314 dir: string,
315 options: AppOptions,
316 saveToDb: boolean,
317 offline: boolean,
318 forceRefresh = false,
319) {
320 try {
321 const cached = coverCache.get(dir);
322 if (!forceRefresh && cached !== undefined) {
323 if (saveToDb) db.covers.add({ path: dir, data: cached.data }).catch(() => {});
324 return cached.data === null ? undefined : URL.createObjectURL(cached.data);
325 }
326
327 if (offline) return undefined;
328 const response = await fetch(`/cover/${encodePath(dir)}${options.transcodeCovers ? "?transcode=true" : ""}`, {
329 method: "GET",
330 credentials: "same-origin",
331 }).catch(() => {});
332 if (handleUnauthorized(response)) return undefined;
333 if (!response || response.status === 404) {
334 coverCache.set(dir, { data: null });
335 if (saveToDb) db.covers.add({ path: dir, data: null }).catch(() => {});
336 //only update a row that already exists, otherwise merely browsing a directory would
337 //persist its cover - note the await, without it the promise is always truthy
338 else if (forceRefresh && (await db.covers.get(dir)) !== undefined)
339 db.covers.put({ path: dir, data: null }).catch(() => {});
340 return undefined;
341 }
342 const blob = await response.blob();
343 coverCache.set(dir, { data: blob });
344 if (saveToDb) db.covers.add({ path: dir, data: blob }).catch(() => {});
345 //see above: the await is what makes this an "already stored" check
346 else if (forceRefresh && (await db.covers.get(dir)) !== undefined)
347 db.covers.put({ path: dir, data: blob }).catch(() => {});
348 return URL.createObjectURL(blob);
349 } catch {
350 return undefined;
351 }
352}
353
354//returns an object url to the blob if synced, otherwise the server url
355//with forceUrl it always resolves, without it returns undefined for an uncached file in offline mode
356export async function getSrc(
357 path: string,
358 metadata: Metadata,
359 options: AppOptions,
360 offline: boolean,
361 seekTo: number,
362 forceUrl: true,
363): Promise<{ src: string; isVideo?: boolean }>;
364export async function getSrc(
365 path: string,
366 metadata: Metadata,
367 options: AppOptions,
368 offline: boolean,
369 seekTo: number,
370): Promise<{ src: string; isVideo?: boolean } | undefined>;
371export async function getSrc(
372 path: string,
373 metadata: Metadata,
374 options: AppOptions,
375 offline: boolean,
376 seekTo: number,
377 forceUrl?: boolean,
378) {
379 if (!forceUrl) {
380 const dbResult = await db.files.get(path);
381 if (dbResult) return { src: URL.createObjectURL(dbResult.data), isVideo: undefined };
382
383 if (offline) {
384 toast("This file is not cached, leave offline mode to play it", "error");
385 return undefined;
386 }
387 }
388 if (options.enableTranscoding) {
389 const isVideo = metadata.videoCodec !== undefined;
390 const settings = isVideo ? options.videoSettings : options.audioSettings;
391 const disableChunkedTranscoding = settings.streamingMode === StreamingMode.buffered;
392 const audioBitrate = audioCodecHasBitrate(settings.audioCodec) ? settings.audioBitrate : undefined;
393
394 return {
395 src:
396 `${joinPath("transcode", encodePath(path))}` +
397 `?disableChunkedTranscoding=${disableChunkedTranscoding}` +
398 `&container=${settings.container}` +
399 `${isVideo ? `&videoCodec=${options.videoSettings.videoCodec}` : ""}` +
400 `${isVideo ? `&videoBitrate=${options.videoSettings.videoBitrate}` : ""}` +
401 `${isVideo ? `&videoEncodingSetting=${options.videoSettings.videoEncodingSetting}` : ""}` +
402 `&audioCodec=${settings.audioCodec}` +
403 `${audioBitrate === undefined ? "" : `&audioBitrate=${audioBitrate}`}` +
404 `${seekTo === 0 ? "" : `&seekTo=${seekTo}`}` +
405 `&languages=${options.languages}`,
406 isVideo: isVideo,
407 };
408 }
409 return { src: joinPath("download", encodePath(path)), isVideo: undefined };
410}
411
412export async function loadPlaylists() {
413 return await db.playlists.toArray();
414}
415
416//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
417export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
418 playlist.items = playlist.items.map((item) => {
419 //some less space in the db by omitting ids
420 return { path: item.path, metadata: unproxy(item.metadata) };
421 });
422
423 if (overwrite) {
424 await db.playlists.put(playlist);
425 } else {
426 await db.playlists.add(playlist);
427 }
428}
429
430export async function deletePlaylist(name: string) {
431 await db.playlists.delete(name);
432}
433
434function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
435 const paths: string[] = [];
436
437 for (const [name, item] of listing.entries()) {
438 const currentPath = joinPath(basePath, name);
439
440 if ("metadata" in item) {
441 paths.push(currentPath);
442 } else if ("files" in item) {
443 paths.push(...collectMediaFilePaths(item.files, currentPath));
444 }
445 }
446
447 return paths;
448}
449
450//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
451export async function deleteFile(
452 fullPath: string,
453 item: MediaFile | Directory,
454 fetchFiles: ResourceActions<FlatFileListing>,
455) {
456 try {
457 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
458 startProgress(fullPath, { amount: 1 });
459
460 let toDelete: string[];
461 if ("files" in item) {
462 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
463 } else {
464 toDelete = [fullPath];
465 }
466
467 //covers are keyed by file path (embedded art) and by directory path (directory art), so deleting a
468 //directory also has to drop its own cover row plus every nested one, or they stay orphaned forever
469 const coversToDelete = new Set(toDelete);
470 if ("files" in item) {
471 coversToDelete.add(fullPath);
472 for (const filePath of toDelete) {
473 let dir = filePath;
474 while (true) {
475 const lastSlash = dir.lastIndexOf("/");
476 if (lastSlash === -1) break;
477 dir = dir.substring(0, lastSlash);
478 //stop at the deleted root, parents of it may still hold other synced files
479 if (dir.length <= fullPath.length) break;
480 coversToDelete.add(dir);
481 }
482 }
483 }
484
485 await db.files.bulkDelete(toDelete); //delete file from database
486 await db.covers.bulkDelete([...coversToDelete]); //delete embedded/directory covers from database
487 //also drop them from the session cache, otherwise a later getCover(saveToDb) would re-add them
488 for (const coverPath of coversToDelete) coverCache.delete(coverPath);
489 deleteInOfflineTree(fullPath); //delete node from tree
490 batch(() => {
491 for (const path of toDelete) {
492 setDownloadStatusMap(path, undefined); // set all children to unsynced
493 }
494 });
495 fetchFiles.refetch();
496 } finally {
497 stopProgress(fullPath, undefined);
498 }
499}
500
501const downloadSemaphore = new AsyncSemaphore(1);
502
503//Download file to indexedDB and updates the offline file tree
504//concurrency control via the downloadSemaphore
505async function downloadFile(
506 path: string,
507 options: AppOptions,
508 item: MediaFile,
509 fetchFiles: ResourceActions<FlatFileListing>,
510 redownload: boolean,
511): Promise<void> {
512 const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
513
514 // file already downloaded or download already in progress
515 const initialProgress = downloadStatusMap[path];
516 if (
517 (!redownload && initialProgress === "Synced") ||
518 //an in-progress download carries a numeric byte count, without this check clicking a
519 //downloading item would start a second fetch racing the first one on db.files.put
520 (typeof initialProgress === "object" && "status" in initialProgress && typeof initialProgress.status === "number")
521 )
522 return;
523
524 const controller = new AbortController();
525 const abortDownload = () => {
526 controller.abort();
527 stopProgress(
528 path,
529 // if it was already synced, restore it
530 initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
531 );
532 };
533 startProgress(path, { status: 0, abort: abortDownload });
534
535 try {
536 //options might change at any point, update the semaphore to the current value
537 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
538
539 //make sure to only allow the set amount of parallel downloads. passing the signal makes an abort
540 //while still queued reject here instead of entering the section below, so the release() in its
541 //finally can't hand out a permit that was never acquired (which would leak a concurrency slot)
542 await downloadSemaphore.acquire(controller.signal);
543 try {
544 const res = await fetch((await getSrc(path, item.metadata, options, false, 0, true)).src, {
545 signal: controller.signal,
546 credentials: "same-origin",
547 });
548
549 handleUnauthorized(res);
550 if (!res.ok || !res.body) throw res.statusText;
551 //read the body ourselves to track progress, and assemble the blob from the same chunks -
552 //res.clone().blob() would buffer the whole body a second time and reject unobserved on a mid-body error
553 const reader = res.body.getReader();
554 //a fetch body never yields SharedArrayBuffer-backed views, so narrowing to ArrayBuffer is safe
555 const chunks: Uint8Array<ArrayBuffer>[] = [];
556 let received = 0;
557
558 let lastUpdate = 0;
559 while (true) {
560 const { done, value } = await reader.read();
561 if (done) break;
562 chunks.push(value as Uint8Array<ArrayBuffer>);
563 received += value.length;
564
565 //debouncing logic to not update UI too fast
566 const now = Date.now();
567 if (now - lastUpdate < 250) continue;
568 lastUpdate = now;
569 setDownloadStatusMap(path, { status: received, abort: abortDownload });
570 }
571 const blob = new Blob(chunks, { type: res.headers.get("content-type") ?? undefined });
572
573 await db.files.put({
574 path,
575 data: blob,
576 metadata: unproxy(item.metadata),
577 });
578
579 updateFileTree(path, item, true);
580 stopProgress(path, "Synced");
581 fetchFiles.refetch();
582
583 //iterate through all parent directories and download their covers to database
584 const parts = path.split("/");
585 while (true) {
586 getCover(parts.join("/"), options, true, false).then((cover) => URL.revokeObjectURL(cover || ""));
587 parts.pop();
588 if (parts.length === 0) break;
589 }
590 } finally {
591 downloadSemaphore.release();
592 }
593 } catch (e) {
594 if (e instanceof Error && e.name === "AbortError") return;
595 stopProgress(
596 path,
597 // if it was already synced, restore it
598 initialProgress === "Synced"
599 ? "Synced"
600 : {
601 status: "Failed",
602 reason: e ? e.toString() : "Unknown error",
603 retry,
604 },
605 );
606 }
607}
608
609export async function download(
610 fullPath: string,
611 item: MediaFile | Directory,
612 options: AppOptions,
613 fetchFiles: ResourceActions<FlatFileListing>,
614 redownload = false,
615) {
616 if ("metadata" in item) {
617 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
618 } else {
619 updateRecursiveStatus(fullPath, 1, true);
620 try {
621 //start downloads for all children to allow them to run in parallel, then wait for the results
622 const promises: Promise<unknown>[] = [];
623 const listing = await listFiles(fullPath, false, false);
624 batch(() => {
625 for (const [childName, childItem] of listing) {
626 promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload));
627 }
628 });
629 await Promise.allSettled(promises);
630 } finally {
631 updateRecursiveStatus(fullPath, -1, true);
632 }
633 }
634}
635
636export const storedFilesLive = from(
637 liveQuery(async () => await db.files.toCollection().keys()),
638 [],
639);
640