pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist } from "./types";
10import { joinPath, toast } from "./utils";
11import type { ResourceActions } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14import type { FlatFileListing } from "./App";
15import { createStore } from "solid-js/store";
16
17export type Progress =
18 | { status: number; abort: () => void }
19 | { status: "Aborted"; retry: () => void }
20 | { status: "Failed"; retry: () => void; reason: string };
21
22interface LoadingChildren {
23 amount: number;
24}
25
26export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
27export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
28
29//creates entry if it doesn't exist, so reactivity works
30export function getDownloadStatus(path: string): DownloadStatus | undefined {
31 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
32 return downloadStatusMap[path];
33}
34
35function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
36 function updateProgressAmount(path: string) {
37 const currentStatus = downloadStatusMap[path];
38 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
39 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
40 return;
41 }
42
43 const newAmount = currentStatus.amount + loadChange;
44 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
45 }
46
47 if (includeCurrent) updateProgressAmount(path);
48 // Update all parent directories
49 let currentPath = path;
50 while (true) {
51 const lastSlash = currentPath.lastIndexOf("/");
52 if (lastSlash === -1) break;
53 currentPath = currentPath.substring(0, lastSlash);
54 updateProgressAmount(currentPath);
55 }
56}
57
58function startProgress(path: string, status: DownloadStatus) {
59 updateRecursiveStatus(path, 1, false);
60 setDownloadStatusMap(path, status);
61}
62
63function stopProgress(path: string, status: DownloadStatus) {
64 updateRecursiveStatus(path, -1, false);
65 setDownloadStatusMap(path, status);
66}
67
68//initialize filetrees and download statuses from indexedDB
69export async function loadDataFromDB() {
70 try {
71 const db = await openDB();
72 const files = await db.listFiles(FileStore);
73 for (const file of files) {
74 setDownloadStatusMap(file.path, "Synced");
75 updateFileTree(file.path, { metadata: file.metadata }, true);
76 }
77 } catch (e) {
78 toast("Loading offline files failed", "error", e);
79 console.error(e);
80 }
81}
82
83//contains all locally synced files
84const offlineTreeRoot: Directory = {
85 files: new Map(),
86};
87//used for caching online file listings
88const onlineTreeRoot: DirectoryWithStatus = {
89 files: new Map(),
90 status: "Unknown",
91};
92
93//merges "Scanned" directories from "other" to "target"
94function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
95 if (other.status === "Unknown") return;
96 target.status = other.status;
97 for (const [fileName, file] of other.files.entries()) {
98 const targetFile = target.files.get(fileName);
99 if (targetFile && "files" in targetFile && "files" in file) {
100 mergeTrees(targetFile, file);
101 } else {
102 target.files.set(fileName, file);
103 }
104 }
105}
106
107//TODO: improve typing to avoid "as"
108//overloads for typecheking
109//offline tree only stores files, directories are created ad-hoc
110function updateFileTree(path: string, file: MediaFile, offline: true): void;
111//online tree only stores cached directory listings
112function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
113// Updates file tree structure with new data (offline or online)
114function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
115 if (offline && path === "") throw new Error("Trying to update root in offline mode");
116 const basePath = path.split("/").slice(0, -1).join("/");
117 const fileName = path.split("/").toReversed()[0];
118 const node = getDirInTree(basePath, true, offline);
119
120 if (offline) {
121 node.files.set(fileName, file);
122 } else {
123 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
124 else {
125 let target = node.files.get(fileName) as DirectoryWithStatus;
126 if (!target) target = { files: new Map(), status: "Unknown" };
127 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
128 node.files.set(fileName, target);
129 }
130 }
131}
132
133//overloads for typecheking
134function getDirInTree(path: string, create: true, offline: true): Directory;
135function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
136function getDirInTree(path: string, create: false, offline: true): Directory | null;
137function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
138function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
139function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
140// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
141function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
142 let current = offline ? offlineTreeRoot : onlineTreeRoot;
143 if (path === "") return current;
144 const parts = path.split("/");
145
146 for (const part of parts) {
147 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
148 if (!next || "metadata" in next) {
149 if (!create) return null;
150 //if mediafile already exists, we overwrite it with a directory anyway
151 if (offline) next = { files: new Map() };
152 else next = { files: new Map(), status: "Unknown" };
153 current.files.set(part, next);
154 }
155 current = next;
156 }
157 return current;
158}
159
160function deleteInOfflineTree(path: string): void {
161 if (path === "") return;
162 const basePath = path.split("/").slice(0, -1).join("/");
163 const fileName = path.split("/").toReversed()[0];
164
165 const node = getDirInTree(basePath, false, true);
166 if (!node) return;
167
168 node.files.delete(fileName);
169 if (node.files.size === 0) deleteInOfflineTree(basePath);
170}
171
172export const clearDirectoryCache = (path: string) => {
173 const node = getDirInTree(path, false, false);
174 if (!node) return;
175 node.files.clear();
176 node.status = "Unknown";
177};
178
179async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
180 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
181 if (response?.status === 404) {
182 toast(`Directory not found: ${dir}`, "error");
183 return;
184 }
185 if (!response || !response.ok) {
186 toast(
187 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
188 "error",
189 );
190 return;
191 }
192
193 return parseMap(await response.text()) as FileListingWithStatus;
194}
195
196function listOfflineFiles(dir: string): FileListing {
197 const node = getDirInTree(dir, false, true);
198 if (!node || !node.files) return new Map();
199 return node.files;
200}
201
202//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
203function fullyKnown(dir: DirectoryWithStatus) {
204 if (dir.status === "Unknown") {
205 return false;
206 }
207 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
208 return false;
209 }
210 return true;
211}
212
213//non-recursive listing should not include nested children
214function pruneChildren(listing: FileListing) {
215 const pruned = new Map();
216 for (const [name, item] of listing.entries()) {
217 if ("metadata" in item) pruned.set(name, item);
218 else pruned.set(name, { files: new Map() });
219 }
220 return pruned;
221}
222
223//used for combining the online and offline results, to be able to show files that only exist locally
224function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
225 const result = new Map(listing1);
226
227 for (const [key, value2] of listing2.entries()) {
228 const value1 = result.get(key);
229
230 if (value1 && "files" in value1 && "files" in value2) {
231 result.set(key, { files: combineFileListings(value1.files, value2.files) });
232 } else {
233 result.set(key, value2);
234 }
235 }
236
237 return result;
238}
239
240// File listing logic with hybrid online/offline support
241// Combines:
242// 1. Direct IndexedDB lookups for offline mode
243// 2. Cached online listings
244// 3. Merged online+offline results
245export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
246 if (offline) {
247 const files = listOfflineFiles(dir);
248 return recursive ? files : pruneChildren(files);
249 }
250 const cached = getDirInTree(dir, false, false);
251 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
252 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
253 return recursive ? combined : pruneChildren(combined);
254 }
255 const files = await listOnlineFiles(dir, recursive);
256 if (!files) return new Map();
257 updateFileTree(dir, { files, status: "Scanned" }, false);
258 const combined = combineFileListings(files, listOfflineFiles(dir));
259 return recursive ? combined : pruneChildren(combined);
260}
261
262//get cover art for a directory with cache-first strategy
263export async function getCover(dir: string, offline: boolean) {
264 const db = await openDB();
265 const dbResult = await db.get(CoverStore, dir);
266 if (dbResult !== undefined) {
267 const cached = dbResult as StoredCover;
268 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
269 }
270
271 if (offline) return undefined;
272
273 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
274 if (!response || response.status === 404) {
275 db.put(CoverStore, { path: dir, data: null });
276 return undefined;
277 }
278 const blob = await response.blob();
279 db.put(CoverStore, { path: dir, data: blob });
280 return URL.createObjectURL(blob);
281}
282
283export const clearCoverCache = async (path: string) => {
284 const db = await openDB();
285 return db.delete(CoverStore, path);
286};
287
288//returns an object url to the blob if synced, otherwise the server url
289export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
290export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
291 if (!forceUrl) {
292 const db = await openDB();
293 const dbResult = (await db.get(FileStore, path)) as StoredFile;
294 if (dbResult) return URL.createObjectURL(dbResult.data);
295
296 if (offline) {
297 toast("This file is not cached, leave offline mode to play it", "error");
298 return undefined;
299 }
300 }
301 if (options.enableTranscoding) {
302 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
303 }
304 return joinPath("/download", encodePath(path));
305}
306
307export async function loadPlaylists() {
308 const db = await openDB();
309 return await db.listPlaylists(PlaylistStore);
310}
311
312//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
313export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
314 const db = await openDB();
315 playlist.items = playlist.items.map((item) => {
316 //some less space in the db by omitting ids
317 //spread item.metadata because it might be a proxy object
318 return { path: item.path, metadata: { ...item.metadata } };
319 });
320 if (overwrite) await db.put(PlaylistStore, playlist);
321 else await db.add(PlaylistStore, playlist);
322}
323
324export async function deletePlaylist(name: string) {
325 const db = await openDB();
326 await db.delete(PlaylistStore, name);
327}
328
329function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
330 const paths: string[] = [];
331
332 for (const [name, item] of listing.entries()) {
333 const currentPath = joinPath(basePath, name);
334
335 if ("metadata" in item) {
336 paths.push(currentPath);
337 } else if ("files" in item) {
338 paths.push(...collectMediaFilePaths(item.files, currentPath));
339 }
340 }
341
342 return paths;
343}
344
345//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
346export async function deleteFile(
347 fileName: string,
348 item: MediaFile | Directory,
349 basePath: string,
350 fetchFiles: ResourceActions<FlatFileListing>,
351) {
352 const fullPath = joinPath(basePath, fileName);
353 try {
354 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
355 startProgress(fullPath, { amount: 1 });
356
357 let toDelete: string[];
358 if ("files" in item) {
359 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
360 } else {
361 toDelete = [fullPath];
362 }
363
364 const db = await openDB();
365 await db.deleteAll(FileStore, toDelete); //delete from database
366 deleteInOfflineTree(fullPath); //delete node from tree
367 for (const path of toDelete) {
368 setDownloadStatusMap(path, undefined); // set all children to unsynced
369 }
370 fetchFiles.refetch();
371 } finally {
372 stopProgress(fullPath, undefined);
373 }
374}
375
376const downloadSemaphore = new AsyncSemaphore(1);
377
378//Download file to indexedDB and updates the offline file tree
379//concurrency control via the downloadSemaphore
380async function downloadFile(
381 path: string,
382 options: AppOptions,
383 item: MediaFile,
384 fetchFiles: ResourceActions<FlatFileListing>,
385 redownload: boolean,
386): Promise<void> {
387 const retry = () => {
388 setDownloadStatusMap(path, undefined);
389 return downloadFile(path, options, item, fetchFiles, redownload);
390 };
391
392 const controller = new AbortController();
393 let abortResolve!: () => void;
394 const abortPromise = new Promise<void>((resolve) => {
395 abortResolve = resolve;
396 }).then(() => {
397 controller.abort();
398 });
399
400 // file already downloaded or download already in progress
401 const initialProgress = downloadStatusMap[path];
402 if (
403 (!redownload && initialProgress === "Synced") ||
404 (typeof initialProgress === "object" && "bytes" in initialProgress)
405 )
406 return;
407 startProgress(path, { status: 0, abort: abortResolve });
408
409 try {
410 const db = await openDB();
411
412 //options might change at any point, update the semaphore to the current value
413 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
414
415 //make sure to only allow the set amount of parallel downloads
416 const semaphorePromise = downloadSemaphore.acquire();
417 await Promise.race([abortPromise, semaphorePromise]);
418 try {
419 const res = await fetch(await getSrc(path, options, false, true), { signal: controller.signal });
420
421 //let the browser do the actual download
422 const blobPromise = res.clone().blob();
423 if (!res.ok || !res.body) throw res.statusText;
424 //we just iterate over the chunks to get the current progress
425 const reader = res.body.getReader();
426 let received = 0;
427
428 let lastUpdate = 0;
429 while (true) {
430 const { done, value } = await reader.read();
431 if (done) break;
432 received += value.length;
433
434 //debouncing logic to not update UI too fast
435 const now = Date.now();
436 if (now - lastUpdate < 250) continue;
437 lastUpdate = now;
438 setDownloadStatusMap(path, { status: received, abort: abortResolve });
439 }
440 const blob = await blobPromise;
441
442 await db.put(FileStore, {
443 path,
444 data: blob,
445 metadata: { ...item.metadata }, //spread needed, because item might be a proxy object from the filebrowser, which can't be stored
446 } as StoredFile);
447 updateFileTree(path, item, true);
448 stopProgress(path, "Synced");
449 fetchFiles.refetch();
450 } finally {
451 downloadSemaphore.release();
452 }
453 } catch (e) {
454 stopProgress(
455 path,
456 // if it was already synced, restore it
457 initialProgress === "Synced"
458 ? "Synced"
459 : //if it was manually aborted, set as aborted
460 e && typeof e === "object" && "name" in e && e.name === "AbortError"
461 ? { status: "Aborted", retry }
462 : //otherwise set as failed and set reason to the error message
463 {
464 status: "Failed",
465 reason: e ? e.toString() : "Unknown error",
466 retry,
467 },
468 );
469 }
470}
471
472export async function download(
473 fileName: string,
474 item: MediaFile | Directory,
475 basePath: string,
476 options: AppOptions,
477 fetchFiles: ResourceActions<FlatFileListing>,
478 redownload = false,
479) {
480 const fullPath = joinPath(basePath, fileName);
481 if ("metadata" in item) {
482 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
483 } else {
484 updateRecursiveStatus(fullPath, 1, true);
485 try {
486 await getCover(fullPath, false);
487 //start downloads for all children to allow them to run in parallel, then wait for the results
488 const promises: Promise<unknown>[] = [];
489 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
490 promises.push(download(childName, childItem, fullPath, options, fetchFiles, redownload));
491 }
492 await Promise.allSettled(promises);
493 } finally {
494 updateRecursiveStatus(fullPath, -1, true);
495 }
496 }
497}
498
499export async function getStoredFiles() {
500 const db = await openDB();
501 return await db.getKeys(FileStore);
502}
503
504interface StoredFile {
505 path: string;
506 data: Blob;
507 metadata: {
508 duration: number;
509 };
510}
511
512interface StoredCover {
513 path: string;
514 data: Blob | null;
515}
516
517const FileStore = "fileblobs";
518const CoverStore = "covers";
519const PlaylistStore = "playlists";
520
521async function openDB() {
522 return new Promise<DatabaseWrapper>((resolve, reject) => {
523 const req = indexedDB.open("music-files");
524 req.onerror = () => reject(req.error);
525 req.onupgradeneeded = () => {
526 const db = req.result;
527 if (!db.objectStoreNames.contains(FileStore))
528 db.createObjectStore(FileStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
529 if (!db.objectStoreNames.contains(CoverStore))
530 db.createObjectStore(CoverStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
531 if (!db.objectStoreNames.contains(PlaylistStore))
532 db.createObjectStore(PlaylistStore, { keyPath: "name" }).createIndex("index", "name", { unique: true });
533 };
534 req.onsuccess = () => resolve(wrapDB(req.result));
535 });
536}
537
538interface DatabaseWrapper {
539 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
540 put(store: typeof FileStore, value: StoredFile): Promise<void>;
541 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
542 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
543 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
544 delete(store: typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
545 deleteAll(store: typeof FileStore, keys: string[]): Promise<void>;
546 clear(store: string): Promise<void>;
547 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
548 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
549 getKeys(store: typeof FileStore): Promise<string[]>;
550}
551
552function wrapDB(db: IDBDatabase): DatabaseWrapper {
553 return {
554 get: (store: string, key: string) =>
555 new Promise((res, rej) => {
556 const tx = db.transaction(store, "readonly");
557 const req = tx.objectStore(store).get(key);
558 req.onsuccess = () => res(req.result);
559 req.onerror = () => rej(req.error);
560 }),
561 put: (store: string, value: unknown) =>
562 new Promise<void>((res, rej) => {
563 const tx = db.transaction(store, "readwrite");
564 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
565 req.onsuccess = () => res();
566 req.onerror = () => rej(req.error);
567 }),
568 add: (store: string, value: unknown) =>
569 new Promise<void>((res, rej) => {
570 const tx = db.transaction(store, "readwrite");
571 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
572 req.onsuccess = () => res();
573 req.onerror = () => rej(req.error);
574 }),
575 delete: (store: string, key: string) =>
576 new Promise<void>((res, rej) => {
577 const tx = db.transaction(store, "readwrite");
578 const req = tx.objectStore(store).delete(key);
579 req.onsuccess = () => res();
580 req.onerror = () => rej(req.error);
581 }),
582 deleteAll: (store: typeof FileStore, keys: string[]) =>
583 new Promise<void>((res, rej) => {
584 const tx = db.transaction(store, "readwrite");
585 const objectStore = tx.objectStore(store);
586 for (const key of keys) {
587 objectStore.delete(key);
588 }
589 tx.oncomplete = () => res();
590 tx.onerror = () => rej(tx.error);
591 }),
592 clear: (store: string) =>
593 new Promise<void>((res, rej) => {
594 const tx = db.transaction(store, "readwrite");
595 const req = tx.objectStore(store).clear();
596 req.onsuccess = () => res();
597 req.onerror = () => rej(req.error);
598 }),
599 listFiles: (store: typeof FileStore) =>
600 new Promise((res, rej) => {
601 const tx = db.transaction(store, "readonly");
602 const req = tx.objectStore(store).getAll();
603 req.onsuccess = () => res(req.result);
604 req.onerror = () => rej(req.error);
605 }),
606 listPlaylists: (store: typeof PlaylistStore) =>
607 new Promise((res, rej) => {
608 const tx = db.transaction(store, "readonly");
609 const req = tx.objectStore(store).getAll();
610 req.onsuccess = () => res(req.result);
611 req.onerror = () => rej(req.error);
612 }),
613 getKeys: (store: typeof FileStore) =>
614 new Promise((res, rej) => {
615 const tx = db.transaction(store, "readonly");
616 const req = tx.objectStore(store).getAllKeys();
617 req.onsuccess = () => res(req.result as string[]);
618 req.onerror = () => rej(req.error);
619 }),
620 };
621}
622