pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import { createSignal, type ResourceActions, type Signal } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14import type { FlatFileListing } from "./App";
15
16interface Progress {
17 bytes: number;
18}
19
20interface LoadingItems {
21 amount: number;
22}
23
24export type DownloadStatus = Progress | LoadingItems | "Synced" | undefined;
25//used to display download status for directories/files in the filebrowser view
26// export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
27export interface StatusWithChildren {
28 status: Signal<DownloadStatus>;
29 children?: RecursiveStatus;
30}
31
32type RecursiveStatus = Map<string, StatusWithChildren>;
33const downloadStatusMap: RecursiveStatus = new Map<string, StatusWithChildren>();
34
35function updateProgressAmount(statusSignal: Signal<DownloadStatus>, loadChange: number) {
36 const [getStatus, setStatus] = statusSignal;
37 let status = getStatus();
38 if (typeof status !== "object" || !("amount" in status)) {
39 status = { amount: 0 };
40 setStatus(status);
41 }
42 const newAmount = status.amount + loadChange;
43 if (newAmount > 0) setStatus({ amount: newAmount });
44 else setStatus(undefined);
45}
46
47export function getDownloadStatus(path: string, loadChange?: number) {
48 const parts = path.split("/");
49 let current = downloadStatusMap;
50 //iterate through the basepath
51 for (const part of parts.slice(0, -1)) {
52 let next = current.get(part);
53 if (!next) {
54 next = { status: createSignal<DownloadStatus>(undefined) };
55 current.set(part, next);
56 }
57 if (!next.children) {
58 //might be a directory whose children have not been referenced before, in which case "children" would be undefined
59 next.children = new Map();
60 }
61 if (loadChange) updateProgressAmount(next.status, loadChange);
62
63 current = next.children as RecursiveStatus;
64 }
65
66 const fileName = parts.toReversed()[0];
67 let ret = current.get(fileName);
68 if (!ret) {
69 ret = { status: createSignal<DownloadStatus>(undefined) };
70 current.set(fileName, ret);
71 }
72 return ret.status;
73}
74
75//increase/decrease load amount for all parents and return requested signal
76//needed to set parent directories to a loading state when any child is being downloaded/deleted
77function startProgress(path: string) {
78 return getDownloadStatus(path, 1);
79}
80function stopProgress(path: string) {
81 return getDownloadStatus(path, -1);
82}
83
84//initialize filetrees and download statuses from indexedDB
85export async function loadDataFromDB() {
86 try {
87 const db = await openDB();
88 const files = await db.listFiles(FileStore);
89 for (const file of files) {
90 getDownloadStatus(file.path)[1]("Synced");
91 updateFileTree(file.path, { metadata: file.metadata }, true);
92 }
93 } catch (e) {
94 toast("Loading offline files failed", "error", e);
95 console.error(e);
96 }
97}
98
99//contains all locally synced files
100const offlineTreeRoot: Directory = {
101 files: new Map(),
102};
103//used for caching online file listings
104const onlineTreeRoot: DirectoryWithStatus = {
105 files: new Map(),
106 status: "Unknown",
107};
108
109//merges "Scanned" directories from "other" to "target"
110function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
111 if (other.status === "Unknown") return;
112 target.status = other.status;
113 for (const [fileName, file] of other.files.entries()) {
114 const targetFile = target.files.get(fileName);
115 if (targetFile && "files" in targetFile && "files" in file) {
116 mergeTrees(targetFile, file);
117 } else {
118 target.files.set(fileName, file);
119 }
120 }
121}
122
123//TODO: improve typing to avoid "as"
124//overloads for typecheking
125//offline tree only stores files, directories are created ad-hoc
126function updateFileTree(path: string, file: MediaFile, offline: true): void;
127//online tree only stores cached directory listings
128function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
129// Updates file tree structure with new data (offline or online)
130function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
131 if (offline && path === "") throw new Error("Trying to update root in offline mode");
132 const basePath = path.split("/").slice(0, -1).join("/");
133 const fileName = path.split("/").toReversed()[0];
134 const node = getDirInTree(basePath, true, offline);
135
136 if (offline) {
137 node.files.set(fileName, file);
138 } else {
139 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
140 else {
141 let target = node.files.get(fileName) as DirectoryWithStatus;
142 if (!target) target = { files: new Map(), status: "Unknown" };
143 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
144 node.files.set(fileName, target);
145 }
146 }
147}
148
149//overloads for typecheking
150function getDirInTree(path: string, create: true, offline: true): Directory;
151function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
152function getDirInTree(path: string, create: false, offline: true): Directory | null;
153function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
154function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
155function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
156// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
157function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
158 let current = offline ? offlineTreeRoot : onlineTreeRoot;
159 if (path === "") return current;
160 const parts = path.split("/");
161
162 for (const part of parts) {
163 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
164 if (!next || "metadata" in next) {
165 if (!create) return null;
166 //if mediafile already exists, we overwrite it with a directory anyway
167 if (offline) next = { files: new Map() };
168 else next = { files: new Map(), status: "Unknown" };
169 current.files.set(part, next);
170 }
171 current = next;
172 }
173 return current;
174}
175
176function deleteInOfflineTree(path: string): void {
177 if (path === "") return;
178 const basePath = path.split("/").slice(0, -1).join("/");
179 const fileName = path.split("/").toReversed()[0];
180
181 const node = getDirInTree(basePath, false, true);
182 if (!node) return;
183
184 node.files.delete(fileName);
185 if (node.files.size === 0) deleteInOfflineTree(basePath);
186}
187
188export const clearDirectoryCache = (path: string) => {
189 const node = getDirInTree(path, false, false);
190 if (!node) return;
191 node.files.clear();
192 node.status = "Unknown";
193};
194
195async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
196 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
197 if (response?.status === 404) {
198 toast(`Directory not found: ${dir}`, "error");
199 return;
200 }
201 if (!response || !response.ok) {
202 toast(
203 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
204 "error",
205 );
206 return;
207 }
208
209 return parseMap(await response.text()) as FileListingWithStatus;
210}
211
212function listOfflineFiles(dir: string): FileListing {
213 const node = getDirInTree(dir, false, true);
214 if (!node || !node.files) return new Map();
215 return node.files;
216}
217
218//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
219function fullyKnown(dir: DirectoryWithStatus) {
220 if (dir.status === "Unknown") {
221 return false;
222 }
223 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
224 return false;
225 }
226 return true;
227}
228
229//non-recursive listing should not include nested children
230function pruneChildren(listing: FileListing) {
231 const pruned = new Map();
232 for (const [name, item] of listing.entries()) {
233 if ("metadata" in item) pruned.set(name, item);
234 else pruned.set(name, { files: new Map() });
235 }
236 return pruned;
237}
238
239//used for combining the online and offline results, to be able to show files that only exist locally
240function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
241 const result = new Map(listing1);
242
243 for (const [key, value2] of listing2.entries()) {
244 const value1 = result.get(key);
245
246 if (value1 && "files" in value1 && "files" in value2) {
247 result.set(key, { files: combineFileListings(value1.files, value2.files) });
248 } else {
249 result.set(key, value2);
250 }
251 }
252
253 return result;
254}
255
256// File listing logic with hybrid online/offline support
257// Combines:
258// 1. Direct IndexedDB lookups for offline mode
259// 2. Cached online listings
260// 3. Merged online+offline results
261export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
262 if (offline) {
263 const files = listOfflineFiles(dir);
264 return recursive ? files : pruneChildren(files);
265 }
266 const cached = getDirInTree(dir, false, false);
267 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
268 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
269 return recursive ? combined : pruneChildren(combined);
270 }
271 const files = await listOnlineFiles(dir, recursive);
272 if (!files) return new Map();
273 updateFileTree(dir, { files, status: "Scanned" }, false);
274 const combined = combineFileListings(files, listOfflineFiles(dir));
275 return recursive ? combined : pruneChildren(combined);
276}
277
278//get cover art for a directory with cache-first strategy
279export async function getCover(dir: string, offline: boolean) {
280 const db = await openDB();
281 const dbResult = await db.get(CoverStore, dir);
282 if (dbResult !== undefined) {
283 const cached = dbResult as StoredCover;
284 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
285 }
286
287 if (offline) return undefined;
288
289 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
290 if (!response || response.status === 404) {
291 db.put(CoverStore, { path: dir, data: null });
292 return undefined;
293 }
294 const blob = await response.blob();
295 db.put(CoverStore, { path: dir, data: blob });
296 return URL.createObjectURL(blob);
297}
298
299export const clearCoverCache = async (path: string) => {
300 const db = await openDB();
301 return db.delete(CoverStore, path);
302};
303
304//returns an object url to the blob if synced, otherwise the server url
305export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
306export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
307 if (!forceUrl) {
308 const db = await openDB();
309 const dbResult = (await db.get(FileStore, path)) as StoredFile;
310 if (dbResult) return URL.createObjectURL(dbResult.data);
311
312 if (offline) {
313 toast("This file is not cached, leave offline mode to play it", "error");
314 return undefined;
315 }
316 }
317 if (options.enableTranscoding) {
318 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
319 }
320 return joinPath("/download", encodePath(path));
321}
322
323export async function loadPlaylists() {
324 const db = await openDB();
325 return await db.listPlaylists(PlaylistStore);
326}
327
328//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
329export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
330 const db = await openDB();
331 playlist.items = playlist.items.map((item) => {
332 //some less space in the db by omitting ids
333 const { id, ...itemWithoutId } = item as PlaylistItem;
334 return itemWithoutId;
335 });
336 if (overwrite) await db.put(PlaylistStore, playlist);
337 else await db.add(PlaylistStore, playlist);
338}
339
340export async function deletePlaylist(name: string) {
341 const db = await openDB();
342 await db.delete(PlaylistStore, name);
343}
344
345function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
346 const paths: string[] = [];
347
348 for (const [name, item] of listing.entries()) {
349 const currentPath = joinPath(basePath, name);
350
351 if ("metadata" in item) {
352 paths.push(currentPath);
353 } else if ("files" in item) {
354 paths.push(...collectMediaFilePaths(item.files, currentPath));
355 }
356 }
357
358 return paths;
359}
360
361//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
362export async function deleteFile(
363 fileName: string,
364 item: MediaFile | Directory,
365 basePath: string,
366 fetchFiles: ResourceActions<FlatFileListing>,
367) {
368 const fullPath = joinPath(basePath, fileName);
369 const [getStatus] = getDownloadStatus(fullPath);
370 try {
371 if ("metadata" in item && getStatus() !== "Synced") return;
372 startProgress(fullPath)[1]({ amount: 1 });
373
374 let toDelete: string[];
375 if ("files" in item) {
376 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
377 } else {
378 toDelete = [fullPath];
379 }
380
381 const db = await openDB();
382 await db.deleteAll(FileStore, toDelete); //delete from database
383 deleteInOfflineTree(fullPath); //delete node from tree
384 for (const path of toDelete) {
385 getDownloadStatus(path)[1](undefined); // set all children to unsynced
386 }
387 fetchFiles.refetch();
388 } finally {
389 stopProgress(fullPath)[1](undefined);
390 }
391}
392
393const downloadSemaphore = new AsyncSemaphore(1);
394
395//Download file to indexedDB and updates the offline file tree
396//concurrency control via the downloadSemaphore
397async function downloadFile(
398 path: string,
399 options: AppOptions,
400 item: MediaFile,
401 fetchFiles: ResourceActions<FlatFileListing>,
402): Promise<void> {
403 const [getStatus, setStatus] = getDownloadStatus(path);
404 const initialProgress = getStatus();
405 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
406
407 try {
408 startProgress(path)[1]({ bytes: 0 });
409 const db = await openDB();
410 let mimeType: string | undefined;
411
412 //options might change at any point, update the semaphore to the current value
413 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
414 let blob: Blob;
415 try {
416 //make sure to only allow the set amount of parallel downloads
417 await downloadSemaphore.acquire();
418 const res = await fetch(await getSrc(path, options, false, true));
419
420 //let the browser do the actual download
421 const blobPromise = res.clone().blob();
422 if (!res.ok || !res.body) throw res.statusText;
423 //we just iterate over the chunks to get the current progress
424 const reader = res.body.getReader();
425 mimeType = res.headers.get("content-type") ?? undefined;
426 let received = 0;
427
428 let lastUpdate = 0;
429 while (true) {
430 const { done, value } = await reader.read();
431 if (done) break;
432 received += value.length;
433
434 //debouncing logic to not update UI too fast
435 const now = Date.now();
436 if (now - lastUpdate < 250) continue;
437 lastUpdate = now;
438 setStatus({ bytes: received });
439 }
440 blob = await blobPromise;
441 } finally {
442 downloadSemaphore.release();
443 }
444 await db.put(FileStore, {
445 path,
446 data: blob,
447 metadata: { ...item.metadata }, //spread needed, because item might be a proxy object from the filebrowser, which can't be stored
448 } as StoredFile);
449 updateFileTree(path, item, true);
450 stopProgress(path)[1]("Synced");
451 fetchFiles.refetch();
452 } catch (e) {
453 toast(`Downloading ${path} failed`, "error", e);
454 //restore original status
455 stopProgress(path)[1](initialProgress === "Synced" ? "Synced" : undefined);
456 }
457}
458
459export async function download(
460 fileName: string,
461 item: MediaFile | Directory,
462 basePath: string,
463 options: AppOptions,
464 fetchFiles: ResourceActions<FlatFileListing>,
465) {
466 const fullPath = joinPath(basePath, fileName);
467 if ("metadata" in item) {
468 await downloadFile(fullPath, options, item, fetchFiles);
469 } else {
470 updateProgressAmount(startProgress(fullPath), 1);
471 try {
472 await getCover(fullPath, false);
473 //start downloads for all children to allow them to run in parallel, then wait for the results
474 const promises: Promise<unknown>[] = [];
475 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
476 promises.push(download(childName, childItem, fullPath, options, fetchFiles));
477 }
478 await Promise.allSettled(promises);
479 } finally {
480 updateProgressAmount(stopProgress(fullPath), -1);
481 }
482 }
483}
484
485export async function getStoredFiles() {
486 const db = await openDB();
487 return await db.getKeys(FileStore);
488}
489
490interface StoredFile {
491 path: string;
492 data: Blob;
493 metadata: {
494 duration: number;
495 };
496}
497
498interface StoredCover {
499 path: string;
500 data: Blob | null;
501}
502
503const FileStore = "fileblobs";
504const CoverStore = "covers";
505const PlaylistStore = "playlists";
506
507async function openDB() {
508 return new Promise<DatabaseWrapper>((resolve, reject) => {
509 const req = indexedDB.open("music-files");
510 req.onerror = () => reject(req.error);
511 req.onupgradeneeded = () => {
512 const db = req.result;
513 if (!db.objectStoreNames.contains(FileStore))
514 db.createObjectStore(FileStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
515 if (!db.objectStoreNames.contains(CoverStore))
516 db.createObjectStore(CoverStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
517 if (!db.objectStoreNames.contains(PlaylistStore))
518 db.createObjectStore(PlaylistStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
519 };
520 req.onsuccess = () => resolve(wrapDB(req.result));
521 });
522}
523
524interface DatabaseWrapper {
525 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
526 put(store: typeof FileStore, value: StoredFile): Promise<void>;
527 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
528 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
529 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
530 delete(store: typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
531 deleteAll(store: typeof FileStore, keys: string[]): Promise<void>;
532 clear(store: string): Promise<void>;
533 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
534 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
535 getKeys(store: typeof FileStore): Promise<string[]>;
536}
537
538function wrapDB(db: IDBDatabase): DatabaseWrapper {
539 return {
540 get: (store: string, key: string) =>
541 new Promise((res, rej) => {
542 const tx = db.transaction(store, "readonly");
543 const req = tx.objectStore(store).get(key);
544 req.onsuccess = () => res(req.result);
545 req.onerror = () => rej(req.error);
546 }),
547 put: (store: string, value: unknown) =>
548 new Promise<void>((res, rej) => {
549 const tx = db.transaction(store, "readwrite");
550 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
551 req.onsuccess = () => res();
552 req.onerror = () => rej(req.error);
553 }),
554 add: (store: string, value: unknown) =>
555 new Promise<void>((res, rej) => {
556 const tx = db.transaction(store, "readwrite");
557 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
558 req.onsuccess = () => res();
559 req.onerror = () => rej(req.error);
560 }),
561 delete: (store: string, key: string) =>
562 new Promise<void>((res, rej) => {
563 const tx = db.transaction(store, "readwrite");
564 const req = tx.objectStore(store).delete(key);
565 req.onsuccess = () => res();
566 req.onerror = () => rej(req.error);
567 }),
568 deleteAll: (store: typeof FileStore, keys: string[]) =>
569 new Promise<void>((res, rej) => {
570 const tx = db.transaction(store, "readwrite");
571 const objectStore = tx.objectStore(store);
572 for (const key of keys) {
573 objectStore.delete(key);
574 }
575 tx.oncomplete = () => res();
576 tx.onerror = () => rej(tx.error);
577 }),
578 clear: (store: string) =>
579 new Promise<void>((res, rej) => {
580 const tx = db.transaction(store, "readwrite");
581 const req = tx.objectStore(store).clear();
582 req.onsuccess = () => res();
583 req.onerror = () => rej(req.error);
584 }),
585 listFiles: (store: typeof FileStore) =>
586 new Promise((res, rej) => {
587 const tx = db.transaction(store, "readonly");
588 const req = tx.objectStore(store).getAll();
589 req.onsuccess = () => res(req.result);
590 req.onerror = () => rej(req.error);
591 }),
592 listPlaylists: (store: typeof PlaylistStore) =>
593 new Promise((res, rej) => {
594 const tx = db.transaction(store, "readonly");
595 const req = tx.objectStore(store).getAll();
596 req.onsuccess = () => res(req.result);
597 req.onerror = () => rej(req.error);
598 }),
599 getKeys: (store: typeof FileStore) =>
600 new Promise((res, rej) => {
601 const tx = db.transaction(store, "readonly");
602 const req = tx.objectStore(store).getAllKeys();
603 req.onsuccess = () => res(req.result as string[]);
604 req.onerror = () => rej(req.error);
605 }),
606 };
607}
608