pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist } from "./types";
10import { joinPath, toast } from "./utils";
11import type { ResourceActions } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14import type { FlatFileListing } from "./App";
15import { createStore } from "solid-js/store";
16import Dexie from "dexie";
17
18interface StoredFile {
19 path: string;
20 data: Blob;
21 metadata: {
22 duration: number;
23 };
24}
25
26interface StoredCover {
27 path: string;
28 data: Blob | null;
29}
30
31interface Database extends Dexie {
32 files: Dexie.Table<StoredFile, string>;
33 covers: Dexie.Table<StoredCover, string>;
34 playlists: Dexie.Table<Playlist, string>;
35}
36
37const db = new Dexie("music-files") as Database;
38db.version(1).stores({
39 files: "path",
40 covers: "path",
41 playlists: "name",
42});
43
44export type Progress =
45 | { status: number; abort: () => void }
46 | { status: "Aborted"; retry: () => void }
47 | { status: "Failed"; retry: () => void; reason: string };
48
49interface LoadingChildren {
50 amount: number;
51}
52
53export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
54export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
55
56//creates entry if it doesn't exist, so reactivity works
57export function getDownloadStatus(path: string): DownloadStatus | undefined {
58 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
59 return downloadStatusMap[path];
60}
61
62function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
63 function updateProgressAmount(path: string) {
64 const currentStatus = downloadStatusMap[path];
65 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
66 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
67 return;
68 }
69
70 const newAmount = currentStatus.amount + loadChange;
71 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
72 }
73
74 if (includeCurrent) updateProgressAmount(path);
75 // Update all parent directories
76 let currentPath = path;
77 while (true) {
78 const lastSlash = currentPath.lastIndexOf("/");
79 if (lastSlash === -1) break;
80 currentPath = currentPath.substring(0, lastSlash);
81 updateProgressAmount(currentPath);
82 }
83}
84
85function startProgress(path: string, status: DownloadStatus) {
86 updateRecursiveStatus(path, 1, false);
87 setDownloadStatusMap(path, status);
88}
89
90function stopProgress(path: string, status: DownloadStatus) {
91 updateRecursiveStatus(path, -1, false);
92 setDownloadStatusMap(path, status);
93}
94
95//initialize filetrees and download statuses from indexedDB
96export async function loadDataFromDB() {
97 try {
98 const files = await db.files.toArray();
99 for (const file of files) {
100 setDownloadStatusMap(file.path, "Synced");
101 updateFileTree(file.path, { metadata: file.metadata }, true);
102 }
103 } catch (e) {
104 toast("Loading offline files failed", "error", e);
105 console.error(e);
106 }
107}
108
109//contains all locally synced files
110const offlineTreeRoot: Directory = {
111 files: new Map(),
112};
113//used for caching online file listings
114const onlineTreeRoot: DirectoryWithStatus = {
115 files: new Map(),
116 status: "Unknown",
117};
118
119//merges "Scanned" directories from "other" to "target"
120function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
121 if (other.status === "Unknown") return;
122 target.status = other.status;
123 for (const [fileName, file] of other.files.entries()) {
124 const targetFile = target.files.get(fileName);
125 if (targetFile && "files" in targetFile && "files" in file) {
126 mergeTrees(targetFile, file);
127 } else {
128 target.files.set(fileName, file);
129 }
130 }
131}
132
133//TODO: improve typing to avoid "as"
134//overloads for typecheking
135//offline tree only stores files, directories are created ad-hoc
136function updateFileTree(path: string, file: MediaFile, offline: true): void;
137//online tree only stores cached directory listings
138function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
139// Updates file tree structure with new data (offline or online)
140function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
141 if (offline && path === "") throw new Error("Trying to update root in offline mode");
142 const basePath = path.split("/").slice(0, -1).join("/");
143 const fileName = path.split("/").toReversed()[0];
144 const node = getDirInTree(basePath, true, offline);
145
146 if (offline) {
147 node.files.set(fileName, file);
148 } else {
149 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
150 else {
151 let target = node.files.get(fileName) as DirectoryWithStatus;
152 if (!target) target = { files: new Map(), status: "Unknown" };
153 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
154 node.files.set(fileName, target);
155 }
156 }
157}
158
159//overloads for typechecking
160function getDirInTree(path: string, create: true, offline: true): Directory;
161function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
162function getDirInTree(path: string, create: false, offline: true): Directory | null;
163function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
164function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
165function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
166// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
167function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
168 let current = offline ? offlineTreeRoot : onlineTreeRoot;
169 if (path === "") return current;
170 const parts = path.split("/");
171
172 for (const part of parts) {
173 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
174 if (!next || "metadata" in next) {
175 if (!create) return null;
176 //if mediafile already exists, we overwrite it with a directory anyway
177 if (offline) next = { files: new Map() };
178 else next = { files: new Map(), status: "Unknown" };
179 current.files.set(part, next);
180 }
181 current = next;
182 }
183 return current;
184}
185
186function deleteInOfflineTree(path: string): void {
187 if (path === "") return;
188 const basePath = path.split("/").slice(0, -1).join("/");
189 const fileName = path.split("/").toReversed()[0];
190
191 const node = getDirInTree(basePath, false, true);
192 if (!node) return;
193
194 node.files.delete(fileName);
195 if (node.files.size === 0) deleteInOfflineTree(basePath);
196}
197
198export const clearDirectoryCache = (path: string) => {
199 const node = getDirInTree(path, false, false);
200 if (!node) return;
201 node.files.clear();
202 node.status = "Unknown";
203};
204
205async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
206 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
207 if (response?.status === 404) {
208 toast(`Directory not found: ${dir}`, "error");
209 return;
210 }
211 if (!response || !response.ok) {
212 toast(
213 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
214 "error",
215 );
216 return;
217 }
218
219 return parseMap(await response.text()) as FileListingWithStatus;
220}
221
222function listOfflineFiles(dir: string): FileListing {
223 const node = getDirInTree(dir, false, true);
224 if (!node || !node.files) return new Map();
225 return node.files;
226}
227
228//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
229function fullyKnown(dir: DirectoryWithStatus) {
230 if (dir.status === "Unknown") {
231 return false;
232 }
233 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
234 return false;
235 }
236 return true;
237}
238
239//non-recursive listing should not include nested children
240function pruneChildren(listing: FileListing) {
241 const pruned = new Map();
242 for (const [name, item] of listing.entries()) {
243 if ("metadata" in item) pruned.set(name, item);
244 else pruned.set(name, { files: new Map() });
245 }
246 return pruned;
247}
248
249//used for combining the online and offline results, to be able to show files that only exist locally
250function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
251 const result = new Map(listing1);
252
253 for (const [key, value2] of listing2.entries()) {
254 const value1 = result.get(key);
255
256 if (value1 && "files" in value1 && "files" in value2) {
257 result.set(key, { files: combineFileListings(value1.files, value2.files) });
258 } else {
259 result.set(key, value2);
260 }
261 }
262
263 return result;
264}
265
266// File listing logic with hybrid online/offline support
267// Combines:
268// 1. Direct IndexedDB lookups for offline mode
269// 2. Cached online listings
270// 3. Merged online+offline results
271export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
272 if (offline) {
273 const files = listOfflineFiles(dir);
274 return recursive ? files : pruneChildren(files);
275 }
276 const cached = getDirInTree(dir, false, false);
277 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
278 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
279 return recursive ? combined : pruneChildren(combined);
280 }
281 const files = await listOnlineFiles(dir, recursive);
282 if (!files) return new Map();
283 updateFileTree(dir, { files, status: "Scanned" }, false);
284 const combined = combineFileListings(files, listOfflineFiles(dir));
285 return recursive ? combined : pruneChildren(combined);
286}
287
288//get cover art for a directory with cache-first strategy
289export async function getCover(dir: string, offline: boolean) {
290 const cached = await db.covers.get(dir);
291 if (cached !== undefined) {
292 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
293 }
294
295 if (offline) return undefined;
296
297 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
298 if (!response || response.status === 404) {
299 await db.covers.put({ path: dir, data: null });
300 return undefined;
301 }
302 const blob = await response.blob();
303 await db.covers.put({ path: dir, data: blob });
304 return URL.createObjectURL(blob);
305}
306
307export const clearCoverCache = async (path: string) => {
308 await db.covers.delete(path);
309};
310
311//returns an object url to the blob if synced, otherwise the server url
312export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
313export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
314 if (!forceUrl) {
315 const dbResult = await db.files.get(path);
316 if (dbResult) return URL.createObjectURL(dbResult.data);
317
318 if (offline) {
319 toast("This file is not cached, leave offline mode to play it", "error");
320 return undefined;
321 }
322 }
323 if (options.enableTranscoding) {
324 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
325 }
326 return joinPath("/download", encodePath(path));
327}
328
329export async function loadPlaylists() {
330 return await db.playlists.toArray();
331}
332
333//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
334export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
335 playlist.items = playlist.items.map((item) => {
336 //some less space in the db by omitting ids
337 //spread item.metadata because it might be a proxy object
338 return { path: item.path, metadata: { ...item.metadata } };
339 });
340
341 if (overwrite) {
342 await db.playlists.put(playlist);
343 } else {
344 await db.playlists.add(playlist);
345 }
346}
347
348export async function deletePlaylist(name: string) {
349 await db.playlists.delete(name);
350}
351
352function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
353 const paths: string[] = [];
354
355 for (const [name, item] of listing.entries()) {
356 const currentPath = joinPath(basePath, name);
357
358 if ("metadata" in item) {
359 paths.push(currentPath);
360 } else if ("files" in item) {
361 paths.push(...collectMediaFilePaths(item.files, currentPath));
362 }
363 }
364
365 return paths;
366}
367
368//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
369export async function deleteFile(
370 fileName: string,
371 item: MediaFile | Directory,
372 basePath: string,
373 fetchFiles: ResourceActions<FlatFileListing>,
374) {
375 const fullPath = joinPath(basePath, fileName);
376 try {
377 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
378 startProgress(fullPath, { amount: 1 });
379
380 let toDelete: string[];
381 if ("files" in item) {
382 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
383 } else {
384 toDelete = [fullPath];
385 }
386
387 await db.files.bulkDelete(toDelete); //delete from database
388 deleteInOfflineTree(fullPath); //delete node from tree
389 for (const path of toDelete) {
390 setDownloadStatusMap(path, undefined); // set all children to unsynced
391 }
392 fetchFiles.refetch();
393 } finally {
394 stopProgress(fullPath, undefined);
395 }
396}
397
398const downloadSemaphore = new AsyncSemaphore(1);
399
400//Download file to indexedDB and updates the offline file tree
401//concurrency control via the downloadSemaphore
402async function downloadFile(
403 path: string,
404 options: AppOptions,
405 item: MediaFile,
406 fetchFiles: ResourceActions<FlatFileListing>,
407 redownload: boolean,
408): Promise<void> {
409 const retry = () => {
410 setDownloadStatusMap(path, undefined);
411 return downloadFile(path, options, item, fetchFiles, redownload);
412 };
413
414 const controller = new AbortController();
415 let abortResolve!: () => void;
416 const abortPromise = new Promise<void>((resolve) => {
417 abortResolve = resolve;
418 }).then(() => {
419 controller.abort();
420 });
421
422 // file already downloaded or download already in progress
423 const initialProgress = downloadStatusMap[path];
424 if (
425 (!redownload && initialProgress === "Synced") ||
426 (typeof initialProgress === "object" && "bytes" in initialProgress)
427 )
428 return;
429 startProgress(path, { status: 0, abort: abortResolve });
430
431 try {
432 //options might change at any point, update the semaphore to the current value
433 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
434
435 //make sure to only allow the set amount of parallel downloads
436 const semaphorePromise = downloadSemaphore.acquire();
437 await Promise.race([abortPromise, semaphorePromise]);
438 try {
439 const res = await fetch(await getSrc(path, options, false, true), { signal: controller.signal });
440
441 //let the browser do the actual download
442 const blobPromise = res.clone().blob();
443 if (!res.ok || !res.body) throw res.statusText;
444 //we just iterate over the chunks to get the current progress
445 const reader = res.body.getReader();
446 let received = 0;
447
448 let lastUpdate = 0;
449 while (true) {
450 const { done, value } = await reader.read();
451 if (done) break;
452 received += value.length;
453
454 //debouncing logic to not update UI too fast
455 const now = Date.now();
456 if (now - lastUpdate < 250) continue;
457 lastUpdate = now;
458 setDownloadStatusMap(path, { status: received, abort: abortResolve });
459 }
460 const blob = await blobPromise;
461
462 await db.files.put({
463 path,
464 data: blob,
465 metadata: { ...item.metadata }, //spread needed, because item might be a proxy object from the filebrowser, which can't be stored
466 });
467
468 updateFileTree(path, item, true);
469 stopProgress(path, "Synced");
470 fetchFiles.refetch();
471 } finally {
472 downloadSemaphore.release();
473 }
474 } catch (e) {
475 stopProgress(
476 path,
477 // if it was already synced, restore it
478 initialProgress === "Synced"
479 ? "Synced"
480 : //if it was manually aborted, set as aborted
481 e && typeof e === "object" && "name" in e && e.name === "AbortError"
482 ? { status: "Aborted", retry }
483 : //otherwise set as failed and set reason to the error message
484 {
485 status: "Failed",
486 reason: e ? e.toString() : "Unknown error",
487 retry,
488 },
489 );
490 }
491}
492
493export async function download(
494 fileName: string,
495 item: MediaFile | Directory,
496 basePath: string,
497 options: AppOptions,
498 fetchFiles: ResourceActions<FlatFileListing>,
499 redownload = false,
500) {
501 const fullPath = joinPath(basePath, fileName);
502 if ("metadata" in item) {
503 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
504 } else {
505 updateRecursiveStatus(fullPath, 1, true);
506 try {
507 await getCover(fullPath, false);
508 //start downloads for all children to allow them to run in parallel, then wait for the results
509 const promises: Promise<unknown>[] = [];
510 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
511 promises.push(download(childName, childItem, fullPath, options, fetchFiles, redownload));
512 }
513 await Promise.allSettled(promises);
514 } finally {
515 updateRecursiveStatus(fullPath, -1, true);
516 }
517 }
518}
519
520export async function getStoredFiles() {
521 return await db.files.toCollection().primaryKeys();
522}
523