pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist } from "./types";
10import { joinPath, toast } from "./utils";
11import { batch, from, observable, untrack, type ResourceActions } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14import type { FlatFileListing } from "./App";
15import { createStore } from "solid-js/store";
16import Dexie, { liveQuery } from "dexie";
17
18interface StoredFile {
19 path: string;
20 data: Blob;
21 metadata: {
22 duration: number;
23 };
24}
25
26interface StoredCover {
27 path: string;
28 data: Blob | null;
29}
30
31interface Database extends Dexie {
32 files: Dexie.Table<StoredFile, string>;
33 covers: Dexie.Table<StoredCover, string>;
34 playlists: Dexie.Table<Playlist, string>;
35}
36
37const db = new Dexie("music-files") as Database;
38db.version(1).stores({
39 files: "path",
40 covers: "path",
41 playlists: "name",
42});
43
44export type Progress =
45 | { status: number; abort: () => void }
46 | { status: "Aborted"; retry: () => void }
47 | { status: "Failed"; retry: () => void; reason: string };
48
49interface LoadingChildren {
50 amount: number;
51}
52
53export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
54export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
55
56//creates entry if it doesn't exist, so reactivity works
57export function getDownloadStatus(path: string): DownloadStatus {
58 untrack(() => {
59 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
60 });
61 return downloadStatusMap[path];
62}
63
64function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
65 function updateProgressAmount(path: string) {
66 const currentStatus = downloadStatusMap[path];
67 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
68 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
69 return;
70 }
71
72 const newAmount = currentStatus.amount + loadChange;
73 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
74 }
75
76 if (includeCurrent) updateProgressAmount(path);
77 // Update all parent directories
78 let currentPath = path;
79 while (true) {
80 const lastSlash = currentPath.lastIndexOf("/");
81 if (lastSlash === -1) break;
82 currentPath = currentPath.substring(0, lastSlash);
83 updateProgressAmount(currentPath);
84 }
85}
86
87function startProgress(path: string, status: DownloadStatus) {
88 updateRecursiveStatus(path, 1, false);
89 setDownloadStatusMap(path, status);
90}
91
92function stopProgress(path: string, status: DownloadStatus) {
93 updateRecursiveStatus(path, -1, false);
94 setDownloadStatusMap(path, status);
95}
96
97//initialize filetrees and download statuses from indexedDB
98export async function loadDataFromDB() {
99 try {
100 const files = await db.files.toArray();
101 for (const file of files) {
102 setDownloadStatusMap(file.path, "Synced");
103 updateFileTree(file.path, { metadata: file.metadata }, true);
104 }
105 } catch (e) {
106 toast("Loading offline files failed", "error", e);
107 console.error(e);
108 }
109}
110
111//contains all locally synced files
112const offlineTreeRoot: Directory = {
113 files: new Map(),
114};
115//used for caching online file listings
116const onlineTreeRoot: DirectoryWithStatus = {
117 files: new Map(),
118 status: "Unknown",
119};
120
121//merges "Scanned" directories from "other" to "target"
122function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
123 if (other.status === "Unknown") return;
124 target.status = other.status;
125 for (const [fileName, file] of other.files.entries()) {
126 const targetFile = target.files.get(fileName);
127 if (targetFile && "files" in targetFile && "files" in file) {
128 mergeTrees(targetFile, file);
129 } else {
130 target.files.set(fileName, file);
131 }
132 }
133}
134
135//TODO: improve typing to avoid "as"
136//overloads for typecheking
137//offline tree only stores files, directories are created ad-hoc
138function updateFileTree(path: string, file: MediaFile, offline: true): void;
139//online tree only stores cached directory listings
140function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
141// Updates file tree structure with new data (offline or online)
142function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
143 if (offline && path === "") throw new Error("Trying to update root in offline mode");
144 const basePath = path.split("/").slice(0, -1).join("/");
145 const fileName = path.split("/").toReversed()[0];
146 const node = getDirInTree(basePath, true, offline);
147
148 if (offline) {
149 node.files.set(fileName, file);
150 } else {
151 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
152 else {
153 let target = node.files.get(fileName) as DirectoryWithStatus;
154 if (!target) target = { files: new Map(), status: "Unknown" };
155 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
156 node.files.set(fileName, target);
157 }
158 }
159}
160
161//overloads for typechecking
162function getDirInTree(path: string, create: true, offline: true): Directory;
163function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
164function getDirInTree(path: string, create: false, offline: true): Directory | null;
165function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
166function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
167function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
168// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
169function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
170 let current = offline ? offlineTreeRoot : onlineTreeRoot;
171 if (path === "") return current;
172 const parts = path.split("/");
173
174 for (const part of parts) {
175 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
176 if (!next || "metadata" in next) {
177 if (!create) return null;
178 //if mediafile already exists, we overwrite it with a directory anyway
179 if (offline) next = { files: new Map() };
180 else next = { files: new Map(), status: "Unknown" };
181 current.files.set(part, next);
182 }
183 current = next;
184 }
185 return current;
186}
187
188function deleteInOfflineTree(path: string): void {
189 if (path === "") return;
190 const basePath = path.split("/").slice(0, -1).join("/");
191 const fileName = path.split("/").toReversed()[0];
192
193 const node = getDirInTree(basePath, false, true);
194 if (!node) return;
195
196 node.files.delete(fileName);
197 if (node.files.size === 0) deleteInOfflineTree(basePath);
198}
199
200export const clearDirectoryCache = (path: string) => {
201 const node = getDirInTree(path, false, false);
202 if (!node) return;
203 node.files.clear();
204 node.status = "Unknown";
205};
206
207async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
208 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
209 if (response?.status === 404) {
210 toast(`Directory not found: ${dir}`, "error");
211 return;
212 }
213 if (!response || !response.ok) {
214 toast(
215 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
216 "error",
217 );
218 return;
219 }
220
221 return parseMap(await response.text()) as FileListingWithStatus;
222}
223
224function listOfflineFiles(dir: string): FileListing {
225 const node = getDirInTree(dir, false, true);
226 if (!node || !node.files) return new Map();
227 return node.files;
228}
229
230//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
231function fullyKnown(dir: DirectoryWithStatus) {
232 if (dir.status === "Unknown") {
233 return false;
234 }
235 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
236 return false;
237 }
238 return true;
239}
240
241//non-recursive listing should not include nested children
242function pruneChildren(listing: FileListing) {
243 const pruned = new Map();
244 for (const [name, item] of listing.entries()) {
245 if ("metadata" in item) pruned.set(name, item);
246 else pruned.set(name, { files: new Map() });
247 }
248 return pruned;
249}
250
251//used for combining the online and offline results, to be able to show files that only exist locally
252function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
253 const result = new Map(listing1);
254
255 for (const [key, value2] of listing2.entries()) {
256 const value1 = result.get(key);
257
258 if (value1 && "files" in value1 && "files" in value2) {
259 result.set(key, { files: combineFileListings(value1.files, value2.files) });
260 } else {
261 result.set(key, value2);
262 }
263 }
264
265 return result;
266}
267
268// File listing logic with hybrid online/offline support
269// Combines:
270// 1. Direct IndexedDB lookups for offline mode
271// 2. Cached online listings
272// 3. Merged online+offline results
273export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
274 if (offline) {
275 const files = listOfflineFiles(dir);
276 return recursive ? files : pruneChildren(files);
277 }
278 const cached = getDirInTree(dir, false, false);
279 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
280 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
281 return recursive ? combined : pruneChildren(combined);
282 }
283 const files = await listOnlineFiles(dir, recursive);
284 if (!files) return new Map();
285 updateFileTree(dir, { files, status: "Scanned" }, false);
286 const combined = combineFileListings(files, listOfflineFiles(dir));
287 return recursive ? combined : pruneChildren(combined);
288}
289
290//get cover art for a directory with cache-first strategy
291export async function getCover(dir: string, offline: boolean) {
292 try {
293 const cached = await db.covers.get(dir);
294 if (cached !== undefined) {
295 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
296 }
297
298 if (offline) return undefined;
299
300 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
301 if (!response || response.status === 404) {
302 await db.covers.put({ path: dir, data: null });
303 return undefined;
304 }
305 const blob = await response.blob();
306 await db.covers.put({ path: dir, data: blob });
307 return URL.createObjectURL(blob);
308 } catch {
309 return undefined;
310 }
311}
312
313export const clearCoverCache = async (path: string) => {
314 await db.covers.delete(path);
315};
316
317//returns an object url to the blob if synced, otherwise the server url
318export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
319export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
320 if (!forceUrl) {
321 const dbResult = await db.files.get(path);
322 if (dbResult) return URL.createObjectURL(dbResult.data);
323
324 if (offline) {
325 toast("This file is not cached, leave offline mode to play it", "error");
326 return undefined;
327 }
328 }
329 if (options.enableTranscoding) {
330 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
331 }
332 return joinPath("/download", encodePath(path));
333}
334
335export async function loadPlaylists() {
336 return await db.playlists.toArray();
337}
338
339//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
340export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
341 playlist.items = playlist.items.map((item) => {
342 //some less space in the db by omitting ids
343 //spread item.metadata because it might be a proxy object
344 return { path: item.path, metadata: { ...item.metadata } };
345 });
346
347 if (overwrite) {
348 await db.playlists.put(playlist);
349 } else {
350 await db.playlists.add(playlist);
351 }
352}
353
354export async function deletePlaylist(name: string) {
355 await db.playlists.delete(name);
356}
357
358function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
359 const paths: string[] = [];
360
361 for (const [name, item] of listing.entries()) {
362 const currentPath = joinPath(basePath, name);
363
364 if ("metadata" in item) {
365 paths.push(currentPath);
366 } else if ("files" in item) {
367 paths.push(...collectMediaFilePaths(item.files, currentPath));
368 }
369 }
370
371 return paths;
372}
373
374//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
375export async function deleteFile(
376 fileName: string,
377 item: MediaFile | Directory,
378 basePath: string,
379 fetchFiles: ResourceActions<FlatFileListing>,
380) {
381 const fullPath = joinPath(basePath, fileName);
382 try {
383 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
384 startProgress(fullPath, { amount: 1 });
385
386 let toDelete: string[];
387 if ("files" in item) {
388 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
389 } else {
390 toDelete = [fullPath];
391 }
392
393 await db.files.bulkDelete(toDelete); //delete from database
394 deleteInOfflineTree(fullPath); //delete node from tree
395 batch(() => {
396 for (const path of toDelete) {
397 setDownloadStatusMap(path, undefined); // set all children to unsynced
398 }
399 });
400 fetchFiles.refetch();
401 } finally {
402 stopProgress(fullPath, undefined);
403 }
404}
405
406const downloadSemaphore = new AsyncSemaphore(1);
407
408//Download file to indexedDB and updates the offline file tree
409//concurrency control via the downloadSemaphore
410async function downloadFile(
411 path: string,
412 options: AppOptions,
413 item: MediaFile,
414 fetchFiles: ResourceActions<FlatFileListing>,
415 redownload: boolean,
416): Promise<void> {
417 const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
418
419 const controller = new AbortController();
420 let abortResolve!: () => void;
421 const abortPromise = new Promise<void>((outerResolve) => {
422 abortResolve = () => {
423 controller.abort();
424 stopProgress(
425 path,
426 // if it was already synced, restore it
427 initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
428 );
429 outerResolve();
430 };
431 });
432 // file already downloaded or download already in progress
433 const initialProgress = downloadStatusMap[path];
434 if (
435 (!redownload && initialProgress === "Synced") ||
436 (typeof initialProgress === "object" && "bytes" in initialProgress)
437 )
438 return;
439 startProgress(path, { status: 0, abort: abortResolve });
440
441 try {
442 //options might change at any point, update the semaphore to the current value
443 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
444
445 //make sure to only allow the set amount of parallel downloads
446 const semaphorePromise = downloadSemaphore.acquire();
447 await Promise.race([abortPromise, semaphorePromise]);
448 try {
449 const res = await fetch(await getSrc(path, options, false, true), { signal: controller.signal });
450
451 //let the browser do the actual download
452 const blobPromise = res.clone().blob();
453 if (!res.ok || !res.body) throw res.statusText;
454 //we just iterate over the chunks to get the current progress
455 const reader = res.body.getReader();
456 let received = 0;
457
458 let lastUpdate = 0;
459 while (true) {
460 const { done, value } = await reader.read();
461 if (done) break;
462 received += value.length;
463
464 //debouncing logic to not update UI too fast
465 const now = Date.now();
466 if (now - lastUpdate < 250) continue;
467 lastUpdate = now;
468 setDownloadStatusMap(path, { status: received, abort: abortResolve });
469 }
470 const blob = await blobPromise;
471
472 await db.files.put({
473 path,
474 data: blob,
475 metadata: { ...item.metadata }, //spread needed, because item might be a proxy object from the filebrowser, which can't be stored
476 });
477
478 updateFileTree(path, item, true);
479 stopProgress(path, "Synced");
480 fetchFiles.refetch();
481 } finally {
482 downloadSemaphore.release();
483 }
484 } catch (e) {
485 if (e && typeof e === "object" && "name" in e && e.name === "AbortError") return;
486 stopProgress(
487 path,
488 // if it was already synced, restore it
489 initialProgress === "Synced"
490 ? "Synced"
491 : {
492 status: "Failed",
493 reason: e ? e.toString() : "Unknown error",
494 retry,
495 },
496 );
497 }
498}
499
500export async function download(
501 fileName: string,
502 item: MediaFile | Directory,
503 basePath: string,
504 options: AppOptions,
505 fetchFiles: ResourceActions<FlatFileListing>,
506 redownload = false,
507) {
508 const fullPath = joinPath(basePath, fileName);
509 if ("metadata" in item) {
510 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
511 } else {
512 updateRecursiveStatus(fullPath, 1, true);
513 try {
514 await getCover(fullPath, false);
515 //start downloads for all children to allow them to run in parallel, then wait for the results
516 const promises: Promise<unknown>[] = [];
517 const listing = await listFiles(fullPath, false, false);
518 batch(() => {
519 for (const [childName, childItem] of listing) {
520 promises.push(download(childName, childItem, fullPath, options, fetchFiles, redownload));
521 }
522 });
523 await Promise.allSettled(promises);
524 } finally {
525 updateRecursiveStatus(fullPath, -1, true);
526 }
527 }
528}
529
530export const storedFilesLive = from(
531 liveQuery(async () => await db.files.toCollection().keys()),
532 [],
533);
534