pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7 Metadata,
8} from "music-server-shared/types";
9import { parseMap } from "music-server-shared/mapconversion";
10import type { AppOptions, Playlist } from "./types";
11import { joinPath, toast, unproxy } from "./utils";
12import { batch, from, untrack, type ResourceActions } from "solid-js";
13import { AsyncSemaphore } from "./semaphore";
14import { encodePath } from "music-server-shared/utils";
15import type { FlatFileListing } from "./App";
16import { createStore } from "solid-js/store";
17import Dexie, { liveQuery } from "dexie";
18
19interface StoredFile {
20 path: string;
21 data: Blob;
22 metadata: Metadata;
23}
24
25interface StoredCover {
26 path: string;
27 data: Blob | null;
28}
29
30interface Database extends Dexie {
31 files: Dexie.Table<StoredFile, string>;
32 covers: Dexie.Table<StoredCover, string>;
33 playlists: Dexie.Table<Playlist, string>;
34}
35
36const db = new Dexie("music-files") as Database;
37db.version(1).stores({
38 files: "path",
39 covers: "path",
40 playlists: "name",
41});
42
43export type Progress =
44 | { status: number; abort: () => void }
45 | { status: "Aborted"; retry: () => void }
46 | { status: "Failed"; retry: () => void; reason: string };
47
48interface LoadingChildren {
49 amount: number;
50}
51
52export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
53export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
54
55//creates entry if it doesn't exist, so reactivity works
56export function getDownloadStatus(path: string): DownloadStatus {
57 untrack(() => {
58 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
59 });
60 return downloadStatusMap[path];
61}
62
63function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
64 function updateProgressAmount(path: string) {
65 const currentStatus = downloadStatusMap[path];
66 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
67 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
68 return;
69 }
70
71 const newAmount = currentStatus.amount + loadChange;
72 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
73 }
74
75 if (includeCurrent) updateProgressAmount(path);
76 // Update all parent directories
77 let currentPath = path;
78 while (true) {
79 const lastSlash = currentPath.lastIndexOf("/");
80 if (lastSlash === -1) break;
81 currentPath = currentPath.substring(0, lastSlash);
82 updateProgressAmount(currentPath);
83 }
84}
85
86function startProgress(path: string, status: DownloadStatus) {
87 updateRecursiveStatus(path, 1, false);
88 setDownloadStatusMap(path, status);
89}
90
91function stopProgress(path: string, status: DownloadStatus) {
92 updateRecursiveStatus(path, -1, false);
93 setDownloadStatusMap(path, status);
94}
95
96//initialize filetrees and download statuses from indexedDB
97export async function loadDataFromDB() {
98 try {
99 const files = await db.files.toArray();
100 for (const file of files) {
101 setDownloadStatusMap(file.path, "Synced");
102 updateFileTree(file.path, { metadata: file.metadata }, true);
103 }
104 } catch (e) {
105 toast("Loading offline files failed", "error", e);
106 console.error(e);
107 }
108}
109
110//contains all locally synced files
111const offlineTreeRoot: Directory = {
112 files: new Map(),
113};
114//used for caching online file listings
115const onlineTreeRoot: DirectoryWithStatus = {
116 files: new Map(),
117 status: "Unknown",
118};
119
120//merges "Scanned" directories from "other" to "target"
121function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
122 if (other.status === "Unknown") return;
123 target.status = other.status;
124 for (const [fileName, file] of other.files.entries()) {
125 const targetFile = target.files.get(fileName);
126 if (targetFile && "files" in targetFile && "files" in file) {
127 mergeTrees(targetFile, file);
128 } else {
129 target.files.set(fileName, file);
130 }
131 }
132}
133
134//TODO: improve typing to avoid "as"
135//overloads for typecheking
136//offline tree only stores files, directories are created ad-hoc
137function updateFileTree(path: string, file: MediaFile, offline: true): void;
138//online tree only stores cached directory listings
139function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
140// Updates file tree structure with new data (offline or online)
141function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
142 if (offline && path === "") throw new Error("Trying to update root in offline mode");
143 const basePath = path.split("/").slice(0, -1).join("/");
144 const fileName = path.split("/").toReversed()[0];
145 const node = getDirInTree(basePath, true, offline);
146
147 if (offline) {
148 node.files.set(fileName, file);
149 } else {
150 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
151 else {
152 let target = node.files.get(fileName) as DirectoryWithStatus;
153 if (!target) target = { files: new Map(), status: "Unknown" };
154 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
155 node.files.set(fileName, target);
156 }
157 }
158}
159
160//overloads for typechecking
161function getDirInTree(path: string, create: true, offline: true): Directory;
162function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
163function getDirInTree(path: string, create: false, offline: true): Directory | null;
164function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
165function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
166function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
167// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
168function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
169 let current = offline ? offlineTreeRoot : onlineTreeRoot;
170 if (path === "") return current;
171 const parts = path.split("/");
172
173 for (const part of parts) {
174 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
175 if (!next || "metadata" in next) {
176 if (!create) return null;
177 //if mediafile already exists, we overwrite it with a directory anyway
178 if (offline) next = { files: new Map() };
179 else next = { files: new Map(), status: "Unknown" };
180 current.files.set(part, next);
181 }
182 current = next;
183 }
184 return current;
185}
186
187function deleteInOfflineTree(path: string): void {
188 if (path === "") return;
189 const basePath = path.split("/").slice(0, -1).join("/");
190 const fileName = path.split("/").toReversed()[0];
191
192 const node = getDirInTree(basePath, false, true);
193 if (!node) return;
194
195 node.files.delete(fileName);
196 if (node.files.size === 0) deleteInOfflineTree(basePath);
197}
198
199export const clearDirectoryCache = (path: string) => {
200 const node = getDirInTree(path, false, false);
201 if (!node) return;
202 node.files.clear();
203 node.status = "Unknown";
204};
205
206async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
207 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
208 if (response?.status === 404) {
209 toast(`Directory not found: ${dir}`, "error");
210 return;
211 }
212 if (!response || !response.ok) {
213 toast(
214 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
215 "error",
216 );
217 return;
218 }
219
220 return parseMap(await response.text()) as FileListingWithStatus;
221}
222
223function listOfflineFiles(dir: string): FileListing {
224 const node = getDirInTree(dir, false, true);
225 if (!node || !node.files) return new Map();
226 return node.files;
227}
228
229//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
230function fullyKnown(dir: DirectoryWithStatus) {
231 if (dir.status === "Unknown") {
232 return false;
233 }
234 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
235 return false;
236 }
237 return true;
238}
239
240//non-recursive listing should not include nested children
241function pruneChildren(listing: FileListing) {
242 const pruned = new Map();
243 for (const [name, item] of listing.entries()) {
244 if ("metadata" in item) pruned.set(name, item);
245 else pruned.set(name, { files: new Map() });
246 }
247 return pruned;
248}
249
250//used for combining the online and offline results, to be able to show files that only exist locally
251function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
252 const result = new Map(listing1);
253
254 for (const [key, value2] of listing2.entries()) {
255 const value1 = result.get(key);
256
257 if (value1 && "files" in value1 && "files" in value2) {
258 result.set(key, { files: combineFileListings(value1.files, value2.files) });
259 } else {
260 result.set(key, value2);
261 }
262 }
263
264 return result;
265}
266
267// File listing logic with hybrid online/offline support
268// Combines:
269// 1. Direct IndexedDB lookups for offline mode
270// 2. Cached online listings
271// 3. Merged online+offline results
272export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
273 if (offline) {
274 const files = listOfflineFiles(dir);
275 return recursive ? files : pruneChildren(files);
276 }
277 const cached = getDirInTree(dir, false, false);
278 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
279 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
280 return recursive ? combined : pruneChildren(combined);
281 }
282 const files = await listOnlineFiles(dir, recursive);
283 if (!files) return new Map();
284 updateFileTree(dir, { files, status: "Scanned" }, false);
285 const combined = combineFileListings(files, listOfflineFiles(dir));
286 return recursive ? combined : pruneChildren(combined);
287}
288
289//get cover art object-url for a file or directory with cache-first strategy
290export async function getCover(dir: string, offline: boolean) {
291 try {
292 const cached = await db.covers.get(dir);
293 if (cached !== undefined) {
294 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
295 }
296
297 if (offline) return undefined;
298
299 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
300 if (!response || response.status === 404) {
301 await db.covers.put({ path: dir, data: null });
302 return undefined;
303 }
304 const blob = await response.blob();
305 await db.covers.put({ path: dir, data: blob });
306 return URL.createObjectURL(blob);
307 } catch {
308 return undefined;
309 }
310}
311
312export async function clearCoverCache(path: string) {
313 await db.covers.delete(path);
314}
315
316//returns an object url to the blob if synced, otherwise the server url
317export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
318export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
319 if (!forceUrl) {
320 const dbResult = await db.files.get(path);
321 if (dbResult) return URL.createObjectURL(dbResult.data);
322
323 if (offline) {
324 toast("This file is not cached, leave offline mode to play it", "error");
325 return undefined;
326 }
327 }
328 if (options.enableTranscoding) {
329 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
330 }
331 return joinPath("/download", encodePath(path));
332}
333
334export async function loadPlaylists() {
335 return await db.playlists.toArray();
336}
337
338//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
339export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
340 playlist.items = playlist.items.map((item) => {
341 //some less space in the db by omitting ids
342 return { path: item.path, metadata: unproxy(item.metadata) };
343 });
344
345 if (overwrite) {
346 await db.playlists.put(playlist);
347 } else {
348 await db.playlists.add(playlist);
349 }
350}
351
352export async function deletePlaylist(name: string) {
353 await db.playlists.delete(name);
354}
355
356function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
357 const paths: string[] = [];
358
359 for (const [name, item] of listing.entries()) {
360 const currentPath = joinPath(basePath, name);
361
362 if ("metadata" in item) {
363 paths.push(currentPath);
364 } else if ("files" in item) {
365 paths.push(...collectMediaFilePaths(item.files, currentPath));
366 }
367 }
368
369 return paths;
370}
371
372//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
373export async function deleteFile(
374 fullPath: string,
375 item: MediaFile | Directory,
376 fetchFiles: ResourceActions<FlatFileListing>,
377) {
378 try {
379 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
380 startProgress(fullPath, { amount: 1 });
381
382 let toDelete: string[];
383 if ("files" in item) {
384 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
385 } else {
386 toDelete = [fullPath];
387 }
388
389 await db.files.bulkDelete(toDelete); //delete file from database
390 await db.covers.bulkDelete(toDelete); //delete embedded cover from database
391 deleteInOfflineTree(fullPath); //delete node from tree
392 batch(() => {
393 for (const path of toDelete) {
394 setDownloadStatusMap(path, undefined); // set all children to unsynced
395 }
396 });
397 fetchFiles.refetch();
398 } finally {
399 stopProgress(fullPath, undefined);
400 }
401}
402
403const downloadSemaphore = new AsyncSemaphore(1);
404
405//Download file to indexedDB and updates the offline file tree
406//concurrency control via the downloadSemaphore
407async function downloadFile(
408 path: string,
409 options: AppOptions,
410 item: MediaFile,
411 fetchFiles: ResourceActions<FlatFileListing>,
412 redownload: boolean,
413): Promise<void> {
414 const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
415
416 const controller = new AbortController();
417 let abortResolve!: () => void;
418 const abortPromise = new Promise<void>((outerResolve) => {
419 abortResolve = () => {
420 controller.abort();
421 stopProgress(
422 path,
423 // if it was already synced, restore it
424 initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
425 );
426 outerResolve();
427 };
428 });
429 // file already downloaded or download already in progress
430 const initialProgress = downloadStatusMap[path];
431 if (
432 (!redownload && initialProgress === "Synced") ||
433 (typeof initialProgress === "object" && "bytes" in initialProgress)
434 )
435 return;
436 startProgress(path, { status: 0, abort: abortResolve });
437
438 try {
439 //options might change at any point, update the semaphore to the current value
440 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
441
442 //make sure to only allow the set amount of parallel downloads
443 const semaphorePromise = downloadSemaphore.acquire();
444 await Promise.race([abortPromise, semaphorePromise]);
445 try {
446 const res = await fetch(await getSrc(path, options, false, true), { signal: controller.signal });
447
448 //let the browser do the actual download
449 const blobPromise = res.clone().blob();
450 if (!res.ok || !res.body) throw res.statusText;
451 //we just iterate over the chunks to get the current progress
452 const reader = res.body.getReader();
453 let received = 0;
454
455 let lastUpdate = 0;
456 while (true) {
457 const { done, value } = await reader.read();
458 if (done) break;
459 received += value.length;
460
461 //debouncing logic to not update UI too fast
462 const now = Date.now();
463 if (now - lastUpdate < 250) continue;
464 lastUpdate = now;
465 setDownloadStatusMap(path, { status: received, abort: abortResolve });
466 }
467 const blob = await blobPromise;
468
469 await db.files.put({
470 path,
471 data: blob,
472 metadata: unproxy(item.metadata),
473 });
474
475 updateFileTree(path, item, true);
476 stopProgress(path, "Synced");
477 fetchFiles.refetch();
478 } finally {
479 downloadSemaphore.release();
480 }
481 } catch (e) {
482 if (e && typeof e === "object" && "name" in e && e.name === "AbortError") return;
483 stopProgress(
484 path,
485 // if it was already synced, restore it
486 initialProgress === "Synced"
487 ? "Synced"
488 : {
489 status: "Failed",
490 reason: e ? e.toString() : "Unknown error",
491 retry,
492 },
493 );
494 }
495}
496
497export async function download(
498 fullPath: string,
499 item: MediaFile | Directory,
500 options: AppOptions,
501 fetchFiles: ResourceActions<FlatFileListing>,
502 redownload = false,
503) {
504 if ("metadata" in item) {
505 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
506 } else {
507 updateRecursiveStatus(fullPath, 1, true);
508 try {
509 getCover(fullPath, false).then((cover) => URL.revokeObjectURL(cover || ""));
510 //start downloads for all children to allow them to run in parallel, then wait for the results
511 const promises: Promise<unknown>[] = [];
512 const listing = await listFiles(fullPath, false, false);
513 batch(() => {
514 for (const [childName, childItem] of listing) {
515 promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload));
516 }
517 });
518 await Promise.allSettled(promises);
519 } finally {
520 updateRecursiveStatus(fullPath, -1, true);
521 }
522 }
523}
524
525export const storedFilesLive = from(
526 liveQuery(async () => await db.files.toCollection().keys()),
527 [],
528);
529