pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7 Metadata,
8 IsVideoResponse,
9} from "music-server-shared/types";
10import { parseMap } from "music-server-shared/mapconversion";
11import type { AppOptions, Playlist } from "./types";
12import { joinPath, toast, unproxy } from "./utils";
13import { batch, from, untrack, type ResourceActions } from "solid-js";
14import { AsyncSemaphore } from "./semaphore";
15import { encodePath } from "music-server-shared/utils";
16import type { FlatFileListing } from "./App";
17import { createStore } from "solid-js/store";
18import Dexie, { liveQuery } from "dexie";
19
20interface StoredFile {
21 path: string;
22 data: Blob;
23 metadata: Metadata;
24}
25
26interface StoredCover {
27 path: string;
28 data: Blob | null;
29}
30
31interface Database extends Dexie {
32 files: Dexie.Table<StoredFile, string>;
33 covers: Dexie.Table<StoredCover, string>;
34 playlists: Dexie.Table<Playlist, string>;
35}
36
37const db = new Dexie("music-files") as Database;
38db.version(1).stores({
39 files: "path",
40 covers: "path",
41 playlists: "name",
42});
43
44export type Progress =
45 | { status: number; abort: () => void }
46 | { status: "Aborted"; retry: () => void }
47 | { status: "Failed"; retry: () => void; reason: string };
48
49interface LoadingChildren {
50 amount: number;
51}
52
53export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
54export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
55
56//creates entry if it doesn't exist, so reactivity works
57export function getDownloadStatus(path: string): DownloadStatus {
58 untrack(() => {
59 if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
60 });
61 return downloadStatusMap[path];
62}
63
64function updateRecursiveStatus(path: string, loadChange: number, includeCurrent: boolean) {
65 function updateProgressAmount(path: string) {
66 const currentStatus = downloadStatusMap[path];
67 if (typeof currentStatus !== "object" || !("amount" in currentStatus)) {
68 setDownloadStatusMap(path, loadChange > 0 ? { amount: loadChange } : undefined);
69 return;
70 }
71
72 const newAmount = currentStatus.amount + loadChange;
73 setDownloadStatusMap(path, newAmount > 0 ? { amount: newAmount } : undefined);
74 }
75
76 if (includeCurrent) updateProgressAmount(path);
77 // Update all parent directories
78 let currentPath = path;
79 while (true) {
80 const lastSlash = currentPath.lastIndexOf("/");
81 if (lastSlash === -1) break;
82 currentPath = currentPath.substring(0, lastSlash);
83 updateProgressAmount(currentPath);
84 }
85}
86
87function startProgress(path: string, status: DownloadStatus) {
88 updateRecursiveStatus(path, 1, false);
89 setDownloadStatusMap(path, status);
90}
91
92function stopProgress(path: string, status: DownloadStatus) {
93 updateRecursiveStatus(path, -1, false);
94 setDownloadStatusMap(path, status);
95}
96
97//initialize filetrees and download statuses from indexedDB
98export async function loadDataFromDB() {
99 try {
100 const files = await db.files.toArray();
101 for (const file of files) {
102 setDownloadStatusMap(file.path, "Synced");
103 updateFileTree(file.path, { metadata: file.metadata }, true);
104 }
105 const covers = await db.covers.toArray();
106 for (const cover of covers) {
107 coverCache.set(cover.path, { data: cover.data });
108 }
109 } catch (e) {
110 toast("Loading offline files failed", "error", e);
111 console.error(e);
112 }
113}
114
115//contains all locally synced files
116const offlineTreeRoot: Directory = {
117 files: new Map(),
118};
119//used for caching online file listings
120const onlineTreeRoot: DirectoryWithStatus = {
121 files: new Map(),
122 status: "Unknown",
123};
124
125//merges "Scanned" directories from "other" to "target"
126function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
127 if (other.status === "Unknown") return;
128 target.status = other.status;
129 for (const [fileName, file] of other.files.entries()) {
130 const targetFile = target.files.get(fileName);
131 if (targetFile && "files" in targetFile && "files" in file) {
132 mergeTrees(targetFile, file);
133 } else {
134 target.files.set(fileName, file);
135 }
136 }
137}
138
139//TODO: improve typing to avoid "as"
140//overloads for typecheking
141//offline tree only stores files, directories are created ad-hoc
142function updateFileTree(path: string, file: MediaFile, offline: true): void;
143//online tree only stores cached directory listings
144function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
145// Updates file tree structure with new data (offline or online)
146function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
147 if (offline && path === "") throw new Error("Trying to update root in offline mode");
148 const basePath = path.split("/").slice(0, -1).join("/");
149 const fileName = path.split("/").toReversed()[0];
150 const node = getDirInTree(basePath, true, offline);
151
152 if (offline) {
153 node.files.set(fileName, file);
154 } else {
155 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
156 else {
157 let target = node.files.get(fileName) as DirectoryWithStatus;
158 if (!target) target = { files: new Map(), status: "Unknown" };
159 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
160 node.files.set(fileName, target);
161 }
162 }
163}
164
165//overloads for typechecking
166function getDirInTree(path: string, create: true, offline: true): Directory;
167function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
168function getDirInTree(path: string, create: false, offline: true): Directory | null;
169function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
170function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
171function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
172// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
173function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
174 let current = offline ? offlineTreeRoot : onlineTreeRoot;
175 if (path === "") return current;
176 const parts = path.split("/");
177
178 for (const part of parts) {
179 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
180 if (!next || "metadata" in next) {
181 if (!create) return null;
182 //if mediafile already exists, we overwrite it with a directory anyway
183 if (offline) next = { files: new Map() };
184 else next = { files: new Map(), status: "Unknown" };
185 current.files.set(part, next);
186 }
187 current = next;
188 }
189 return current;
190}
191
192function deleteInOfflineTree(path: string): void {
193 if (path === "") return;
194 const basePath = path.split("/").slice(0, -1).join("/");
195 const fileName = path.split("/").toReversed()[0];
196
197 const node = getDirInTree(basePath, false, true);
198 if (!node) return;
199
200 node.files.delete(fileName);
201 if (node.files.size === 0) {
202 db.covers.delete(basePath); //remove stored cover for this dir if there are not files left
203 deleteInOfflineTree(basePath);
204 }
205}
206
207export const clearDirectoryCache = async (path: string) => {
208 const node = getDirInTree(path, false, false);
209 if (!node) return;
210
211 // Save existing children before clearing
212 const preservedChildren = new Map(node.files);
213
214 node.files.clear();
215 node.status = "Unknown";
216
217 // restore any preserved subtrees where the parent directory still exist
218 const newListing = await listFiles(path, false, false);
219 for (const [name, child] of preservedChildren.entries()) {
220 if ("files" in child && newListing.has(name)) {
221 node.files.set(name, child);
222 }
223 }
224};
225async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
226 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`, { credentials: "same-origin" }).catch(
227 () => {},
228 );
229 if (response?.status === 404) {
230 toast(`Directory not found: ${dir}`, "error");
231 return;
232 }
233 if (!response || !response.ok) {
234 toast(
235 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
236 "error",
237 );
238 return;
239 }
240
241 return parseMap(await response.text()) as FileListingWithStatus;
242}
243
244function listOfflineFiles(dir: string): FileListing {
245 const node = getDirInTree(dir, false, true);
246 if (!node || !node.files) return new Map();
247 return node.files;
248}
249
250//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
251function fullyKnown(dir: DirectoryWithStatus) {
252 if (dir.status === "Unknown") {
253 return false;
254 }
255 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
256 return false;
257 }
258 return true;
259}
260
261//non-recursive listing should not include nested children
262function pruneChildren(listing: FileListing) {
263 const pruned = new Map();
264 for (const [name, item] of listing.entries()) {
265 if ("metadata" in item) pruned.set(name, item);
266 else pruned.set(name, { files: new Map() });
267 }
268 return pruned;
269}
270
271//used for combining the online and offline results, to be able to show files that only exist locally
272function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
273 const result = new Map(listing1);
274
275 for (const [key, value2] of listing2.entries()) {
276 const value1 = result.get(key);
277
278 if (value1 && "files" in value1 && "files" in value2) {
279 result.set(key, { files: combineFileListings(value1.files, value2.files) });
280 } else {
281 result.set(key, value2);
282 }
283 }
284
285 return result;
286}
287
288// File listing logic with hybrid online/offline support
289// Combines:
290// 1. Direct IndexedDB lookups for offline mode
291// 2. Cached online listings
292// 3. Merged online+offline results
293export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
294 if (offline) {
295 const files = listOfflineFiles(dir);
296 return recursive ? files : pruneChildren(files);
297 }
298 const cached = getDirInTree(dir, false, false);
299 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
300 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
301 return recursive ? combined : pruneChildren(combined);
302 }
303 const files = await listOnlineFiles(dir, recursive);
304 if (!files) return new Map();
305 updateFileTree(dir, { files, status: "Scanned" }, false);
306 const combined = combineFileListings(files, listOfflineFiles(dir));
307 return recursive ? combined : pruneChildren(combined);
308}
309
310const coverCache = new Map<string, { data: Blob | null }>();
311
312//get cover art object-url for a file or directory with cache-first strategy
313export async function getCover(
314 dir: string,
315 options: AppOptions,
316 saveToDb: boolean,
317 offline: boolean,
318 forceRefresh = false,
319) {
320 try {
321 const cached = coverCache.get(dir);
322 if (!forceRefresh && cached !== undefined) {
323 if (saveToDb) db.covers.add({ path: dir, data: cached.data }).catch(() => {});
324 return cached.data === null ? undefined : URL.createObjectURL(cached.data);
325 }
326
327 if (offline) return undefined;
328 const response = await fetch(`/cover/${encodePath(dir)}${options.transcodeCovers ? "?transcode=true" : ""}`, {
329 method: "GET",
330 credentials: "same-origin",
331 }).catch(() => {});
332 if (!response || response.status === 404) {
333 coverCache.set(dir, { data: null });
334 if (saveToDb) db.covers.add({ path: dir, data: null }).catch(() => {});
335 //if cover already exists in database, update it
336 else if (forceRefresh && db.covers.get(dir) !== undefined) db.covers.put({ path: dir, data: null });
337 return undefined;
338 }
339 const blob = await response.blob();
340 coverCache.set(dir, { data: blob });
341 if (saveToDb) db.covers.add({ path: dir, data: blob }).catch(() => {});
342 //if cover already exists in database, update it
343 else if (forceRefresh && db.covers.get(dir) !== undefined) db.covers.put({ path: dir, data: blob });
344 return URL.createObjectURL(blob);
345 } catch {
346 return undefined;
347 }
348}
349
350//returns an object url to the blob if synced, otherwise the server url
351export async function getSrc(
352 path: string,
353 options: AppOptions,
354 offline: boolean,
355 seekTo: number,
356 forceUrl?: true,
357): Promise<{ src: string; isVideo?: boolean }>;
358export async function getSrc(path: string, options: AppOptions, offline: boolean, seekTo: number, forceUrl?: boolean) {
359 if (!forceUrl) {
360 const dbResult = await db.files.get(path);
361 if (dbResult) return { src: URL.createObjectURL(dbResult.data), isVideo: undefined };
362
363 if (offline) {
364 toast("This file is not cached, leave offline mode to play it", "error");
365 return undefined;
366 }
367 }
368 if (options.enableTranscoding) {
369 const isVideo = (
370 (await fetch(joinPath("isVideo", encodePath(path)))
371 .then((res) => res.json())
372 .catch((err) => {
373 console.error(err);
374 return { isVideo: false };
375 })) as IsVideoResponse
376 ).isVideo;
377
378 return {
379 src:
380 `${joinPath("transcode", encodePath(path))}` +
381 `?disableChunkedTranscoding=${options.disableChunkedTranscoding}` +
382 `&container=${isVideo ? options.videoSettings.container : options.audioSettings.container}` +
383 `${isVideo ? `&videoCodec=${options.videoSettings.videoCodec}` : ""}` +
384 `${isVideo ? `&videoBitrate=${options.videoSettings.videoBitrate}` : ""}` +
385 `${isVideo ? `&videoEncodingSetting=${options.videoSettings.videoEncodingSetting}` : ""}` +
386 `&audioCodec=${isVideo ? options.videoSettings.audioCodec : options.audioSettings.audioCodec}` +
387 `&audioBitrate=${isVideo ? options.videoSettings.audioBitrate : options.audioSettings.audioBitrate}` +
388 `${seekTo === 0 ? "" : `&seekTo=${seekTo}`}`,
389 isVideo: isVideo,
390 };
391 }
392 return { src: joinPath("download", encodePath(path)), isVideo: undefined };
393}
394
395export async function loadPlaylists() {
396 return await db.playlists.toArray();
397}
398
399//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
400export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
401 playlist.items = playlist.items.map((item) => {
402 //some less space in the db by omitting ids
403 return { path: item.path, metadata: unproxy(item.metadata) };
404 });
405
406 if (overwrite) {
407 await db.playlists.put(playlist);
408 } else {
409 await db.playlists.add(playlist);
410 }
411}
412
413export async function deletePlaylist(name: string) {
414 await db.playlists.delete(name);
415}
416
417function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
418 const paths: string[] = [];
419
420 for (const [name, item] of listing.entries()) {
421 const currentPath = joinPath(basePath, name);
422
423 if ("metadata" in item) {
424 paths.push(currentPath);
425 } else if ("files" in item) {
426 paths.push(...collectMediaFilePaths(item.files, currentPath));
427 }
428 }
429
430 return paths;
431}
432
433//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
434export async function deleteFile(
435 fullPath: string,
436 item: MediaFile | Directory,
437 fetchFiles: ResourceActions<FlatFileListing>,
438) {
439 try {
440 if ("metadata" in item && downloadStatusMap[fullPath] !== "Synced") return;
441 startProgress(fullPath, { amount: 1 });
442
443 let toDelete: string[];
444 if ("files" in item) {
445 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
446 } else {
447 toDelete = [fullPath];
448 }
449
450 await db.files.bulkDelete(toDelete); //delete file from database
451 await db.covers.bulkDelete(toDelete); //delete embedded cover from database
452 deleteInOfflineTree(fullPath); //delete node from tree
453 batch(() => {
454 for (const path of toDelete) {
455 setDownloadStatusMap(path, undefined); // set all children to unsynced
456 }
457 });
458 fetchFiles.refetch();
459 } finally {
460 stopProgress(fullPath, undefined);
461 }
462}
463
464const downloadSemaphore = new AsyncSemaphore(1);
465
466//Download file to indexedDB and updates the offline file tree
467//concurrency control via the downloadSemaphore
468async function downloadFile(
469 path: string,
470 options: AppOptions,
471 item: MediaFile,
472 fetchFiles: ResourceActions<FlatFileListing>,
473 redownload: boolean,
474): Promise<void> {
475 const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
476
477 const controller = new AbortController();
478 let abortResolve!: () => void;
479 const abortPromise = new Promise<void>((outerResolve) => {
480 abortResolve = () => {
481 controller.abort();
482 stopProgress(
483 path,
484 // if it was already synced, restore it
485 initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
486 );
487 outerResolve();
488 };
489 });
490 // file already downloaded or download already in progress
491 const initialProgress = downloadStatusMap[path];
492 if (
493 (!redownload && initialProgress === "Synced") ||
494 (typeof initialProgress === "object" && "bytes" in initialProgress)
495 )
496 return;
497 startProgress(path, { status: 0, abort: abortResolve });
498
499 try {
500 //options might change at any point, update the semaphore to the current value
501 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
502
503 //make sure to only allow the set amount of parallel downloads
504 const semaphorePromise = downloadSemaphore.acquire();
505 await Promise.race([abortPromise, semaphorePromise]);
506 try {
507 const res = await fetch((await getSrc(path, options, false, 0, true)).src, {
508 signal: controller.signal,
509 credentials: "same-origin",
510 });
511
512 //let the browser do the actual download
513 const blobPromise = res.clone().blob();
514 if (!res.ok || !res.body) throw res.statusText;
515 //we just iterate over the chunks to get the current progress
516 const reader = res.body.getReader();
517 let received = 0;
518
519 let lastUpdate = 0;
520 while (true) {
521 const { done, value } = await reader.read();
522 if (done) break;
523 received += value.length;
524
525 //debouncing logic to not update UI too fast
526 const now = Date.now();
527 if (now - lastUpdate < 250) continue;
528 lastUpdate = now;
529 setDownloadStatusMap(path, { status: received, abort: abortResolve });
530 }
531 const blob = await blobPromise;
532
533 await db.files.put({
534 path,
535 data: blob,
536 metadata: unproxy(item.metadata),
537 });
538
539 updateFileTree(path, item, true);
540 stopProgress(path, "Synced");
541 fetchFiles.refetch();
542
543 //iterate through all parent directories and download their covers to database
544 const parts = path.split("/");
545 while (true) {
546 getCover(parts.join("/"), options, true, false).then((cover) => URL.revokeObjectURL(cover || ""));
547 parts.pop();
548 if (parts.length === 0) break;
549 }
550 } finally {
551 downloadSemaphore.release();
552 }
553 } catch (e) {
554 if (e instanceof Error && e.name === "AbortError") return;
555 stopProgress(
556 path,
557 // if it was already synced, restore it
558 initialProgress === "Synced"
559 ? "Synced"
560 : {
561 status: "Failed",
562 reason: e ? e.toString() : "Unknown error",
563 retry,
564 },
565 );
566 }
567}
568
569export async function download(
570 fullPath: string,
571 item: MediaFile | Directory,
572 options: AppOptions,
573 fetchFiles: ResourceActions<FlatFileListing>,
574 redownload = false,
575) {
576 if ("metadata" in item) {
577 await downloadFile(fullPath, options, item, fetchFiles, redownload || false);
578 } else {
579 updateRecursiveStatus(fullPath, 1, true);
580 try {
581 //start downloads for all children to allow them to run in parallel, then wait for the results
582 const promises: Promise<unknown>[] = [];
583 const listing = await listFiles(fullPath, false, false);
584 batch(() => {
585 for (const [childName, childItem] of listing) {
586 promises.push(download(joinPath(fullPath, childName), childItem, options, fetchFiles, redownload));
587 }
588 });
589 await Promise.allSettled(promises);
590 } finally {
591 updateRecursiveStatus(fullPath, -1, true);
592 }
593 }
594}
595
596export const storedFilesLive = from(
597 liveQuery(async () => await db.files.toCollection().keys()),
598 [],
599);
600