offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import { createSignal, type Accessor, type ResourceActions, type Signal } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14import type { FlatFileListing } from "./App";
15
16interface Progress {
17 bytes: number;
18}
19
20export type DownloadStatus = Progress | "UnspecificProgress" | "Synced" | undefined;
21//used to display download status for directories/files in the filebrowser view
22// export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
23export interface StatusWithChildren {
24 status: Signal<DownloadStatus>;
25 children?: RecursiveStatus;
26}
27
28type RecursiveStatus = Map<string, StatusWithChildren>;
29const downloadStatusMap: RecursiveStatus = new Map<string, StatusWithChildren>();
30
31export function getDownloadStatusWithChildren(path: string) {
32 const parts = path.split("/");
33 let current = downloadStatusMap;
34 //iterate through the basepath
35 for (const part of parts.slice(0, -1)) {
36 let next = current.get(part);
37 if (!next) {
38 next = { status: createSignal<DownloadStatus>(undefined) };
39 current.set(part, next);
40 }
41 if (!next.children) {
42 //might be a directory whose children have not been referenced before, in which case "children" would be undefined
43 next.children = new Map();
44 }
45 current = next.children as RecursiveStatus;
46 }
47
48 const fileName = parts.toReversed()[0];
49 let ret = current.get(fileName);
50 if (!ret) {
51 ret = { status: createSignal<DownloadStatus>(undefined) };
52 current.set(fileName, ret);
53 }
54 return ret;
55}
56
57export function getDownloadStatus(path: string) {
58 return getDownloadStatusWithChildren(path).status;
59}
60
61//initialize filetrees and download statuses from indexedDB
62export async function loadDataFromDB() {
63 try {
64 const db = await openDB();
65 const files = await db.listFiles(FileStore);
66 for (const file of files) {
67 const [, setStatus] = getDownloadStatus(file.path);
68 setStatus("Synced");
69 updateFileTree(file.path, { metadata: file.metadata }, true);
70 }
71 } catch (e) {
72 toast("Loading offline files failed", "error", e);
73 console.error(e);
74 }
75}
76
77//contains all locally synced files
78const offlineTreeRoot: Directory = {
79 files: new Map(),
80};
81//used for caching online file listings
82const onlineTreeRoot: DirectoryWithStatus = {
83 files: new Map(),
84 status: "Unknown",
85};
86
87//merges "Scanned" directories from "other" to "target"
88function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
89 if (other.status === "Unknown") return;
90 target.status = other.status;
91 for (const [fileName, file] of other.files.entries()) {
92 const targetFile = target.files.get(fileName);
93 if (targetFile && "files" in targetFile && "files" in file) {
94 mergeTrees(targetFile, file);
95 } else {
96 target.files.set(fileName, file);
97 }
98 }
99}
100
101//TODO: improve typing to avoid "as"
102//overloads for typecheking
103//offline tree only stores files, directories are created ad-hoc
104function updateFileTree(path: string, file: MediaFile, offline: true): void;
105//online tree only stores cached directory listings
106function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
107// Updates file tree structure with new data (offline or online)
108function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
109 if (offline && path === "") throw new Error("Trying to update root in offline mode");
110 const basePath = path.split("/").slice(0, -1).join("/");
111 const fileName = path.split("/").toReversed()[0];
112 const node = getDirInTree(basePath, true, offline);
113
114 if (offline) {
115 node.files.set(fileName, file);
116 } else {
117 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
118 else {
119 let target = node.files.get(fileName) as DirectoryWithStatus;
120 if (!target) target = { files: new Map(), status: "Unknown" };
121 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
122 node.files.set(fileName, target);
123 }
124 }
125}
126
127//overloads for typecheking
128function getDirInTree(path: string, create: true, offline: true): Directory;
129function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
130function getDirInTree(path: string, create: false, offline: true): Directory | null;
131function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
132function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
133function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
134// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
135function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
136 let current = offline ? offlineTreeRoot : onlineTreeRoot;
137 if (path === "") return current;
138 const parts = path.split("/");
139
140 for (const part of parts) {
141 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
142 if (!next || "metadata" in next) {
143 if (!create) return null;
144 //if mediafile already exists, we overwrite it with a directory anyway
145 if (offline) next = { files: new Map() };
146 else next = { files: new Map(), status: "Unknown" };
147 current.files.set(part, next);
148 }
149 current = next;
150 }
151 return current;
152}
153
154function deleteInOfflineTree(path: string): void {
155 if (path === "") return;
156 const basePath = path.split("/").slice(0, -1).join("/");
157 const fileName = path.split("/").toReversed()[0];
158
159 const node = getDirInTree(basePath, false, true);
160 if (!node) return;
161
162 node.files.delete(fileName);
163 if (node.files.size === 0) deleteInOfflineTree(basePath);
164}
165
166export const clearDirectoryCache = (path: string) => {
167 const node = getDirInTree(path, false, false);
168 if (!node) return;
169 node.files.clear();
170 node.status = "Unknown";
171};
172
173async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
174 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
175 if (response?.status === 404) {
176 toast(`Directory not found: ${dir}`, "error");
177 return;
178 }
179 if (!response || !response.ok) {
180 toast(
181 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
182 "error",
183 );
184 return;
185 }
186
187 return parseMap(await response.text()) as FileListingWithStatus;
188}
189
190function listOfflineFiles(dir: string): FileListing {
191 const node = getDirInTree(dir, false, true);
192 if (!node || !node.files) return new Map();
193 return node.files;
194}
195
196//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
197function fullyKnown(dir: DirectoryWithStatus) {
198 if (dir.status === "Unknown") {
199 return false;
200 }
201 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
202 return false;
203 }
204 return true;
205}
206
207//non-recursive listing should not include nested children
208function pruneChildren(listing: FileListing) {
209 const pruned = new Map();
210 for (const [name, item] of listing.entries()) {
211 if ("metadata" in item) pruned.set(name, item);
212 else pruned.set(name, { files: new Map() });
213 }
214 return pruned;
215}
216
217//used for combining the online and offline results, to be able to show files that only exist locally
218function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
219 const result = new Map(listing1);
220
221 for (const [key, value2] of listing2.entries()) {
222 const value1 = result.get(key);
223
224 if (value1 && "files" in value1 && "files" in value2) {
225 result.set(key, { files: combineFileListings(value1.files, value2.files) });
226 } else {
227 result.set(key, value2);
228 }
229 }
230
231 return result;
232}
233
234// File listing logic with hybrid online/offline support
235// Combines:
236// 1. Direct IndexedDB lookups for offline mode
237// 2. Cached online listings
238// 3. Merged online+offline results
239export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
240 if (offline) {
241 const files = listOfflineFiles(dir);
242 return recursive ? files : pruneChildren(files);
243 }
244 const cached = getDirInTree(dir, false, false);
245 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
246 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
247 return recursive ? combined : pruneChildren(combined);
248 }
249 const files = await listOnlineFiles(dir, recursive);
250 if (!files) return new Map();
251 updateFileTree(dir, { files, status: "Scanned" }, false);
252 const combined = combineFileListings(files, listOfflineFiles(dir));
253 return recursive ? combined : pruneChildren(combined);
254}
255
256//get cover art for a directory with cache-first strategy
257export async function getCover(dir: string, offline: boolean) {
258 const db = await openDB();
259 const dbResult = await db.get(CoverStore, dir);
260 if (dbResult !== undefined) {
261 const cached = dbResult as StoredCover;
262 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
263 }
264
265 if (offline) return undefined;
266
267 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
268 if (!response || response.status === 404) {
269 db.put(CoverStore, { path: dir, data: null });
270 return undefined;
271 }
272 const blob = await response.blob();
273 db.put(CoverStore, { path: dir, data: blob });
274 return URL.createObjectURL(blob);
275}
276
277export const clearCoverCache = async (path: string) => {
278 const db = await openDB();
279 return db.delete(CoverStore, path);
280};
281
282//returns an object url to the blob if synced, otherwise the server url
283export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
284export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
285 if (!forceUrl) {
286 const db = await openDB();
287 const dbResult = (await db.get(FileStore, path)) as StoredFile;
288 if (dbResult) return URL.createObjectURL(dbResult.data);
289
290 if (offline) {
291 toast("This file is not cached, leave offline mode to play it", "error");
292 return undefined;
293 }
294 }
295 if (options.enableTranscoding) {
296 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
297 }
298 return joinPath("/download", encodePath(path));
299}
300
301export async function loadPlaylists() {
302 const db = await openDB();
303 return await db.listPlaylists(PlaylistStore);
304}
305
306//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
307export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
308 const db = await openDB();
309 playlist.items = playlist.items.map((item) => {
310 //some less space in the db by omitting ids
311 const { id, ...itemWithoutId } = item as PlaylistItem;
312 return itemWithoutId;
313 });
314 if (overwrite) await db.put(PlaylistStore, playlist);
315 else await db.add(PlaylistStore, playlist);
316}
317
318export async function deletePlaylist(name: string) {
319 const db = await openDB();
320 await db.delete(PlaylistStore, name);
321}
322
323function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
324 const paths: string[] = [];
325
326 for (const [name, item] of listing.entries()) {
327 const currentPath = joinPath(basePath, name);
328
329 if ("metadata" in item) {
330 paths.push(currentPath);
331 } else if ("files" in item) {
332 paths.push(...collectMediaFilePaths(item.files, currentPath));
333 }
334 }
335
336 return paths;
337}
338
339//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
340export async function deleteFile(
341 fileName: string,
342 item: MediaFile | Directory,
343 basePath: string,
344 fetchFiles: ResourceActions<FlatFileListing>,
345) {
346 const fullPath = joinPath(basePath, fileName);
347 const [getStatus, setStatus] = getDownloadStatus(fullPath);
348 try {
349 if ("metadata" in item && getStatus() !== "Synced") return;
350 setStatus("UnspecificProgress");
351
352 let toDelete: string[];
353 if ("files" in item) {
354 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
355 } else {
356 toDelete = [fullPath];
357 }
358
359 const db = await openDB();
360 await db.deleteAll(FileStore, toDelete);
361 deleteInOfflineTree(fullPath);
362 for (const path of toDelete) {
363 getDownloadStatus(path)[1](undefined);
364 }
365 fetchFiles.refetch();
366 } finally {
367 setStatus(undefined);
368 }
369}
370
371const downloadSemaphore = new AsyncSemaphore(1);
372
373//Download file to indexedDB and updates the offline file tree
374//concurrency control via the downloadSemaphore
375async function downloadFile(
376 path: string,
377 currentDir: Accessor<string>,
378 options: AppOptions,
379 item: MediaFile,
380 fetchFiles: ResourceActions<FlatFileListing>,
381): Promise<void> {
382 const [getStatus, setStatus] = getDownloadStatus(path);
383 const initialProgress = getStatus();
384 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
385
386 try {
387 setStatus({ bytes: 0 });
388 const db = await openDB();
389 let mimeType: string | undefined;
390
391 //options might change at any point, update the semaphore to the current value
392 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
393 let blob: Blob;
394 try {
395 //make sure to only allow the set amount of parallel downloads
396 await downloadSemaphore.acquire();
397 const res = await fetch(await getSrc(path, options, false, true));
398
399 //let the browser do the actual download
400 const blobPromise = res.clone().blob();
401 if (!res.ok || !res.body) throw res.statusText;
402 //we just iterate over the chunks to get the current progress
403 const reader = res.body.getReader();
404 mimeType = res.headers.get("content-type") ?? undefined;
405 let received = 0;
406
407 let lastUpdate = 0;
408 while (true) {
409 const { done, value } = await reader.read();
410 if (done) break;
411 received += value.length;
412
413 //debouncing logic to not update UI too fast
414 const now = Date.now();
415 if (now - lastUpdate < 250) continue;
416 lastUpdate = now;
417 setStatus({ bytes: received });
418 }
419 blob = await blobPromise;
420 } finally {
421 downloadSemaphore.release();
422 }
423 await db.put(FileStore, {
424 path,
425 data: blob,
426 metadata: { ...item.metadata }, //spread needed, because item might be a proxy object from the filebrowser, which can't be stored
427 } as StoredFile);
428 updateFileTree(path, item, true);
429 setStatus("Synced");
430 fetchFiles.refetch();
431 } catch (e) {
432 toast(`Downloading ${path} failed`, "error", e);
433 //restore original status
434 setStatus(initialProgress === "Synced" ? "Synced" : undefined);
435 }
436}
437
438export async function download(
439 fileName: string,
440 item: MediaFile | Directory,
441 basePath: string,
442 currentDir: Accessor<string>,
443 options: AppOptions,
444 fetchFiles: ResourceActions<FlatFileListing>,
445) {
446 const fullPath = joinPath(basePath, fileName);
447 if ("metadata" in item) {
448 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
449 } else {
450 await getCover(fullPath, false);
451 //start downloads for all children to allow them to run in parallel, then wait for the results
452 const promises: Promise<unknown>[] = [];
453 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
454 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
455 }
456 await Promise.allSettled(promises);
457 }
458}
459
460export async function getStoredFiles() {
461 const db = await openDB();
462 return await db.getKeys(FileStore);
463}
464
465interface StoredFile {
466 path: string;
467 data: Blob;
468 metadata: {
469 duration: number;
470 };
471}
472
473interface StoredCover {
474 path: string;
475 data: Blob | null;
476}
477
478const FileStore = "fileblobs";
479const CoverStore = "covers";
480const PlaylistStore = "playlists";
481
482async function openDB() {
483 return new Promise<DatabaseWrapper>((resolve, reject) => {
484 const req = indexedDB.open("music-files");
485 req.onerror = () => reject(req.error);
486 req.onupgradeneeded = () => {
487 const db = req.result;
488 if (!db.objectStoreNames.contains(FileStore))
489 db.createObjectStore(FileStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
490 if (!db.objectStoreNames.contains(CoverStore))
491 db.createObjectStore(CoverStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
492 if (!db.objectStoreNames.contains(PlaylistStore))
493 db.createObjectStore(PlaylistStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
494 };
495 req.onsuccess = () => resolve(wrapDB(req.result));
496 });
497}
498
499interface DatabaseWrapper {
500 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
501 put(store: typeof FileStore, value: StoredFile): Promise<void>;
502 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
503 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
504 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
505 delete(store: typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
506 deleteAll(store: typeof FileStore, keys: string[]): Promise<void>;
507 clear(store: string): Promise<void>;
508 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
509 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
510 getKeys(store: typeof FileStore): Promise<string[]>;
511}
512
513function wrapDB(db: IDBDatabase): DatabaseWrapper {
514 return {
515 get: (store: string, key: string) =>
516 new Promise((res, rej) => {
517 const tx = db.transaction(store, "readonly");
518 const req = tx.objectStore(store).get(key);
519 req.onsuccess = () => res(req.result);
520 req.onerror = () => rej(req.error);
521 }),
522 put: (store: string, value: unknown) =>
523 new Promise<void>((res, rej) => {
524 const tx = db.transaction(store, "readwrite");
525 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
526 req.onsuccess = () => res();
527 req.onerror = () => rej(req.error);
528 }),
529 add: (store: string, value: unknown) =>
530 new Promise<void>((res, rej) => {
531 const tx = db.transaction(store, "readwrite");
532 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
533 req.onsuccess = () => res();
534 req.onerror = () => rej(req.error);
535 }),
536 delete: (store: string, key: string) =>
537 new Promise<void>((res, rej) => {
538 const tx = db.transaction(store, "readwrite");
539 const req = tx.objectStore(store).delete(key);
540 req.onsuccess = () => res();
541 req.onerror = () => rej(req.error);
542 }),
543 deleteAll: (store: typeof FileStore, keys: string[]) =>
544 new Promise<void>((res, rej) => {
545 const tx = db.transaction(store, "readwrite");
546 const objectStore = tx.objectStore(store);
547 for (const key of keys) {
548 objectStore.delete(key);
549 }
550 tx.oncomplete = () => res();
551 tx.onerror = () => rej(tx.error);
552 }),
553 clear: (store: string) =>
554 new Promise<void>((res, rej) => {
555 const tx = db.transaction(store, "readwrite");
556 const req = tx.objectStore(store).clear();
557 req.onsuccess = () => res();
558 req.onerror = () => rej(req.error);
559 }),
560 listFiles: (store: typeof FileStore) =>
561 new Promise((res, rej) => {
562 const tx = db.transaction(store, "readonly");
563 const req = tx.objectStore(store).getAll();
564 req.onsuccess = () => res(req.result);
565 req.onerror = () => rej(req.error);
566 }),
567 listPlaylists: (store: typeof PlaylistStore) =>
568 new Promise((res, rej) => {
569 const tx = db.transaction(store, "readonly");
570 const req = tx.objectStore(store).getAll();
571 req.onsuccess = () => res(req.result);
572 req.onerror = () => rej(req.error);
573 }),
574 getKeys: (store: typeof FileStore) =>
575 new Promise((res, rej) => {
576 const tx = db.transaction(store, "readonly");
577 const req = tx.objectStore(store).getAllKeys();
578 req.onsuccess = () => res(req.result as string[]);
579 req.onerror = () => rej(req.error);
580 }),
581 };
582}
583