offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import { createSignal, type Accessor, type ResourceActions, type Signal } from "solid-js";
12import { AsyncSemaphore } from "./semaphore";
13import { encodePath } from "music-server-shared/utils";
14
15interface Progress {
16 bytes: number;
17}
18
19export type DownloadStatus = Progress | "UnspecificProgress" | "Synced" | undefined;
20//used to display download status for directories/files in the filebrowser view
21// export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
22interface StatusWithChildren {
23 status: Signal<DownloadStatus>;
24 children?: RecursiveStatus;
25}
26
27type RecursiveStatus = Map<string, StatusWithChildren>;
28const downloadStatusMap: RecursiveStatus = new Map<string, StatusWithChildren>();
29
30export function getDownloadStatus(path: string) {
31 const parts = path.split("/");
32 let current = downloadStatusMap;
33 //iterate through the basepath
34 for (const part of parts.slice(0, -1)) {
35 let next = current.get(part);
36 if (!next) {
37 next = { status: createSignal<DownloadStatus>(undefined) };
38 current.set(part, next);
39 }
40 if (!next.children) {
41 //might be a directory whose children have not been referenced before, in which case "children" would be undefined
42 next.children = new Map();
43 }
44 current = next.children as RecursiveStatus;
45 }
46
47 const fileName = parts.toReversed()[0];
48 let ret = current.get(fileName);
49 if (!ret) {
50 ret = { status: createSignal<DownloadStatus>(undefined) };
51 current.set(fileName, ret);
52 }
53 return ret.status;
54}
55
56//initialize filetrees and download statuses from indexedDB
57export async function loadDataFromDB() {
58 try {
59 const db = await openDB();
60 const files = await db.listFiles(FileStore);
61 for (const file of files) {
62 const [, setStatus] = getDownloadStatus(file.path);
63 setStatus("Synced");
64 updateFileTree(file.path, { metadata: file.metadata }, true);
65 }
66 } catch (e) {
67 toast("Loading offline files failed", "error", e);
68 console.error(e);
69 }
70}
71
72//contains all locally synced files
73const offlineTreeRoot: Directory = {
74 files: new Map(),
75};
76//used for caching online file listings
77const onlineTreeRoot: DirectoryWithStatus = {
78 files: new Map(),
79 status: "Unknown",
80};
81
82//merges "Scanned" directories from "other" to "target"
83function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
84 if (other.status === "Unknown") return;
85 target.status = other.status;
86 for (const [fileName, file] of other.files.entries()) {
87 const targetFile = target.files.get(fileName);
88 if (targetFile && "files" in targetFile && "files" in file) {
89 mergeTrees(targetFile, file);
90 } else {
91 target.files.set(fileName, file);
92 }
93 }
94}
95
96//TODO: improve typing to avoid "as"
97//overloads for typecheking
98//offline tree only stores files, directories are created ad-hoc
99function updateFileTree(path: string, file: MediaFile, offline: true): void;
100//online tree only stores cached directory listings
101function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
102// Updates file tree structure with new data (offline or online)
103function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
104 if (offline && path === "") throw new Error("Trying to update root in offline mode");
105 const basePath = path.split("/").slice(0, -1).join("/");
106 const fileName = path.split("/").toReversed()[0];
107 const node = getDirInTree(basePath, true, offline);
108
109 if (offline) {
110 node.files.set(fileName, file);
111 } else {
112 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
113 else {
114 let target = node.files.get(fileName) as DirectoryWithStatus;
115 if (!target) target = { files: new Map(), status: "Unknown" };
116 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
117 node.files.set(fileName, target);
118 }
119 }
120}
121
122//overloads for typecheking
123function getDirInTree(path: string, create: true, offline: true): Directory;
124function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
125function getDirInTree(path: string, create: false, offline: true): Directory | null;
126function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
127function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
128function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
129// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
130function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
131 let current = offline ? offlineTreeRoot : onlineTreeRoot;
132 if (path === "") return current;
133 const parts = path.split("/");
134
135 for (const part of parts) {
136 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
137 if (!next || "metadata" in next) {
138 if (!create) return null;
139 //if mediafile already exists, we overwrite it with a directory anyway
140 if (offline) next = { files: new Map() };
141 else next = { files: new Map(), status: "Unknown" };
142 current.files.set(part, next);
143 }
144 current = next;
145 }
146 return current;
147}
148
149function deleteOfflinePath(path: string): void {
150 const basePath = path.split("/").slice(0, -1).join("/");
151 const fileName = path.split("/").toReversed()[0];
152
153 const node = getDirInTree(basePath, false, true);
154 if (!node) return;
155
156 node.files.delete(fileName);
157 if (node.files.size === 0) deleteOfflinePath(basePath);
158}
159
160export const clearDirectoryCache = (path: string) => {
161 const node = getDirInTree(path, false, false);
162 if (!node) return;
163 node.files.clear();
164 node.status = "Unknown";
165};
166
167async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
168 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
169 if (response?.status === 404) {
170 toast(`Directory not found: ${dir}`, "error");
171 return;
172 }
173 if (!response || !response.ok) {
174 toast(
175 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
176 "error",
177 );
178 return;
179 }
180
181 return parseMap(await response.text()) as FileListingWithStatus;
182}
183
184function listOfflineFiles(dir: string): FileListing {
185 const node = getDirInTree(dir, false, true);
186 if (!node || !node.files) return new Map();
187 return node.files;
188}
189
190//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
191function fullyKnown(dir: DirectoryWithStatus) {
192 if (dir.status === "Unknown") {
193 return false;
194 }
195 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
196 return false;
197 }
198 return true;
199}
200
201//non-recursive listing should not include nested children
202function pruneChildren(listing: FileListing) {
203 const cloned = structuredClone(listing);
204 for (const [, item] of cloned.entries()) {
205 if ("files" in item) item.files.clear();
206 }
207 return cloned;
208}
209
210//used for combining the online and offline results, to be able to show files that only exist locally
211function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
212 const result = new Map(listing1);
213
214 for (const [key, value2] of listing2.entries()) {
215 const value1 = result.get(key);
216
217 if (value1 && "files" in value1 && "files" in value2) {
218 result.set(key, { files: combineFileListings(value1.files, value2.files) });
219 } else {
220 result.set(key, value2);
221 }
222 }
223
224 return result;
225}
226
227// File listing logic with hybrid online/offline support
228// Combines:
229// 1. Direct IndexedDB lookups for offline mode
230// 2. Cached online listings
231// 3. Merged online+offline results
232export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
233 if (offline) {
234 const files = listOfflineFiles(dir);
235 return recursive ? files : pruneChildren(files);
236 }
237 const cached = getDirInTree(dir, false, false);
238 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
239 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
240 return recursive ? combined : pruneChildren(combined);
241 }
242 const files = await listOnlineFiles(dir, recursive);
243 if (!files) return new Map();
244 updateFileTree(dir, { files, status: "Scanned" }, false);
245 const combined = combineFileListings(files, listOfflineFiles(dir));
246 return recursive ? combined : pruneChildren(combined);
247}
248
249//get cover art for a directory with cache-first strategy
250export async function getCover(dir: string, offline: boolean) {
251 const db = await openDB();
252 const dbResult = await db.get(CoverStore, dir);
253 if (dbResult !== undefined) {
254 const cached = dbResult as StoredCover;
255 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
256 }
257
258 if (offline) return undefined;
259
260 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
261 if (!response || response.status === 404) {
262 db.put(CoverStore, { path: dir, data: null });
263 return undefined;
264 }
265 const blob = await response.blob();
266 db.put(CoverStore, { path: dir, data: blob });
267 return URL.createObjectURL(blob);
268}
269
270export const clearCoverCache = async (path: string) => {
271 const db = await openDB();
272 return db.delete(CoverStore, path);
273};
274
275//returns an object url to the blob if synced, otherwise the server url
276export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
277export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
278 if (!forceUrl) {
279 const db = await openDB();
280 const dbResult = (await db.get(FileStore, path)) as StoredFile;
281 if (dbResult) return URL.createObjectURL(dbResult.data);
282
283 if (offline) {
284 toast("This file is not cached, leave offline mode to play it", "error");
285 return undefined;
286 }
287 }
288 if (options.enableTranscoding) {
289 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
290 }
291 return joinPath("/download", encodePath(path));
292}
293
294export async function loadPlaylists() {
295 const db = await openDB();
296 return await db.listPlaylists(PlaylistStore);
297}
298
299//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
300export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
301 const db = await openDB();
302 playlist.items = playlist.items.map((item) => {
303 //some less space in the db by omitting ids
304 const { id, ...itemWithoutId } = item as PlaylistItem;
305 return itemWithoutId;
306 });
307 if (overwrite) await db.put(PlaylistStore, playlist);
308 else await db.add(PlaylistStore, playlist);
309}
310
311export async function deletePlaylist(name: string) {
312 const db = await openDB();
313 await db.delete(PlaylistStore, name);
314}
315
316function collectMediaFilePaths(listing: FileListing, basePath: string): string[] {
317 const paths: string[] = [];
318
319 for (const [name, item] of listing.entries()) {
320 const currentPath = joinPath(basePath, name);
321
322 if ("metadata" in item) {
323 paths.push(currentPath);
324 } else if ("files" in item) {
325 paths.push(...collectMediaFilePaths(item.files, currentPath));
326 }
327 }
328
329 return paths;
330}
331
332//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
333export async function deleteFile(
334 fileName: string,
335 item: MediaFile | Directory,
336 basePath: string,
337 fetchFiles: ResourceActions<FileListing>,
338) {
339 const fullPath = joinPath(basePath, fileName);
340 const [getStatus, setStatus] = getDownloadStatus(fullPath);
341 try {
342 if ("metadata" in item && getStatus() !== "Synced") return;
343 setStatus("UnspecificProgress");
344
345 let toDelete: string[];
346 if ("files" in item) {
347 toDelete = collectMediaFilePaths(listOfflineFiles(fullPath), fullPath);
348 } else {
349 toDelete = [fullPath];
350 }
351
352 const db = await openDB();
353 await db.deleteAll(FileStore, toDelete);
354 deleteOfflinePath(fullPath);
355 for (const path of toDelete) {
356 setStatus(undefined);
357 }
358 fetchFiles.refetch();
359 } finally {
360 setStatus(undefined);
361 }
362}
363
364const downloadSemaphore = new AsyncSemaphore(1);
365
366//Download file to indexedDB and updates the offline file tree
367//concurrency control via the downloadSemaphore
368async function downloadFile(
369 path: string,
370 currentDir: Accessor<string>,
371 options: AppOptions,
372 item: MediaFile,
373 fetchFiles: ResourceActions<FileListing>,
374): Promise<void> {
375 const [getStatus, setStatus] = getDownloadStatus(path);
376 const initialProgress = getStatus();
377 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
378
379 try {
380 setStatus({ bytes: 0 });
381 const db = await openDB();
382 let mimeType: string | undefined;
383
384 //options might change at any point, update the semaphore to the current value
385 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
386 let blob: Blob;
387 try {
388 //make sure to only allow the set amount of parallel downloads
389 await downloadSemaphore.acquire();
390 const res = await fetch(await getSrc(path, options, false, true));
391
392 //let the browser do the actual download
393 const blobPromise = res.clone().blob();
394 if (!res.ok || !res.body) throw res.statusText;
395 //we just iterate over the chunks to get the current progress
396 const reader = res.body.getReader();
397 mimeType = res.headers.get("content-type") ?? undefined;
398 let received = 0;
399
400 let lastUpdate = 0;
401 while (true) {
402 const { done, value } = await reader.read();
403 if (done) break;
404 received += value.length;
405
406 //debouncing logic to not update UI too fast
407 const now = Date.now();
408 if (now - lastUpdate < 250) continue;
409 lastUpdate = now;
410 setStatus({ bytes: received });
411 }
412 blob = await blobPromise;
413 } finally {
414 downloadSemaphore.release();
415 }
416
417 await db.put(FileStore, {
418 path,
419 data: blob,
420 metadata: item.metadata,
421 } as StoredFile);
422 updateFileTree(path, item, true);
423 setStatus("Synced");
424 const basePath = path.split("/").slice(0, -1).join("/");
425 const fileName = path.split("/").toReversed()[0];
426 //add files to listing if the containing directory is currently shown
427 if (currentDir() === basePath) {
428 //directly mutate files to not cause unnecessary rerenders
429 fetchFiles.mutate((prev) => {
430 prev.set(fileName, item);
431 //new map to force rerender
432 return new Map(prev);
433 });
434 }
435 } catch (e) {
436 toast(`Downloading ${path} failed`, "error", e);
437 //restore original status
438 setStatus(initialProgress === "Synced" ? "Synced" : undefined);
439 }
440}
441
442export async function download(
443 fileName: string,
444 item: MediaFile | Directory,
445 basePath: string,
446 currentDir: Accessor<string>,
447 options: AppOptions,
448 fetchFiles: ResourceActions<FileListing>,
449) {
450 const fullPath = joinPath(basePath, fileName);
451 if ("metadata" in item) {
452 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
453 } else {
454 const [, setStatus] = getDownloadStatus(fullPath);
455 setStatus("UnspecificProgress");
456 try {
457 await getCover(fullPath, false);
458 //start downloads for all children to allow them to run in parallel, then wait for the results
459 const promises: Promise<unknown>[] = [];
460 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
461 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
462 }
463 await Promise.allSettled(promises);
464 } finally {
465 //always set to undefined afterwards to show the normal sync icon, since we don't know for sure if the directory is completely synced
466 setStatus(undefined);
467 }
468 }
469}
470
471export async function getStoredFiles() {
472 const db = await openDB();
473 return await db.getKeys(FileStore);
474}
475
476interface StoredFile {
477 path: string;
478 data: Blob;
479 metadata: {
480 duration: number;
481 };
482}
483
484interface StoredCover {
485 path: string;
486 data: Blob | null;
487}
488
489const FileStore = "fileblobs";
490const CoverStore = "covers";
491const PlaylistStore = "playlists";
492
493async function openDB() {
494 return new Promise<DatabaseWrapper>((resolve, reject) => {
495 const req = indexedDB.open("music-files");
496 req.onerror = () => reject(req.error);
497 req.onupgradeneeded = () => {
498 const db = req.result;
499 if (!db.objectStoreNames.contains(FileStore))
500 db.createObjectStore(FileStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
501 if (!db.objectStoreNames.contains(CoverStore))
502 db.createObjectStore(CoverStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
503 if (!db.objectStoreNames.contains(PlaylistStore))
504 db.createObjectStore(PlaylistStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
505 };
506 req.onsuccess = () => resolve(wrapDB(req.result));
507 });
508}
509
510interface DatabaseWrapper {
511 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
512 put(store: typeof FileStore, value: StoredFile): Promise<void>;
513 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
514 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
515 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
516 delete(store: typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
517 deleteAll(store: typeof FileStore, keys: string[]): Promise<void>;
518 clear(store: string): Promise<void>;
519 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
520 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
521 getKeys(store: typeof FileStore): Promise<string[]>;
522}
523
524function wrapDB(db: IDBDatabase): DatabaseWrapper {
525 return {
526 get: (store: string, key: string) =>
527 new Promise((res, rej) => {
528 const tx = db.transaction(store, "readonly");
529 const req = tx.objectStore(store).get(key);
530 req.onsuccess = () => res(req.result);
531 req.onerror = () => rej(req.error);
532 }),
533 put: (store: string, value: unknown) =>
534 new Promise<void>((res, rej) => {
535 const tx = db.transaction(store, "readwrite");
536 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
537 req.onsuccess = () => res();
538 req.onerror = () => rej(req.error);
539 }),
540 add: (store: string, value: unknown) =>
541 new Promise<void>((res, rej) => {
542 const tx = db.transaction(store, "readwrite");
543 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
544 req.onsuccess = () => res();
545 req.onerror = () => rej(req.error);
546 }),
547 delete: (store: string, key: string) =>
548 new Promise<void>((res, rej) => {
549 const tx = db.transaction(store, "readwrite");
550 const req = tx.objectStore(store).delete(key);
551 req.onsuccess = () => res();
552 req.onerror = () => rej(req.error);
553 }),
554 deleteAll: (store: typeof FileStore, keys: string[]) =>
555 new Promise<void>((res, rej) => {
556 const tx = db.transaction(store, "readwrite");
557 const objectStore = tx.objectStore(store);
558 for (const key of keys) {
559 objectStore.delete(key);
560 }
561 tx.oncomplete = () => res();
562 tx.onerror = () => rej(tx.error);
563 }),
564 clear: (store: string) =>
565 new Promise<void>((res, rej) => {
566 const tx = db.transaction(store, "readwrite");
567 const req = tx.objectStore(store).clear();
568 req.onsuccess = () => res();
569 req.onerror = () => rej(req.error);
570 }),
571 listFiles: (store: typeof FileStore) =>
572 new Promise((res, rej) => {
573 const tx = db.transaction(store, "readonly");
574 const req = tx.objectStore(store).getAll();
575 req.onsuccess = () => res(req.result);
576 req.onerror = () => rej(req.error);
577 }),
578 listPlaylists: (store: typeof PlaylistStore) =>
579 new Promise((res, rej) => {
580 const tx = db.transaction(store, "readonly");
581 const req = tx.objectStore(store).getAll();
582 req.onsuccess = () => res(req.result);
583 req.onerror = () => rej(req.error);
584 }),
585 getKeys: (store: typeof FileStore) =>
586 new Promise((res, rej) => {
587 const tx = db.transaction(store, "readonly");
588 const req = tx.objectStore(store).getAllKeys();
589 req.onsuccess = () => res(req.result as string[]);
590 req.onerror = () => rej(req.error);
591 }),
592 };
593}
594