offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import type { Accessor, ResourceActions } from "solid-js";
12import { createStore } from "solid-js/store";
13import { AsyncSemaphore } from "./semaphore";
14import { encodePath } from "music-server-shared/utils";
15
16interface Progress {
17 bytes: number;
18}
19
20export type DownloadStatus = Progress | "DirectoryProgress" | "Synced" | undefined;
21//used to display download status for directories/files in the filebrowser view
22export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
23
24//initialize filetrees and download statuses from indexedDB
25export async function loadDataFromDB() {
26 try {
27 const db = await openDB();
28 const files = await db.listFiles(FileStore);
29 for (const file of files) {
30 setDownloadStatusMap(file.path, "Synced");
31 updateFileTree(file.path, { metadata: file.metadata }, true);
32 }
33 } catch (e) {
34 toast("Loading offline files failed", "error", e);
35 console.error(e);
36 }
37}
38
39//contains all locally synced files
40const offlineTreeRoot: Directory = {
41 files: new Map(),
42};
43//used for caching online file listings
44const onlineTreeRoot: DirectoryWithStatus = {
45 files: new Map(),
46 status: "Unknown",
47};
48
49//merges "Scanned" directories from "other" to "target"
50function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
51 if (other.status === "Unknown") return;
52 target.status = other.status;
53 for (const [fileName, file] of other.files.entries()) {
54 const targetFile = target.files.get(fileName);
55 if (targetFile && "files" in targetFile && "files" in file) {
56 mergeTrees(targetFile, file);
57 } else {
58 target.files.set(fileName, file);
59 }
60 }
61}
62
63//TODO: improve typing to avoid "as"
64//overloads for typecheking
65//offline tree only stores files, directories are created ad-hoc
66function updateFileTree(path: string, file: MediaFile, offline: true): void;
67//online tree only stores cached directory listings
68function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
69// Updates file tree structure with new data (offline or online)
70function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
71 if (offline && path === "") throw new Error("Trying to update root in offline mode");
72 const basePath = path.split("/").slice(0, -1).join("/");
73 const fileName = path.split("/").toReversed()[0];
74 const node = getDirInTree(basePath, true, offline);
75
76 if (offline) {
77 node.files.set(fileName, file);
78 } else {
79 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
80 else {
81 let target = node.files.get(fileName) as DirectoryWithStatus;
82 if (!target) target = { files: new Map(), status: "Unknown" };
83 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
84 node.files.set(fileName, target);
85 }
86 }
87}
88
89//overloads for typecheking
90function getDirInTree(path: string, create: true, offline: true): Directory;
91function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
92function getDirInTree(path: string, create: false, offline: true): Directory | null;
93function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
94function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
95function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
96// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
97function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
98 let current = offline ? offlineTreeRoot : onlineTreeRoot;
99 if (path === "") return current;
100 const parts = path.split("/");
101
102 for (const part of parts) {
103 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
104 if (!next || "metadata" in next) {
105 if (!create) return null;
106 //if mediafile already exists, we overwrite it with a directory anyway
107 if (offline) next = { files: new Map() };
108 else next = { files: new Map(), status: "Unknown" };
109 current.files.set(part, next);
110 }
111 current = next;
112 }
113 return current;
114}
115
116function deleteOfflinePath(path: string): void {
117 const basePath = path.split("/").slice(0, -1).join("/");
118 const fileName = path.split("/").toReversed()[0];
119
120 const node = getDirInTree(basePath, false, true);
121 if (!node) return;
122
123 node.files.delete(fileName);
124 if (node.files.size === 0) deleteOfflinePath(basePath);
125}
126
127export const clearDirectoryCache = (path: string) => {
128 const node = getDirInTree(path, false, false);
129 if (!node) return;
130 node.files.clear();
131 node.status = "Unknown";
132};
133
134async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
135 const response = await fetch(`/list/${encodePath(dir)}?recursive=${recursive}`).catch(() => {});
136 if (response?.status === 404) {
137 toast(`Directory not found: ${dir}`, "error");
138 return;
139 }
140 if (!response || !response.ok) {
141 toast(
142 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
143 "error",
144 );
145 return;
146 }
147
148 return parseMap(await response.text()) as FileListingWithStatus;
149}
150
151function listOfflineFiles(dir: string): FileListing {
152 const node = getDirInTree(dir, false, true);
153 if (!node || !node.files) return new Map();
154 return node.files;
155}
156
157//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
158function fullyKnown(dir: DirectoryWithStatus) {
159 if (dir.status === "Unknown") {
160 return false;
161 }
162 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
163 return false;
164 }
165 return true;
166}
167
168//non-recursive listing should not include nested children
169function pruneChildren(listing: FileListing) {
170 const cloned = structuredClone(listing);
171 for (const [, item] of cloned.entries()) {
172 if ("files" in item) item.files.clear();
173 }
174 return cloned;
175}
176
177//used for combining the online and offline results, to be able to show files that only exist locally
178function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
179 const result = new Map(listing1);
180
181 for (const [key, value2] of listing2.entries()) {
182 const value1 = result.get(key);
183
184 if (value1 && "files" in value1 && "files" in value2) {
185 result.set(key, { files: combineFileListings(value1.files, value2.files) });
186 } else {
187 result.set(key, value2);
188 }
189 }
190
191 return result;
192}
193
194// File listing logic with hybrid online/offline support
195// Combines:
196// 1. Direct IndexedDB lookups for offline mode
197// 2. Cached online listings
198// 3. Merged online+offline results
199export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
200 if (offline) {
201 const files = listOfflineFiles(dir);
202 return recursive ? files : pruneChildren(files);
203 }
204 const cached = getDirInTree(dir, false, false);
205 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
206 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
207 return recursive ? combined : pruneChildren(combined);
208 }
209 const files = await listOnlineFiles(dir, recursive);
210 if (!files) return new Map();
211 updateFileTree(dir, { files, status: "Scanned" }, false);
212 const combined = combineFileListings(files, listOfflineFiles(dir));
213 return recursive ? combined : pruneChildren(combined);
214}
215
216//get cover art for a directory with cache-first strategy
217export async function getCover(dir: string, offline: boolean) {
218 const db = await openDB();
219 const dbResult = await db.get(CoverStore, dir);
220 if (dbResult !== undefined) {
221 const cached = dbResult as StoredCover;
222 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
223 }
224
225 if (offline) return undefined;
226
227 const response = await fetch(`/cover/${encodePath(dir)}`, { method: "GET" }).catch(() => {});
228 if (!response || response.status === 404) {
229 db.put(CoverStore, { path: dir, data: null });
230 return undefined;
231 }
232 const blob = await response.blob();
233 db.put(CoverStore, { path: dir, data: blob });
234 return URL.createObjectURL(blob);
235}
236
237export const clearCoverCache = async (path: string) => {
238 const db = await openDB();
239 return db.delete(CoverStore, path);
240};
241
242//returns an object url to the blob if synced, otherwise the server url
243export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: true): Promise<string>;
244export async function getSrc(path: string, options: AppOptions, offline: boolean, forceUrl?: boolean) {
245 if (!forceUrl) {
246 const db = await openDB();
247 const dbResult = (await db.get(FileStore, path)) as StoredFile;
248 if (dbResult) return URL.createObjectURL(dbResult.data);
249
250 if (offline) {
251 toast("This file is not cached, leave offline mode to play it", "error");
252 return undefined;
253 }
254 }
255 if (options.enableTranscoding) {
256 return `${joinPath("/transcode", encodePath(path))}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
257 }
258 return joinPath("/download", encodePath(path));
259}
260
261export async function loadPlaylists() {
262 const db = await openDB();
263 return await db.listPlaylists(PlaylistStore);
264}
265
266//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
267export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
268 const db = await openDB();
269 playlist.items = playlist.items.map((item) => {
270 //some less space in the db by omitting ids
271 const { id, ...itemWithoutId } = item as PlaylistItem;
272 return itemWithoutId;
273 });
274 if (overwrite) await db.put(PlaylistStore, playlist);
275 else await db.add(PlaylistStore, playlist);
276}
277
278export async function deletePlaylist(name: string) {
279 const db = await openDB();
280 await db.delete(PlaylistStore, name);
281}
282
283//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
284export async function deleteFile(
285 fileName: string,
286 item: MediaFile | Directory,
287 basePath: string,
288 fetchFiles: ResourceActions<FileListing>,
289) {
290 const fullPath = joinPath(basePath, fileName);
291 if ("metadata" in item && !downloadStatusMap[fullPath]) return;
292
293 if (!("metadata" in item)) {
294 const newBasePath = joinPath(basePath, fileName);
295 const result = listOfflineFiles(newBasePath);
296 for (const [childName, childItem] of result) {
297 await deleteFile(childName, childItem, newBasePath, fetchFiles);
298 }
299 }
300
301 deleteOfflinePath(fullPath);
302 const db = await openDB();
303 await db.delete(FileStore, fullPath);
304 setDownloadStatusMap(fullPath, undefined);
305 fetchFiles.refetch();
306}
307
308const downloadSemaphore = new AsyncSemaphore(1);
309
310//Download file to indexedDB and updates the offline file tree
311//concurrency control via the downloadSemaphore
312async function downloadFile(
313 path: string,
314 currentDir: Accessor<string>,
315 options: AppOptions,
316 item: MediaFile,
317 fetchFiles: ResourceActions<FileListing>,
318): Promise<void> {
319 const initialProgress = downloadStatusMap[path];
320 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
321
322 try {
323 setDownloadStatusMap(path, { bytes: 0 });
324 const db = await openDB();
325 let mimeType: string | undefined;
326
327 //options might change at any point, update the semaphore to the current value
328 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
329 let blob: Blob;
330 try {
331 //make sure to only allow the set amount of parallel downloads
332 await downloadSemaphore.acquire();
333 const res = await fetch(await getSrc(path, options, false, true));
334
335 //let the browser do the actual download
336 const blobPromise = res.clone().blob();
337 if (!res.ok || !res.body) throw res.statusText;
338 //we just iterate over the chunks to get the current progress
339 const reader = res.body.getReader();
340 mimeType = res.headers.get("content-type") ?? undefined;
341 let received = 0;
342
343 let lastUpdate = 0;
344 while (true) {
345 const { done, value } = await reader.read();
346 if (done) break;
347 received += value.length;
348
349 //debouncing logic to not update UI too fast
350 const now = Date.now();
351 if (now - lastUpdate < 250) continue;
352 lastUpdate = now;
353 setDownloadStatusMap(path, { bytes: received });
354 }
355 blob = await blobPromise;
356 } finally {
357 downloadSemaphore.release();
358 }
359
360 await db.put(FileStore, {
361 path,
362 data: blob,
363 metadata: item.metadata,
364 } as StoredFile);
365 updateFileTree(path, item, true);
366 setDownloadStatusMap(path, "Synced");
367 const basePath = path.split("/").slice(0, -1).join("/");
368 const fileName = path.split("/").toReversed()[0];
369 //add files to listing if the containing directory is currently shown
370 if (currentDir() === basePath) {
371 //directly mutate files to not cause unnecessary rerenders
372 fetchFiles.mutate((prev) => {
373 prev.set(fileName, item);
374 //new map to force rerender
375 return new Map(prev);
376 });
377 }
378 } catch (e) {
379 toast(`Downloading ${path} failed`, "error", e);
380 //restore original status
381 setDownloadStatusMap(path, initialProgress === "Synced" ? "Synced" : undefined);
382 }
383}
384
385export async function download(
386 fileName: string,
387 item: MediaFile | Directory,
388 basePath: string,
389 currentDir: Accessor<string>,
390 options: AppOptions,
391 fetchFiles: ResourceActions<FileListing>,
392) {
393 const fullPath = joinPath(basePath, fileName);
394 if ("metadata" in item) {
395 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
396 } else {
397 setDownloadStatusMap(fullPath, "DirectoryProgress");
398 await getCover(fullPath, false);
399 try {
400 //start downloads for all children to allow them to run in parallel, then wait for the results
401 const promises: Promise<unknown>[] = [];
402 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
403 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
404 }
405 await Promise.allSettled(promises);
406 } finally {
407 //always set to undefined afterwards to show the normal sync icon, since we don't know for sure if the directory is completely synced
408 setDownloadStatusMap(fullPath, undefined);
409 }
410 }
411}
412
413export async function getStoredFiles() {
414 const db = await openDB();
415 return await db.getKeys(FileStore);
416}
417
418interface StoredFile {
419 path: string;
420 data: Blob;
421 metadata: {
422 duration: number;
423 };
424}
425
426interface StoredCover {
427 path: string;
428 data: Blob | null;
429}
430
431const FileStore = "fileblobs";
432const CoverStore = "covers";
433const PlaylistStore = "playlists";
434
435async function openDB() {
436 return new Promise<DatabaseWrapper>((resolve, reject) => {
437 const req = indexedDB.open("music-files");
438 req.onerror = () => reject(req.error);
439 req.onupgradeneeded = () => {
440 const db = req.result;
441 if (!db.objectStoreNames.contains(FileStore))
442 db.createObjectStore(FileStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
443 if (!db.objectStoreNames.contains(CoverStore))
444 db.createObjectStore(CoverStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
445 if (!db.objectStoreNames.contains(PlaylistStore))
446 db.createObjectStore(PlaylistStore, { keyPath: "path" }).createIndex("index", "path", { unique: true });
447 };
448 req.onsuccess = () => resolve(wrapDB(req.result));
449 });
450}
451
452interface DatabaseWrapper {
453 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
454 put(store: typeof FileStore, value: StoredFile): Promise<void>;
455 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
456 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
457 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
458 delete(store: typeof FileStore | typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
459 clear(store: string): Promise<void>;
460 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
461 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
462 getKeys(store: typeof FileStore): Promise<string[]>;
463}
464
465function wrapDB(db: IDBDatabase): DatabaseWrapper {
466 return {
467 get: (store: string, key: string) =>
468 new Promise((res, rej) => {
469 const tx = db.transaction(store, "readonly");
470 const req = tx.objectStore(store).get(key);
471 req.onsuccess = () => res(req.result);
472 req.onerror = () => rej(req.error);
473 }),
474 put: (store: string, value: unknown) =>
475 new Promise<void>((res, rej) => {
476 const tx = db.transaction(store, "readwrite");
477 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
478 req.onsuccess = () => res();
479 req.onerror = () => rej(req.error);
480 }),
481 add: (store: string, value: unknown) =>
482 new Promise<void>((res, rej) => {
483 const tx = db.transaction(store, "readwrite");
484 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
485 req.onsuccess = () => res();
486 req.onerror = () => rej(req.error);
487 }),
488 delete: (store: string, key: string) =>
489 new Promise<void>((res, rej) => {
490 const tx = db.transaction(store, "readwrite");
491 const req = tx.objectStore(store).delete(key);
492 req.onsuccess = () => res();
493 req.onerror = () => rej(req.error);
494 }),
495 clear: (store: string) =>
496 new Promise<void>((res, rej) => {
497 const tx = db.transaction(store, "readwrite");
498 const req = tx.objectStore(store).clear();
499 req.onsuccess = () => res();
500 req.onerror = () => rej(req.error);
501 }),
502 listFiles: (store: typeof FileStore) =>
503 new Promise((res, rej) => {
504 const tx = db.transaction(store, "readonly");
505 const req = tx.objectStore(store).getAll();
506 req.onsuccess = () => res(req.result);
507 req.onerror = () => rej(req.error);
508 }),
509 listPlaylists: (store: typeof PlaylistStore) =>
510 new Promise((res, rej) => {
511 const tx = db.transaction(store, "readonly");
512 const req = tx.objectStore(store).getAll();
513 req.onsuccess = () => res(req.result);
514 req.onerror = () => rej(req.error);
515 }),
516 getKeys: (store: typeof FileStore) =>
517 new Promise((res, rej) => {
518 const tx = db.transaction(store, "readonly");
519 const req = tx.objectStore(store).getAllKeys();
520 req.onsuccess = () => res(req.result as string[]);
521 req.onerror = () => rej(req.error);
522 }),
523 };
524}
525