offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import type { Accessor, ResourceActions } from "solid-js";
12import { createStore } from "solid-js/store";
13import { AsyncSemaphore } from "./semaphore";
14
15interface Progress {
16 bytes: number;
17}
18
19export type DownloadStatus = Progress | "DirectoryProgress" | "Synced" | undefined;
20//used to display download status for directories/files in the filebrowser view
21export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
22
23//initialize filetrees and download statuses from indexedDB
24export async function loadDataFromDB() {
25 try {
26 const db = await openDB();
27 const files = await db.listFiles(FileStore);
28 for (const file of files) {
29 setDownloadStatusMap(file.path, "Synced");
30 updateFileTree(file.path, { metadata: file.metadata }, true);
31 }
32 } catch (e) {
33 toast("Loading offline files failed", "error", e);
34 console.error(e);
35 }
36}
37
38//contains all locally synced files
39const offlineTreeRoot: Directory = {
40 files: new Map(),
41};
42//used for caching online file listings
43const onlineTreeRoot: DirectoryWithStatus = {
44 files: new Map(),
45 status: "Unknown",
46};
47
48//merges "Scanned" directories from "other" to "target"
49function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
50 if (other.status === "Unknown") return;
51 target.status = other.status;
52 for (const [fileName, file] of other.files.entries()) {
53 const targetFile = target.files.get(fileName);
54 if (targetFile && "files" in targetFile && "files" in file) {
55 mergeTrees(targetFile, file);
56 } else {
57 target.files.set(fileName, file);
58 }
59 }
60}
61
62//TODO: improve typing to avoid "as"
63//overloads for typecheking
64//offline tree only stores files, directories are created ad-hoc
65function updateFileTree(path: string, file: MediaFile, offline: true): void;
66//online tree only stores cached directory listings
67function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
68// Updates file tree structure with new data (offline or online)
69function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
70 if (offline && path === "") throw new Error("Trying to update root in offline mode");
71 const basePath = path.split("/").slice(0, -1).join("/");
72 const fileName = path.split("/").toReversed()[0];
73 const node = getDirInTree(basePath, true, offline);
74
75 if (offline) {
76 node.files.set(fileName, file);
77 } else {
78 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
79 else {
80 let target = node.files.get(fileName) as DirectoryWithStatus;
81 if (!target) target = { files: new Map(), status: "Unknown" };
82 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
83 node.files.set(fileName, target);
84 }
85 }
86}
87
88//overloads for typecheking
89function getDirInTree(path: string, create: true, offline: true): Directory;
90function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
91function getDirInTree(path: string, create: false, offline: true): Directory | null;
92function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
93function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
94function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
95// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
96function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
97 let current = offline ? offlineTreeRoot : onlineTreeRoot;
98 if (path === "") return current;
99 const parts = path.split("/");
100
101 for (const part of parts) {
102 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
103 if (!next || "metadata" in next) {
104 if (!create) return null;
105 //if mediafile already exists, we overwrite it with a directory anyway
106 if (offline) next = { files: new Map() };
107 else next = { files: new Map(), status: "Unknown" };
108 current.files.set(part, next);
109 }
110 current = next;
111 }
112 return current;
113}
114
115function deleteOfflinePath(path: string): void {
116 const basePath = path.split("/").slice(0, -1).join("/");
117 const fileName = path.split("/").toReversed()[0];
118
119 const node = getDirInTree(basePath, false, true);
120 if (!node) return;
121
122 node.files.delete(fileName);
123 if (node.files.size === 0) deleteOfflinePath(basePath);
124}
125
126export const clearDirectoryCache = (path: string) => {
127 const node = getDirInTree(path, false, false);
128 if (!node) return;
129 node.files.clear();
130 node.status = "Unknown";
131};
132
133async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
134 const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {});
135 if (response?.status === 404) {
136 toast(`Directory not found: ${dir}`, "error");
137 return;
138 }
139 if (!response || !response.ok) {
140 toast(
141 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
142 "error",
143 );
144 return;
145 }
146
147 return parseMap(await response.text()) as FileListingWithStatus;
148}
149
150function listOfflineFiles(dir: string): FileListing {
151 const node = getDirInTree(dir, false, true);
152 if (!node || !node.files) return new Map();
153 return node.files;
154}
155
156//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
157function fullyKnown(dir: DirectoryWithStatus) {
158 if (dir.status === "Unknown") {
159 return false;
160 }
161 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
162 return false;
163 }
164 return true;
165}
166
167//non-recursive listing should not include nested children
168function pruneChildren(listing: FileListing) {
169 const cloned = structuredClone(listing);
170 for (const [, item] of cloned.entries()) {
171 if ("files" in item) item.files.clear();
172 }
173 return cloned;
174}
175
176//used for combining the online and offline results, to be able to show files that only exist locally
177function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
178 const result = new Map(listing1);
179
180 for (const [key, value2] of listing2.entries()) {
181 const value1 = result.get(key);
182
183 if (value1 && "files" in value1 && "files" in value2) {
184 result.set(key, { files: combineFileListings(value1.files, value2.files) });
185 } else {
186 result.set(key, value2);
187 }
188 }
189
190 return result;
191}
192
193// File listing logic with hybrid online/offline support
194// Combines:
195// 1. Direct IndexedDB lookups for offline mode
196// 2. Cached online listings
197// 3. Merged online+offline results
198export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
199 if (offline) {
200 const files = listOfflineFiles(dir);
201 return recursive ? files : pruneChildren(files);
202 }
203 const cached = getDirInTree(dir, false, false);
204 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
205 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
206 return recursive ? combined : pruneChildren(combined);
207 }
208 const files = await listOnlineFiles(dir, recursive);
209 if (!files) return new Map();
210 updateFileTree(dir, { files, status: "Scanned" }, false);
211 const combined = combineFileListings(files, listOfflineFiles(dir));
212 return recursive ? combined : pruneChildren(combined);
213}
214
215//get cover art for a directory with cache-first strategy
216export async function getCover(dir: string, offline: boolean) {
217 const db = await openDB();
218 const dbResult = await db.get(CoverStore, dir);
219 if (dbResult !== undefined) {
220 const cached = dbResult as StoredCover;
221 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
222 }
223
224 if (offline) return undefined;
225
226 const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {});
227 if (!response || response.status === 404) {
228 db.put(CoverStore, { path: dir, data: null });
229 return undefined;
230 }
231 const blob = await response.blob();
232 db.put(CoverStore, { path: dir, data: blob });
233 return URL.createObjectURL(blob);
234}
235
236export const clearCoverCache = async (path: string) => {
237 const db = await openDB();
238 return db.delete(CoverStore, path);
239};
240
241//returns an object url to the blob if synced, otherwise the server url
242export async function getSrc(path: string, options: AppOptions, offline: boolean) {
243 const db = await openDB();
244 const dbResult = (await db.get(FileStore, path)) as StoredFile;
245 if (dbResult) return URL.createObjectURL(dbResult.data);
246
247 if (offline) {
248 toast("This file is not cached, leave offline mode to play it", "error");
249 return undefined;
250 }
251 if (options.enableTranscoding) {
252 return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
253 }
254 return joinPath("/download", path);
255}
256
257export async function loadPlaylists() {
258 const db = await openDB();
259 return await db.listPlaylists(PlaylistStore);
260}
261
262//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
263export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
264 const db = await openDB();
265 playlist.items = playlist.items.map((item) => {
266 //some less space in the db by omitting ids
267 const { id, ...itemWithoutId } = item as PlaylistItem;
268 return itemWithoutId;
269 });
270 if (overwrite) await db.put(PlaylistStore, playlist);
271 else await db.add(PlaylistStore, playlist);
272}
273
274export async function deletePlaylist(name: string) {
275 const db = await openDB();
276 await db.delete(PlaylistStore, name);
277}
278
279//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
280export async function deleteFile(
281 fileName: string,
282 item: MediaFile | Directory,
283 basePath: string,
284 fetchFiles: ResourceActions<FileListing>,
285) {
286 const fullPath = joinPath(basePath, fileName);
287 if ("metadata" in item && !downloadStatusMap[fullPath]) return;
288
289 if (!("metadata" in item)) {
290 const newBasePath = joinPath(basePath, fileName);
291 const result = listOfflineFiles(newBasePath);
292 for (const [childName, childItem] of result) {
293 await deleteFile(childName, childItem, newBasePath, fetchFiles);
294 }
295 }
296
297 deleteOfflinePath(fullPath);
298 const db = await openDB();
299 await db.delete(FileStore, fullPath);
300 setDownloadStatusMap(fullPath, undefined);
301 fetchFiles.refetch();
302}
303
304const downloadSemaphore = new AsyncSemaphore(1);
305
306//Download file to indexedDB and updates the offline file tree
307//concurrency control via the downloadSemaphore
308async function downloadFile(
309 path: string,
310 currentDir: Accessor<string>,
311 options: AppOptions,
312 item: MediaFile,
313 fetchFiles: ResourceActions<FileListing>,
314): Promise<void> {
315 const initialProgress = downloadStatusMap[path];
316 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
317
318 try {
319 setDownloadStatusMap(path, { bytes: 0 });
320 const db = await openDB();
321 let mimeType: string | undefined;
322
323 //options might change at any point, update the semaphore to the current value
324 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
325 let blob: Blob;
326 try {
327 //make sure to only allow the set amount of parallel downloads
328 await downloadSemaphore.acquire();
329 let res: Response;
330 if (options.enableTranscoding) {
331 res = await fetch(
332 `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`,
333 );
334 } else {
335 res = await fetch(joinPath("/download", path));
336 }
337 //let the browser do the actual download
338 const blobPromise = res.clone().blob();
339 if (!res.ok || !res.body) throw res.statusText;
340 //we just iterate over the chunks to get the current progress
341 const reader = res.body.getReader();
342 mimeType = res.headers.get("content-type") ?? undefined;
343 let received = 0;
344
345 let lastUpdate = 0;
346 while (true) {
347 const { done, value } = await reader.read();
348 if (done) break;
349 received += value.length;
350
351 //debouncing logic to not update UI too fast
352 const now = Date.now();
353 if (now - lastUpdate < 250) continue;
354 lastUpdate = now;
355 setDownloadStatusMap(path, { bytes: received });
356 }
357 blob = await blobPromise;
358 } finally {
359 downloadSemaphore.release();
360 }
361
362 await db.put(FileStore, {
363 path,
364 data: blob,
365 metadata: item.metadata,
366 } as StoredFile);
367 updateFileTree(path, item, true);
368 setDownloadStatusMap(path, "Synced");
369 const basePath = path.split("/").slice(0, -1).join("/");
370 const fileName = path.split("/").toReversed()[0];
371 //add files to listing if the containing directory is currently shown
372 if (currentDir() === basePath) {
373 //directly mutate files to not cause unnecessary rerenders
374 fetchFiles.mutate((prev) => {
375 prev.set(fileName, item);
376 //new map to force rerender
377 return new Map(prev);
378 });
379 }
380 } catch (e) {
381 toast(`Downloading ${path} failed`, "error", e);
382 //restore original status
383 setDownloadStatusMap(path, initialProgress === "Synced" ? "Synced" : undefined);
384 }
385}
386
387export async function download(
388 fileName: string,
389 item: MediaFile | Directory,
390 basePath: string,
391 currentDir: Accessor<string>,
392 options: AppOptions,
393 fetchFiles: ResourceActions<FileListing>,
394) {
395 const fullPath = joinPath(basePath, fileName);
396 if ("metadata" in item) {
397 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
398 } else {
399 setDownloadStatusMap(fullPath, "DirectoryProgress");
400 await getCover(fullPath, false);
401 try {
402 //start downloads for all children to allow them to run in parallel, then wait for the results
403 const promises: Promise<unknown>[] = [];
404 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
405 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
406 }
407 await Promise.allSettled(promises);
408 } finally {
409 //always set to undefined afterwards to show the normal sync icon, since we don't know for sure if the directory is completely synced
410 setDownloadStatusMap(fullPath, undefined);
411 }
412 }
413}
414
415export async function isFileSynced(path: string, status: DownloadStatus | undefined): Promise<boolean> {
416 if (status === "Synced") return true;
417 //if in progress, it might already exist anyway and just is being re-synced right now
418 if (typeof status === "object" && "bytes" in status) {
419 const db = await openDB();
420 return await db.exist(FileStore, path);
421 }
422 return false;
423}
424
425interface StoredFile {
426 path: string;
427 data: Blob;
428 metadata: {
429 duration: number;
430 };
431}
432
433interface StoredCover {
434 path: string;
435 data: Blob | null;
436}
437
438const FileStore = "fileblobs";
439const CoverStore = "covers";
440const PlaylistStore = "playlists";
441
442async function openDB() {
443 return new Promise<DatabaseWrapper>((resolve, reject) => {
444 const req = indexedDB.open("music-files");
445 req.onerror = () => reject(req.error);
446 req.onupgradeneeded = () => {
447 const db = req.result;
448 if (!db.objectStoreNames.contains(FileStore)) db.createObjectStore(FileStore, { keyPath: "path" });
449 if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" });
450 if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" });
451 };
452 req.onsuccess = () => resolve(wrapDB(req.result));
453 });
454}
455
456interface DatabaseWrapper {
457 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
458 put(store: typeof FileStore, value: StoredFile): Promise<void>;
459 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
460 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
461 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
462 delete(store: typeof FileStore | typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
463 clear(store: string): Promise<void>;
464 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
465 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
466 exist(store: typeof FileStore, key: string): Promise<boolean>;
467}
468
469function wrapDB(db: IDBDatabase): DatabaseWrapper {
470 return {
471 get: (store: string, key: string) =>
472 new Promise((res, rej) => {
473 const tx = db.transaction(store, "readonly");
474 const req = tx.objectStore(store).get(key);
475 req.onsuccess = () => res(req.result);
476 req.onerror = () => rej(req.error);
477 }),
478 put: (store: string, value: unknown) =>
479 new Promise<void>((res, rej) => {
480 const tx = db.transaction(store, "readwrite");
481 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
482 req.onsuccess = () => res();
483 req.onerror = () => rej(req.error);
484 }),
485 add: (store: string, value: unknown) =>
486 new Promise<void>((res, rej) => {
487 const tx = db.transaction(store, "readwrite");
488 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
489 req.onsuccess = () => res();
490 req.onerror = () => rej(req.error);
491 }),
492 delete: (store: string, key: string) =>
493 new Promise<void>((res, rej) => {
494 const tx = db.transaction(store, "readwrite");
495 const req = tx.objectStore(store).delete(key);
496 req.onsuccess = () => res();
497 req.onerror = () => rej(req.error);
498 }),
499 clear: (store: string) =>
500 new Promise<void>((res, rej) => {
501 const tx = db.transaction(store, "readwrite");
502 const req = tx.objectStore(store).clear();
503 req.onsuccess = () => res();
504 req.onerror = () => rej(req.error);
505 }),
506 listFiles: (store: typeof FileStore) =>
507 new Promise((res, rej) => {
508 const tx = db.transaction(store, "readonly");
509 const req = tx.objectStore(store).getAll();
510 req.onsuccess = () => res(req.result);
511 req.onerror = () => rej(req.error);
512 }),
513 listPlaylists: (store: typeof PlaylistStore) =>
514 new Promise((res, rej) => {
515 const tx = db.transaction(store, "readonly");
516 const req = tx.objectStore(store).getAll();
517 req.onsuccess = () => res(req.result);
518 req.onerror = () => rej(req.error);
519 }),
520 exist: (store: typeof FileStore, key: string) =>
521 new Promise((res, rej) => {
522 const tx = db.transaction(store, "readonly");
523 const req = tx.objectStore(store).count(key);
524 req.onsuccess = () => res(req.result !== 0);
525 req.onerror = () => rej(req.error);
526 }),
527 };
528}
529