offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import type { Accessor, ResourceActions } from "solid-js";
12import { createStore } from "solid-js/store";
13import { AsyncSemaphore } from "./semaphore";
14
15interface Progress {
16 bytes: number;
17}
18
19export type DownloadStatus = Progress | "DirectoryProgress" | "Synced" | undefined;
20//used to display download status for directories/files in the filebrowser view
21export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
22
23//initialize filetrees and download statuses from indexedDB
24export async function loadDataFromDB() {
25 try {
26 const db = await openDB();
27 const files = await db.listFiles(FileStore);
28 for (const file of files) {
29 const parts = file.path.split("/");
30 let current = "";
31 for (let index = 0; index < parts.length; index++) {
32 current = joinPath(current, parts[index]);
33 if (index === parts.length - 1) setDownloadStatusMap(current, "Synced");
34 }
35 updateFileTree(file.path, { metadata: file.metadata }, true);
36 }
37 } catch (e) {
38 toast("Loading offline files failed", "error", e);
39 console.error(e);
40 }
41}
42
43//contains all locally synced files
44const offlineTreeRoot: Directory = {
45 files: new Map(),
46};
47//used for caching online file listings
48const onlineTreeRoot: DirectoryWithStatus = {
49 files: new Map(),
50 status: "Unknown",
51};
52
53//merges "Scanned" directories from "other" to "target"
54function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
55 if (other.status === "Unknown") return;
56 target.status = other.status;
57 for (const [fileName, file] of other.files.entries()) {
58 const targetFile = target.files.get(fileName);
59 if (targetFile && "files" in targetFile && "files" in file) {
60 mergeTrees(targetFile, file);
61 } else {
62 target.files.set(fileName, file);
63 }
64 }
65}
66
67//TODO: improve typing to avoid "as"
68//overloads for typecheking
69//offline tree only stores files, directories are created ad-hoc
70function updateFileTree(path: string, file: MediaFile, offline: true): void;
71//online tree only stores cached directory listings
72function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
73// Updates file tree structure with new data (offline or online)
74function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean) {
75 if (offline && path === "") throw new Error("Trying to update root in offline mode");
76 const basePath = path.split("/").slice(0, -1).join("/");
77 const fileName = path.split("/").toReversed()[0];
78 const node = getDirInTree(basePath, true, offline);
79
80 if (offline) {
81 node.files.set(fileName, file);
82 } else {
83 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
84 else {
85 let target = node.files.get(fileName) as DirectoryWithStatus;
86 if (!target) target = { files: new Map(), status: "Unknown" };
87 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
88 node.files.set(fileName, target);
89 }
90 }
91}
92
93//overloads for typecheking
94function getDirInTree(path: string, create: true, offline: true): Directory;
95function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
96function getDirInTree(path: string, create: false, offline: true): Directory | null;
97function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
98function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
99function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
100// get a DirectoryNode from the directory tree matching the path, and optionally creating missing nodes it if they don't exist yet
101function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
102 let current = offline ? offlineTreeRoot : onlineTreeRoot;
103 if (path === "") return current;
104 const parts = path.split("/");
105
106 for (const part of parts) {
107 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
108 if (!next || "metadata" in next) {
109 if (!create) return null;
110 //if mediafile already exists, we overwrite it with a directory anyway
111 if (offline) next = { files: new Map() };
112 else next = { files: new Map(), status: "Unknown" };
113 current.files.set(part, next);
114 }
115 current = next;
116 }
117 return current;
118}
119
120function deleteOfflinePath(path: string): void {
121 const basePath = path.split("/").slice(0, -1).join("/");
122 const fileName = path.split("/").toReversed()[0];
123
124 const node = getDirInTree(basePath, false, true);
125 if (!node) return;
126
127 node.files.delete(fileName);
128 if (node.files.size === 0) deleteOfflinePath(basePath);
129}
130
131export const clearDirectoryCache = (path: string) => {
132 const node = getDirInTree(path, false, false);
133 if (!node) return;
134 node.files.clear();
135 node.status = "Unknown";
136};
137
138async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
139 const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {});
140 if (response?.status === 404) {
141 toast(`Directory not found: ${dir}`, "error");
142 return;
143 }
144 if (!response || !response.ok) {
145 toast(
146 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
147 "error",
148 );
149 return;
150 }
151
152 return parseMap(await response.text()) as FileListingWithStatus;
153}
154
155function listOfflineFiles(dir: string): FileListing {
156 const node = getDirInTree(dir, false, true);
157 if (!node || !node.files) return new Map();
158 return node.files;
159}
160
161//checks if this Directory is completely scanned, i.e. suitable for a recursive listing result
162function fullyKnown(dir: DirectoryWithStatus) {
163 if (dir.status === "Unknown") {
164 return false;
165 }
166 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
167 return false;
168 }
169 return true;
170}
171
172//non-recursive listing should not include nested children
173function pruneChildren(listing: FileListing) {
174 const cloned = structuredClone(listing);
175 for (const [, item] of cloned.entries()) {
176 if ("files" in item) item.files.clear();
177 }
178 return cloned;
179}
180
181//used for combining the online and offline results, to be able to show files that only exist locally
182function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
183 const result = new Map(listing1);
184
185 for (const [key, value2] of listing2.entries()) {
186 const value1 = result.get(key);
187
188 if (value1 && "files" in value1 && "files" in value2) {
189 result.set(key, { files: combineFileListings(value1.files, value2.files) });
190 } else {
191 result.set(key, value2);
192 }
193 }
194
195 return result;
196}
197
198// File listing logic with hybrid online/offline support
199// Combines:
200// 1. Direct IndexedDB lookups for offline mode
201// 2. Cached online listings
202// 3. Merged online+offline results
203export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
204 if (offline) {
205 const files = listOfflineFiles(dir);
206 return recursive ? files : pruneChildren(files);
207 }
208 const cached = getDirInTree(dir, false, false);
209 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
210 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
211 return recursive ? combined : pruneChildren(combined);
212 }
213 const files = await listOnlineFiles(dir, recursive);
214 if (!files) return new Map();
215 updateFileTree(dir, { files, status: "Scanned" }, false);
216 const combined = combineFileListings(files, listOfflineFiles(dir));
217 return recursive ? combined : pruneChildren(combined);
218}
219
220//get cover art for a directory with cache-first strategy
221export async function getCover(dir: string, offline: boolean) {
222 const db = await openDB();
223 const dbResult = await db.get(CoverStore, dir);
224 if (dbResult !== undefined) {
225 const cached = dbResult as StoredCover;
226 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
227 }
228
229 if (offline) return undefined;
230
231 const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {});
232 if (!response || response.status === 404) {
233 db.put(CoverStore, { path: dir, data: null });
234 return undefined;
235 }
236 const blob = await response.blob();
237 db.put(CoverStore, { path: dir, data: blob });
238 return URL.createObjectURL(blob);
239}
240
241export const clearCoverCache = async (path: string) => {
242 const db = await openDB();
243 return db.delete(CoverStore, path);
244};
245
246//returns an object url to the blob if synced, otherwise the server url
247export async function getSrc(path: string, options: AppOptions, offline: boolean) {
248 const db = await openDB();
249 const dbResult = (await db.get(FileStore, path)) as StoredFile;
250 if (dbResult) return URL.createObjectURL(dbResult.data);
251
252 if (offline) {
253 toast("This file is not cached, leave offline mode to play it", "error");
254 return undefined;
255 }
256 if (options.enableTranscoding) {
257 return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
258 }
259 return joinPath("/download", path);
260}
261
262export async function loadPlaylists() {
263 const db = await openDB();
264 return await db.listPlaylists(PlaylistStore);
265}
266
267//save playlist to indexedDB, throws error if playlist with that name already exists and overwrite=false
268export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
269 const db = await openDB();
270 playlist.items = playlist.items.map((item) => {
271 //some less space in the db by omitting ids
272 const { id, ...itemWithoutId } = item as PlaylistItem;
273 return itemWithoutId;
274 });
275 if (overwrite) await db.put(PlaylistStore, playlist);
276 else await db.add(PlaylistStore, playlist);
277}
278
279export async function deletePlaylist(name: string) {
280 const db = await openDB();
281 await db.delete(PlaylistStore, name);
282}
283
284//deletes file or directory(+recursive contents) from indexedDB and updates their download statuses
285export async function deleteFile(
286 fileName: string,
287 item: MediaFile | Directory,
288 basePath: string,
289 fetchFiles: ResourceActions<FileListing>,
290) {
291 const fullPath = joinPath(basePath, fileName);
292 if ("metadata" in item && !downloadStatusMap[fullPath]) return;
293
294 if (!("metadata" in item)) {
295 const newBasePath = joinPath(basePath, fileName);
296 const result = listOfflineFiles(newBasePath);
297 for (const [childName, childItem] of result) {
298 await deleteFile(childName, childItem, newBasePath, fetchFiles);
299 }
300 }
301
302 deleteOfflinePath(fullPath);
303 const db = await openDB();
304 await db.delete(FileStore, fullPath);
305 setDownloadStatusMap(fullPath, undefined);
306 fetchFiles.refetch();
307}
308
309const downloadSemaphore = new AsyncSemaphore(1);
310
311//Download file to indexedDB and updates the offline file tree
312//concurrency control via the downloadSemaphore
313async function downloadFile(
314 path: string,
315 currentDir: Accessor<string>,
316 options: AppOptions,
317 item: MediaFile,
318 fetchFiles: ResourceActions<FileListing>,
319): Promise<void> {
320 const initialProgress = downloadStatusMap[path];
321 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
322
323 try {
324 setDownloadStatusMap(path, { bytes: 0 });
325 const db = await openDB();
326 let mimeType: string | undefined;
327
328 //options might change at any point, update the semaphore to the current value
329 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
330 let blob: Blob;
331 try {
332 //make sure to only allow the set amount of parallel downloads
333 await downloadSemaphore.acquire();
334 let res: Response;
335 if (options.enableTranscoding) {
336 res = await fetch(
337 `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`,
338 );
339 } else {
340 res = await fetch(joinPath("/download", path));
341 }
342 //let the browser do the actual download
343 const blobPromise = res.clone().blob();
344 if (!res.ok || !res.body) throw res.statusText;
345 //we just iterate over the chunks to get the current progress
346 const reader = res.body.getReader();
347 mimeType = res.headers.get("content-type") ?? undefined;
348 let received = 0;
349
350 let lastUpdate = 0;
351 while (true) {
352 const { done, value } = await reader.read();
353 if (done) break;
354 received += value.length;
355
356 //debouncing logic to not update UI too fast
357 const now = Date.now();
358 if (now - lastUpdate < 250) continue;
359 lastUpdate = now;
360 setDownloadStatusMap(path, { bytes: received });
361 }
362 blob = await blobPromise;
363 } finally {
364 downloadSemaphore.release();
365 }
366
367 await db.put(FileStore, {
368 path,
369 data: blob,
370 metadata: item.metadata,
371 } as StoredFile);
372 updateFileTree(path, item, true);
373 setDownloadStatusMap(path, "Synced");
374 const basePath = path.split("/").slice(0, -1).join("/");
375 const fileName = path.split("/").toReversed()[0];
376 //add files to listing if the containing directory is currently shown
377 if (currentDir() === basePath) {
378 //directly mutate files to not cause unnecessary rerenders
379 fetchFiles.mutate((prev) => {
380 prev.set(fileName, item);
381 //new map to force rerender
382 return new Map(prev);
383 });
384 }
385 } catch (e) {
386 toast(`Downloading ${path} failed`, "error", e);
387 //restore original status
388 setDownloadStatusMap(path, initialProgress === "Synced" ? "Synced" : undefined);
389 }
390}
391
392export async function download(
393 fileName: string,
394 item: MediaFile | Directory,
395 basePath: string,
396 currentDir: Accessor<string>,
397 options: AppOptions,
398 fetchFiles: ResourceActions<FileListing>,
399) {
400 const fullPath = joinPath(basePath, fileName);
401 if ("metadata" in item) {
402 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
403 } else {
404 setDownloadStatusMap(fullPath, "DirectoryProgress");
405 await getCover(fullPath, false);
406 try {
407 //start downloads for all children to allow them to run in parallel, then wait for the results
408 const promises: Promise<unknown>[] = [];
409 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
410 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
411 }
412 await Promise.allSettled(promises);
413 } finally {
414 //always set to undefined afterwards to show the normal sync icon, since we don't know for sure if the directory is completely synced
415 setDownloadStatusMap(fullPath, undefined);
416 }
417 }
418}
419
420export async function isFileSynced(path: string, status: DownloadStatus | undefined): Promise<boolean> {
421 if (status === "Synced") return true;
422 //if in progress, it might already exist anyway and just is being re-synced right now
423 if (typeof status === "object" && "bytes" in status) {
424 const db = await openDB();
425 return await db.exist(FileStore, path);
426 }
427 return false;
428}
429
430interface StoredFile {
431 path: string;
432 data: Blob;
433 metadata: {
434 duration: number;
435 };
436}
437
438interface StoredCover {
439 path: string;
440 data: Blob | null;
441}
442
443const FileStore = "fileblobs";
444const CoverStore = "covers";
445const PlaylistStore = "playlists";
446
447async function openDB() {
448 return new Promise<DatabaseWrapper>((resolve, reject) => {
449 const req = indexedDB.open("music-files");
450 req.onerror = () => reject(req.error);
451 req.onupgradeneeded = () => {
452 const db = req.result;
453 if (!db.objectStoreNames.contains(FileStore)) db.createObjectStore(FileStore, { keyPath: "path" });
454 if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" });
455 if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" });
456 };
457 req.onsuccess = () => resolve(wrapDB(req.result));
458 });
459}
460
461interface DatabaseWrapper {
462 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
463 put(store: typeof FileStore, value: StoredFile): Promise<void>;
464 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
465 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
466 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
467 delete(store: typeof FileStore | typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
468 clear(store: string): Promise<void>;
469 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
470 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
471 exist(store: typeof FileStore, key: string): Promise<boolean>;
472}
473
474function wrapDB(db: IDBDatabase): DatabaseWrapper {
475 return {
476 get: (store: string, key: string) =>
477 new Promise((res, rej) => {
478 const tx = db.transaction(store, "readonly");
479 const req = tx.objectStore(store).get(key);
480 req.onsuccess = () => res(req.result);
481 req.onerror = () => rej(req.error);
482 }),
483 put: (store: string, value: unknown) =>
484 new Promise<void>((res, rej) => {
485 const tx = db.transaction(store, "readwrite");
486 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
487 req.onsuccess = () => res();
488 req.onerror = () => rej(req.error);
489 }),
490 add: (store: string, value: unknown) =>
491 new Promise<void>((res, rej) => {
492 const tx = db.transaction(store, "readwrite");
493 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
494 req.onsuccess = () => res();
495 req.onerror = () => rej(req.error);
496 }),
497 delete: (store: string, key: string) =>
498 new Promise<void>((res, rej) => {
499 const tx = db.transaction(store, "readwrite");
500 const req = tx.objectStore(store).delete(key);
501 req.onsuccess = () => res();
502 req.onerror = () => rej(req.error);
503 }),
504 clear: (store: string) =>
505 new Promise<void>((res, rej) => {
506 const tx = db.transaction(store, "readwrite");
507 const req = tx.objectStore(store).clear();
508 req.onsuccess = () => res();
509 req.onerror = () => rej(req.error);
510 }),
511 listFiles: (store: typeof FileStore) =>
512 new Promise((res, rej) => {
513 const tx = db.transaction(store, "readonly");
514 const req = tx.objectStore(store).getAll();
515 req.onsuccess = () => res(req.result);
516 req.onerror = () => rej(req.error);
517 }),
518 listPlaylists: (store: typeof PlaylistStore) =>
519 new Promise((res, rej) => {
520 const tx = db.transaction(store, "readonly");
521 const req = tx.objectStore(store).getAll();
522 req.onsuccess = () => res(req.result);
523 req.onerror = () => rej(req.error);
524 }),
525 exist: (store: typeof FileStore, key: string) =>
526 new Promise((res, rej) => {
527 const tx = db.transaction(store, "readonly");
528 const req = tx.objectStore(store).count(key);
529 req.onsuccess = () => res(req.result !== 0);
530 req.onerror = () => rej(req.error);
531 }),
532 };
533}
534