pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import type { Accessor, ResourceActions } from "solid-js";
12import { createStore } from "solid-js/store";
13import { AsyncSemaphore } from "./semaphore";
14
15interface Progress {
16 bytes: number;
17}
18
19export type DownloadStatus = Progress | "DirectoryProgress" | "SyncedFile" | undefined;
20export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
21export async function loadDataFromDB() {
22 try {
23 const db = await openDB();
24 const files = await db.listFiles(FileStore);
25 for (const file of files) {
26 const parts = file.path.split("/");
27 let current = "";
28 for (let index = 0; index < parts.length; index++) {
29 current = joinPath(current, parts[index]);
30 setDownloadStatusMap(current, "SyncedFile");
31 }
32 updateFileTree(file.path, { metadata: file.metadata }, true);
33 }
34 } catch (e) {
35 toast("Loading offline files failed", "error", e);
36 console.error(e);
37 }
38}
39
40const offlineTreeRoot: Directory = {
41 files: new Map(),
42};
43
44const onlineTreeRoot: DirectoryWithStatus = {
45 files: new Map(),
46 status: "Unknown",
47};
48
49function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
50 if (other.status === "Unknown") return;
51 target.status = other.status;
52 for (const [fileName, file] of other.files.entries()) {
53 const targetFile = target.files.get(fileName);
54 if (targetFile && "files" in targetFile && "files" in file) {
55 mergeTrees(targetFile, file);
56 } else {
57 target.files.set(fileName, file);
58 }
59 }
60}
61
62//TODO: improve typing to avoid "as"
63//overloads for typecheking
64function updateFileTree(path: string, file: MediaFile, offline: true): void;
65function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
66function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean): void {
67 if (offline && path === "") throw new Error("Trying to update root in offline mode");
68 const basePath = path.split("/").slice(0, -1).join("/");
69 const fileName = path.split("/").toReversed()[0];
70 const node = getDirInTree(basePath, true, offline);
71
72 if (offline) {
73 node.files.set(fileName, file);
74 } else {
75 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
76 else {
77 let target = node.files.get(fileName) as DirectoryWithStatus;
78 if (!target) target = { files: new Map(), status: "Unknown" };
79 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
80 node.files.set(fileName, target);
81 }
82 }
83}
84
85//overloads for typecheking
86function getDirInTree(path: string, create: true, offline: true): Directory;
87function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
88function getDirInTree(path: string, create: false, offline: true): Directory | null;
89function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
90function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
91function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
92
93// get a DirectoryNode from the directory tree, and optionally create it if it doesn't exist yet
94function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
95 let current = offline ? offlineTreeRoot : onlineTreeRoot;
96 if (path === "") return current;
97 const parts = path.split("/");
98
99 for (const part of parts) {
100 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
101 if (!next || "metadata" in next) {
102 if (!create) return null;
103 //if mediafile already exists, we overwrite it with a directory anyway
104 if (offline) next = { files: new Map() };
105 else next = { files: new Map(), status: "Unknown" };
106 current.files.set(part, next);
107 }
108 current = next;
109 }
110 return current;
111}
112
113function deleteOfflinePath(path: string): void {
114 const basePath = path.split("/").slice(0, -1).join("/");
115 const fileName = path.split("/").toReversed()[0];
116
117 const node = getDirInTree(basePath, false, true);
118 if (!node) return;
119
120 node.files.delete(fileName);
121}
122
123export const clearDirectoryCache = (path: string) => {
124 const node = getDirInTree(path, false, false);
125 if (!node) return;
126 node.files.clear();
127 node.status = "Unknown";
128};
129
130async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
131 const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {});
132 if (response?.status === 404) {
133 toast(`Directory not found: ${dir}`, "error");
134 return;
135 }
136 if (!response || !response.ok) {
137 toast(
138 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
139 "error",
140 );
141 return;
142 }
143
144 return parseMap(await response.text()) as FileListingWithStatus;
145}
146
147function listOfflineFiles(dir: string): FileListing {
148 const node = getDirInTree(dir, false, true);
149 if (!node || !node.files) return new Map();
150 return node.files;
151}
152
153function fullyKnown(dir: DirectoryWithStatus) {
154 if (dir.status === "Unknown") {
155 return false;
156 }
157 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
158 return false;
159 }
160 return true;
161}
162
163//non-recursive listing should not include nested children
164function pruneChildren(listing: FileListing) {
165 const cloned = structuredClone(listing);
166 for (const [, item] of cloned.entries()) {
167 if ("files" in item) item.files.clear();
168 }
169 return cloned;
170}
171
172function combineFileListings(listing1: FileListing, listing2: FileListing): FileListing {
173 const result = new Map(listing1);
174
175 for (const [key, value2] of listing2.entries()) {
176 const value1 = result.get(key);
177
178 if (value1 && "files" in value1 && "files" in value2) {
179 result.set(key, { files: combineFileListings(value1.files, value2.files) });
180 } else {
181 result.set(key, value2);
182 }
183 }
184
185 return result;
186}
187
188export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
189 if (offline) {
190 const files = listOfflineFiles(dir);
191 return recursive ? files : pruneChildren(files);
192 }
193 const cached = getDirInTree(dir, false, false);
194 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
195 const combined = combineFileListings(cached.files, listOfflineFiles(dir));
196 return recursive ? combined : pruneChildren(combined);
197 }
198 const files = await listOnlineFiles(dir, recursive);
199 if (!files) return new Map();
200 updateFileTree(dir, { files, status: "Scanned" }, false);
201 const combined = combineFileListings(files, listOfflineFiles(dir));
202 return recursive ? combined : pruneChildren(combined);
203}
204
205export async function getCover(dir: string, offline: boolean) {
206 const db = await openDB();
207 const dbResult = await db.get(CoverStore, dir);
208 if (dbResult !== undefined) {
209 const cached = dbResult as StoredCover;
210 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
211 }
212
213 if (offline) return undefined;
214
215 const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {});
216 if (!response || response.status === 404) {
217 db.put(CoverStore, { path: dir, data: null });
218 return undefined;
219 }
220 const blob = await response.blob();
221 db.put(CoverStore, { path: dir, data: blob });
222 return URL.createObjectURL(blob);
223}
224
225export const clearCoverCache = async (path: string) => {
226 const db = await openDB();
227 return db.delete(CoverStore, path);
228};
229
230export async function getSrc(path: string, options: AppOptions, offline: boolean) {
231 const db = await openDB();
232 const dbResult = (await db.get(FileStore, path)) as StoredFile;
233 if (dbResult) return URL.createObjectURL(dbResult.data);
234
235 if (offline) {
236 toast("This file is not cached, leave offline mode to play it", "error");
237 return undefined;
238 }
239 if (options.enableTranscoding) {
240 return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
241 }
242 return joinPath("/download", path);
243}
244
245export async function loadPlaylists() {
246 const db = await openDB();
247 return await db.listPlaylists(PlaylistStore);
248}
249
250export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
251 const db = await openDB();
252 playlist.items = playlist.items.map((item) => {
253 //some less space in the db by omitting ids
254 const { id, ...itemWithoutId } = item as PlaylistItem;
255 return itemWithoutId;
256 });
257 if (overwrite) await db.put(PlaylistStore, playlist);
258 else await db.add(PlaylistStore, playlist);
259}
260
261export async function deletePlaylist(name: string) {
262 const db = await openDB();
263 await db.delete(PlaylistStore, name);
264}
265
266export async function deleteFile(
267 fileName: string,
268 item: MediaFile | Directory,
269 basePath: string,
270 fetchFiles: ResourceActions<FileListing>,
271) {
272 const fullPath = joinPath(basePath, fileName);
273 if (!downloadStatusMap[fullPath]) return;
274
275 if (!("metadata" in item)) {
276 const newBasePath = joinPath(basePath, fileName);
277 const result = listOfflineFiles(newBasePath);
278 for (const [childName, childItem] of result) {
279 await deleteFile(childName, childItem, newBasePath, fetchFiles);
280 }
281 }
282
283 deleteOfflinePath(fullPath);
284 const db = await openDB();
285 await db.delete(FileStore, fullPath);
286 setDownloadStatusMap(fullPath, undefined);
287 fetchFiles.refetch();
288}
289
290const downloadSemaphore = new AsyncSemaphore(1);
291
292async function downloadFile(
293 path: string,
294 currentDir: Accessor<string>,
295 options: AppOptions,
296 item: MediaFile,
297 fetchFiles: ResourceActions<FileListing>,
298): Promise<void> {
299 const initialProgress = downloadStatusMap[path];
300 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
301
302 try {
303 setDownloadStatusMap(path, { bytes: 0 });
304 const db = await openDB();
305 let mimeType: string | undefined;
306
307 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
308 let blob: Blob;
309 try {
310 await downloadSemaphore.acquire();
311 let res: Response;
312 if (options.enableTranscoding) {
313 res = await fetch(
314 `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`,
315 );
316 } else {
317 res = await fetch(joinPath("/download", path));
318 }
319 //let the browser do the actual download
320 const blobPromise = res.clone().blob();
321 if (!res.ok || !res.body) throw res.statusText;
322 //we just iterate over the chunks to get the current progress
323 const reader = res.body.getReader();
324 mimeType = res.headers.get("content-type") ?? undefined;
325 let received = 0;
326
327 let lastUpdate = 0;
328 while (true) {
329 const { done, value } = await reader.read();
330 if (done) break;
331 received += value.length;
332
333 const now = Date.now();
334 if (now - lastUpdate < 250) continue;
335 lastUpdate = now;
336 setDownloadStatusMap(path, { bytes: received });
337 }
338 blob = await blobPromise;
339 } finally {
340 downloadSemaphore.release();
341 }
342
343 await db.put(FileStore, {
344 path,
345 data: blob,
346 metadata: item.metadata,
347 } as StoredFile);
348 updateFileTree(path, item, true);
349 setDownloadStatusMap(path, "SyncedFile");
350 const basePath = path.split("/").slice(0, -1).join("/");
351 const fileName = path.split("/").toReversed()[0];
352 if (currentDir() === basePath) {
353 fetchFiles.mutate((prev) => {
354 prev.set(fileName, item);
355 //new map to force rerender
356 return new Map(prev);
357 });
358 }
359 } catch (e) {
360 toast(`Downloading ${path} failed`, "error", e);
361 setDownloadStatusMap(path, initialProgress === "SyncedFile" ? "SyncedFile" : undefined);
362 }
363}
364
365export async function download(
366 fileName: string,
367 item: MediaFile | Directory,
368 basePath: string,
369 currentDir: Accessor<string>,
370 options: AppOptions,
371 fetchFiles: ResourceActions<FileListing>,
372) {
373 const fullPath = joinPath(basePath, fileName);
374 if ("metadata" in item) {
375 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
376 } else {
377 const initialProgress = downloadStatusMap[fullPath];
378 setDownloadStatusMap(fullPath, "DirectoryProgress");
379 await getCover(fullPath, false);
380 try {
381 const promises: Promise<unknown>[] = [];
382 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
383 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
384 }
385 await Promise.allSettled(promises);
386 setDownloadStatusMap(fullPath, "SyncedFile");
387 } catch {
388 setDownloadStatusMap(fullPath, initialProgress === "SyncedFile" ? "SyncedFile" : undefined);
389 }
390 }
391}
392
393export async function isFileSynced(path: string, status: DownloadStatus | undefined): Promise<boolean> {
394 if (status === "SyncedFile") return true;
395 //if in progress, it might already exist anyway and just is being re-synced right now
396 if (typeof status === "object" && "bytes" in status) {
397 const db = await openDB();
398 return await db.exist(FileStore, path);
399 }
400 return false;
401}
402
403interface StoredFile {
404 path: string;
405 data: Blob;
406 metadata: {
407 duration: number;
408 };
409}
410
411interface StoredCover {
412 path: string;
413 data: Blob | null;
414}
415
416//separate blobs and metadata, so we don't have to load all file contents at startup, which would consume huge amounts of ram, and be very slow
417const FileStore = "fileblobs";
418const CoverStore = "covers";
419const PlaylistStore = "playlists";
420
421async function openDB() {
422 return new Promise<DatabaseWrapper>((resolve, reject) => {
423 const req = indexedDB.open("music-files");
424 req.onerror = () => reject(req.error);
425 req.onupgradeneeded = () => {
426 const db = req.result;
427 if (!db.objectStoreNames.contains(FileStore)) db.createObjectStore(FileStore, { keyPath: "path" });
428 if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" });
429 if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" });
430 };
431 req.onsuccess = () => resolve(wrapDB(req.result));
432 });
433}
434
435interface DatabaseWrapper {
436 get(store: typeof CoverStore | typeof FileStore, key: string): Promise<unknown>;
437 put(store: typeof FileStore, value: StoredFile): Promise<void>;
438 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
439 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
440 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
441 delete(store: typeof FileStore | typeof CoverStore | typeof PlaylistStore, key: string): Promise<void>;
442 clear(store: string): Promise<void>;
443 listFiles(store: typeof FileStore): Promise<StoredFile[]>;
444 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
445 exist(store: typeof FileStore, key: string): Promise<boolean>;
446}
447
448function wrapDB(db: IDBDatabase): DatabaseWrapper {
449 return {
450 get: (store: string, key: string) =>
451 new Promise((res, rej) => {
452 const tx = db.transaction(store, "readonly");
453 const req = tx.objectStore(store).get(key);
454 req.onsuccess = () => res(req.result);
455 req.onerror = () => rej(req.error);
456 }),
457 put: (store: string, value: unknown) =>
458 new Promise<void>((res, rej) => {
459 const tx = db.transaction(store, "readwrite");
460 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
461 req.onsuccess = () => res();
462 req.onerror = () => rej(req.error);
463 }),
464 add: (store: string, value: unknown) =>
465 new Promise<void>((res, rej) => {
466 const tx = db.transaction(store, "readwrite");
467 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
468 req.onsuccess = () => res();
469 req.onerror = () => rej(req.error);
470 }),
471 delete: (store: string, key: string) =>
472 new Promise<void>((res, rej) => {
473 const tx = db.transaction(store, "readwrite");
474 const req = tx.objectStore(store).delete(key);
475 req.onsuccess = () => res();
476 req.onerror = () => rej(req.error);
477 }),
478 clear: (store: string) =>
479 new Promise<void>((res, rej) => {
480 const tx = db.transaction(store, "readwrite");
481 const req = tx.objectStore(store).clear();
482 req.onsuccess = () => res();
483 req.onerror = () => rej(req.error);
484 }),
485 listFiles: (store: typeof FileStore) =>
486 new Promise((res, rej) => {
487 const tx = db.transaction(store, "readonly");
488 const req = tx.objectStore(store).getAll();
489 req.onsuccess = () => res(req.result);
490 req.onerror = () => rej(req.error);
491 }),
492 listPlaylists: (store: typeof PlaylistStore) =>
493 new Promise((res, rej) => {
494 const tx = db.transaction(store, "readonly");
495 const req = tx.objectStore(store).getAll();
496 req.onsuccess = () => res(req.result);
497 req.onerror = () => rej(req.error);
498 }),
499 exist: (store: typeof FileStore, key: string) =>
500 new Promise((res, rej) => {
501 const tx = db.transaction(store, "readonly");
502 const req = tx.objectStore(store).count(key);
503 req.onsuccess = () => res(req.result !== 0);
504 req.onerror = () => rej(req.error);
505 }),
506 };
507}
508