pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type {
2 MediaFile,
3 Directory,
4 FileListing,
5 FileListingWithStatus,
6 DirectoryWithStatus,
7} from "music-server-shared/types";
8import { parseMap } from "music-server-shared/mapconversion";
9import type { AppOptions, Playlist, PlaylistItem } from "./types";
10import { joinPath, toast } from "./utils";
11import { type Accessor, createSignal, type ResourceActions, type Signal } from "solid-js";
12import { createStore } from "solid-js/store";
13import { AsyncSemaphore } from "./semaphore";
14
15interface Progress {
16 bytes: number;
17}
18
19export type DownloadStatus = undefined | Progress | "DirectoryProgress" | "SyncedFile";
20export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
21
22export async function loadDataFromDB() {
23 try {
24 const db = await openDB();
25 const files = await db.listFiles(FileMetadataStore);
26 for (const file of files) {
27 setDownloadStatusMap(file.path, "SyncedFile");
28 updateFileTree(file.path, { metadata: { duration: file.duration } }, true);
29 }
30 } catch (e) {
31 toast("Loading offline files failed", "error", e);
32 console.error(e);
33 }
34}
35
36const offlineTreeRoot: Directory = {
37 files: new Map(),
38};
39
40const onlineTreeRoot: DirectoryWithStatus = {
41 files: new Map(),
42 status: "Unknown",
43};
44
45function mergeTrees(target: DirectoryWithStatus, other: DirectoryWithStatus) {
46 if (other.status === "Unknown") return;
47 target.status = other.status;
48 for (const [fileName, file] of other.files.entries()) {
49 const targetFile = target.files.get(fileName);
50 if (targetFile && "files" in targetFile && "files" in file) {
51 mergeTrees(targetFile, file);
52 } else {
53 target.files.set(fileName, file);
54 }
55 }
56}
57
58//TODO: improve typing to avoid "as"
59//overloads for typecheking
60function updateFileTree(path: string, file: MediaFile, offline: true): void;
61function updateFileTree(path: string, file: DirectoryWithStatus, offline: false): void;
62function updateFileTree(path: string, file: DirectoryWithStatus | MediaFile, offline: boolean): void {
63 if (offline && path === "") throw new Error("Trying to update root in offline mode");
64 const basePath = path.split("/").slice(0, -1).join("/");
65 const fileName = path.split("/").toReversed()[0];
66 const node = getDirInTree(basePath, true, offline);
67
68 if (offline) {
69 node.files.set(fileName, file);
70 } else {
71 if (path === "") mergeTrees(node as DirectoryWithStatus, file as DirectoryWithStatus);
72 else {
73 let target = node.files.get(fileName) as DirectoryWithStatus;
74 if (!target) target = { files: new Map(), status: "Unknown" };
75 mergeTrees(target as DirectoryWithStatus, file as DirectoryWithStatus);
76 node.files.set(fileName, target);
77 }
78 }
79}
80
81//overloads for typecheking
82function getDirInTree(path: string, create: true, offline: true): Directory;
83function getDirInTree(path: string, create: true, offline: false): DirectoryWithStatus;
84function getDirInTree(path: string, create: false, offline: true): Directory | null;
85function getDirInTree(path: string, create: false, offline: false): DirectoryWithStatus | null;
86function getDirInTree(path: string, create: true, offline: boolean): DirectoryWithStatus | Directory;
87function getDirInTree(path: string, create: false, offline: boolean): DirectoryWithStatus | Directory | null;
88
89// get a DirectoryNode from the directory tree, and optionally create it if it doesn't exist yet
90function getDirInTree(path: string, create: boolean, offline: boolean): DirectoryWithStatus | Directory | null {
91 let current = offline ? offlineTreeRoot : onlineTreeRoot;
92 if (path === "") return current;
93 const parts = path.split("/");
94
95 for (const part of parts) {
96 let next: DirectoryWithStatus | Directory | MediaFile | undefined = current.files.get(part);
97 if (!next || "metadata" in next) {
98 if (!create) return null;
99 //if mediafile already exists, we overwrite it with a directory anyway
100 if (offline) next = { files: new Map() };
101 else next = { files: new Map(), status: "Unknown" };
102 current.files.set(part, next);
103 }
104 current = next;
105 }
106 return current;
107}
108
109function deleteOfflinePath(path: string): void {
110 const basePath = path.split("/").slice(0, -1).join("/");
111 const fileName = path.split("/").toReversed()[0];
112
113 const node = getDirInTree(basePath, false, true);
114 if (!node) return;
115
116 node.files.delete(fileName);
117}
118
119export const clearDirectoryCache = (path: string) => {
120 const node = getDirInTree(path, false, false);
121 if (!node) return;
122 node.files.clear();
123 node.status = "Unknown";
124};
125
126async function listOnlineFiles(dir: string, recursive: boolean): Promise<FileListingWithStatus | undefined> {
127 const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {});
128 if (response?.status === 404) {
129 toast(`Directory not found: ${dir}`, "error");
130 return;
131 }
132 if (!response || !response.ok) {
133 toast(
134 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
135 "error",
136 );
137 return;
138 }
139
140 return parseMap(await response.text()) as FileListingWithStatus;
141}
142
143function listOfflineFiles(dir: string): FileListing {
144 const node = getDirInTree(dir, false, true);
145 if (!node || !node.files) return new Map();
146 return node.files;
147}
148
149function fullyKnown(dir: DirectoryWithStatus) {
150 if (dir.status === "Unknown") {
151 return false;
152 }
153 if (dir.files.values().some((value) => "files" in value && !fullyKnown(value))) {
154 return false;
155 }
156 return true;
157}
158
159//non-recursive listing should not include nested children
160function pruneChildren(listing: FileListing) {
161 const cloned = structuredClone(listing);
162 for (const [, item] of cloned.entries()) {
163 if ("files" in item) item.files.clear();
164 }
165 return cloned;
166}
167
168export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
169 if (offline) {
170 const files = listOfflineFiles(dir);
171 return recursive ? files : pruneChildren(files);
172 }
173 const cached = getDirInTree(dir, false, false);
174 if (cached !== null && cached.status === "Scanned" && (!recursive || fullyKnown(cached))) {
175 return recursive ? cached.files : pruneChildren(cached.files);
176 }
177 const files = await listOnlineFiles(dir, recursive);
178 if (!files) return new Map();
179 updateFileTree(dir, { files, status: "Scanned" }, false);
180 return recursive ? files : pruneChildren(files);
181}
182
183export async function getCover(dir: string, offline: boolean) {
184 const db = await openDB();
185 const dbResult = await db.get(CoverStore, dir);
186 if (dbResult !== undefined) {
187 const cached = dbResult as StoredCover;
188 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
189 }
190
191 if (offline) return undefined;
192
193 const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {});
194 if (!response || response.status === 404) {
195 db.put(CoverStore, { path: dir, data: null });
196 return undefined;
197 }
198 const blob = await response.blob();
199 db.put(CoverStore, { path: dir, data: blob });
200 return URL.createObjectURL(blob);
201}
202
203export const clearCoverCache = async (path: string) => {
204 const db = await openDB();
205 return db.delete(CoverStore, path);
206};
207
208export async function getSrc(path: string, options: AppOptions, offline: boolean) {
209 const db = await openDB();
210 const dbResult = (await db.get(FileBlobStore, path)) as StoredFileBlob;
211 if (dbResult) return URL.createObjectURL(dbResult.data);
212
213 if (offline) {
214 toast("This file is not cached, leave offline mode to play it", "error");
215 return undefined;
216 }
217 if (options.enableTranscoding) {
218 return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`;
219 }
220 return joinPath("/download", path);
221}
222
223export async function loadPlaylists() {
224 const db = await openDB();
225 return await db.listPlaylists(PlaylistStore);
226}
227
228export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
229 const db = await openDB();
230 playlist.items = playlist.items.map((item) => {
231 //some less space in the db by omitting ids
232 const { id, ...itemWithoutId } = item as PlaylistItem;
233 return itemWithoutId;
234 });
235 if (overwrite) await db.put(PlaylistStore, playlist);
236 else await db.add(PlaylistStore, playlist);
237}
238
239export async function deletePlaylist(name: string) {
240 const db = await openDB();
241 await db.delete(PlaylistStore, name);
242}
243
244export async function deleteFile(
245 fileName: string,
246 item: MediaFile | Directory,
247 basePath: string,
248 fetchFiles: ResourceActions<FileListing>,
249) {
250 const fullPath = joinPath(basePath, fileName);
251 if (!downloadStatusMap[fullPath]) return;
252
253 //TODO: can techinally remove the recursion, but need to update the signals
254 if (!("duration" in item)) {
255 const newBasePath = joinPath(basePath, fileName);
256 const result = listOfflineFiles(newBasePath);
257 for (const [childName, childItem] of result) {
258 await deleteFile(childName, childItem, newBasePath, fetchFiles);
259 }
260 }
261
262 deleteOfflinePath(fullPath);
263 const db = await openDB();
264 await db.delete(FileBlobStore, fullPath);
265 await db.delete(FileMetadataStore, fullPath);
266 setDownloadStatusMap(fullPath, undefined);
267 fetchFiles.refetch();
268}
269
270const downloadSemaphore = new AsyncSemaphore(1);
271
272async function downloadFile(
273 path: string,
274 currentDir: Accessor<string>,
275 options: AppOptions,
276 item: MediaFile,
277 fetchFiles: ResourceActions<FileListing>,
278): Promise<void> {
279 const initialProgress = downloadStatusMap[path];
280 if (typeof initialProgress === "object" && "bytes" in initialProgress) return; // download already in progress
281
282 const db = await openDB();
283 let mimeType: string | undefined;
284 try {
285 setDownloadStatusMap(path, { bytes: 0 });
286
287 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
288 let blob: Blob;
289 try {
290 await downloadSemaphore.acquire();
291 let res: Response;
292 if (options.enableTranscoding) {
293 res = await fetch(
294 `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}&disableChunkedTranscoding=${options.disableChunkedTranscoding}`,
295 );
296 } else {
297 res = await fetch(joinPath("/download", path));
298 }
299 //let the browser do the actual download
300 const blobPromise = res.clone().blob();
301 if (!res.ok || !res.body) throw res.statusText;
302 //we just iterate over the chunks to get the current progress
303 const reader = res.body.getReader();
304 mimeType = res.headers.get("content-type") ?? undefined;
305 let received = 0;
306
307 let lastUpdate = 0;
308 while (true) {
309 const { done, value } = await reader.read();
310 if (done) break;
311 received += value.length;
312
313 const now = Date.now();
314 if (now - lastUpdate < 250) continue;
315 lastUpdate = now;
316 setDownloadStatusMap(path, { bytes: received });
317 }
318 blob = await blobPromise;
319 } finally {
320 downloadSemaphore.release();
321 }
322
323 await db.put(FileBlobStore, {
324 path,
325 data: blob,
326 } as StoredFileBlob);
327 await db.put(FileMetadataStore, {
328 path,
329 duration: item.metadata.duration,
330 } as StoredFileMetadata);
331 updateFileTree(path, item, true);
332 setDownloadStatusMap(path, "SyncedFile");
333 const basePath = path.split("/").slice(0, -1).join("/");
334 const fileName = path.split("/").toReversed()[0];
335 if (currentDir() === basePath) {
336 fetchFiles.mutate((prev) => {
337 prev.set(fileName, item);
338 //new map to force rerender
339 return new Map(prev);
340 });
341 }
342 } catch (e) {
343 toast(`Downloading ${path} failed`, "error", e);
344 setDownloadStatusMap(path, initialProgress === "SyncedFile" ? "SyncedFile" : undefined);
345 }
346}
347
348export async function download(
349 fileName: string,
350 item: MediaFile | Directory,
351 basePath: string,
352 currentDir: Accessor<string>,
353 options: AppOptions,
354 fetchFiles: ResourceActions<FileListing>,
355) {
356 const fullPath = joinPath(basePath, fileName);
357 setDownloadStatusMap(fullPath, undefined);
358 if ("metadata" in item) {
359 await downloadFile(fullPath, currentDir, options, item, fetchFiles);
360 } else {
361 setDownloadStatusMap(fullPath, "DirectoryProgress");
362 await getCover(fullPath, false);
363 try {
364 const promises: Promise<unknown>[] = [];
365 for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
366 promises.push(download(childName, childItem, fullPath, currentDir, options, fetchFiles));
367 }
368 await Promise.allSettled(promises);
369 } finally {
370 setDownloadStatusMap(fullPath, undefined);
371 }
372 }
373}
374
375interface StoredFileBlob {
376 path: string;
377 data: Blob;
378}
379
380interface StoredFileMetadata {
381 path: string;
382 duration: number;
383}
384
385interface StoredCover {
386 path: string;
387 data: Blob | null;
388}
389
390//separate blobs and metadata, so we don't have to load all file contents at startup, which would consume huge amounts of ram, and be very slow
391const FileBlobStore = "fileblobs";
392const FileMetadataStore = "filemetadata";
393const CoverStore = "covers";
394const PlaylistStore = "playlists";
395
396async function openDB() {
397 return new Promise<DatabaseWrapper>((resolve, reject) => {
398 const req = indexedDB.open("music-files");
399 req.onerror = () => reject(req.error);
400 req.onupgradeneeded = () => {
401 const db = req.result;
402 if (!db.objectStoreNames.contains(FileBlobStore)) db.createObjectStore(FileBlobStore, { keyPath: "path" });
403 if (!db.objectStoreNames.contains(FileMetadataStore))
404 db.createObjectStore(FileMetadataStore, { keyPath: "path" });
405 if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" });
406 if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" });
407 };
408 req.onsuccess = () => resolve(wrapDB(req.result));
409 });
410}
411
412interface DatabaseWrapper {
413 get(store: typeof CoverStore | typeof FileBlobStore, key: string): Promise<unknown>;
414 put(store: typeof FileBlobStore, value: StoredFileBlob): Promise<void>;
415 put(store: typeof FileMetadataStore, value: StoredFileMetadata): Promise<void>;
416 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
417 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
418 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
419 delete(
420 store: typeof FileBlobStore | typeof CoverStore | typeof FileMetadataStore | typeof PlaylistStore,
421 key: string,
422 ): Promise<void>;
423 clear(store: string): Promise<void>;
424 listFiles(store: typeof FileMetadataStore): Promise<StoredFileMetadata[]>;
425 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
426 exist(store: typeof FileMetadataStore, key: string): Promise<boolean>;
427}
428
429function wrapDB(db: IDBDatabase): DatabaseWrapper {
430 return {
431 get: (store: string, key: string) =>
432 new Promise((res, rej) => {
433 const tx = db.transaction(store, "readonly");
434 const req = tx.objectStore(store).get(key);
435 req.onsuccess = () => res(req.result);
436 req.onerror = () => rej(req.error);
437 }),
438 put: (store: string, value: unknown) =>
439 new Promise<void>((res, rej) => {
440 const tx = db.transaction(store, "readwrite");
441 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
442 req.onsuccess = () => res();
443 req.onerror = () => rej(req.error);
444 }),
445 add: (store: string, value: unknown) =>
446 new Promise<void>((res, rej) => {
447 const tx = db.transaction(store, "readwrite");
448 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
449 req.onsuccess = () => res();
450 req.onerror = () => rej(req.error);
451 }),
452 delete: (store: string, key: string) =>
453 new Promise<void>((res, rej) => {
454 const tx = db.transaction(store, "readwrite");
455 const req = tx.objectStore(store).delete(key);
456 req.onsuccess = () => res();
457 req.onerror = () => rej(req.error);
458 }),
459 clear: (store: string) =>
460 new Promise<void>((res, rej) => {
461 const tx = db.transaction(store, "readwrite");
462 const req = tx.objectStore(store).clear();
463 req.onsuccess = () => res();
464 req.onerror = () => rej(req.error);
465 }),
466 listFiles: (store: typeof FileMetadataStore) =>
467 new Promise((res, rej) => {
468 const tx = db.transaction(store, "readonly");
469 const req = tx.objectStore(store).getAll();
470 req.onsuccess = () => res(req.result);
471 req.onerror = () => rej(req.error);
472 }),
473 listPlaylists: (store: typeof PlaylistStore) =>
474 new Promise((res, rej) => {
475 const tx = db.transaction(store, "readonly");
476 const req = tx.objectStore(store).getAll();
477 req.onsuccess = () => res(req.result);
478 req.onerror = () => rej(req.error);
479 }),
480 exist: (store: typeof FileMetadataStore, key: string) =>
481 new Promise((res, rej) => {
482 const tx = db.transaction(store, "readonly");
483 const req = tx.objectStore(store).count(key);
484 req.onsuccess = () => res(req.result !== 0);
485 req.onerror = () => rej(req.error);
486 }),
487 };
488}
489
490export async function isFileCached(path: string, status: DownloadStatus): Promise<boolean> {
491 if (status === "SyncedFile") return true;
492 //if in progress, it might already exist anyway and just is being re-synced right now
493 if (typeof status === "object" && "bytes" in status) {
494 const db = await openDB();
495 return await db.exist(FileMetadataStore, path);
496 }
497 return false;
498}