pico-pixel-player

pico-pixel-player/webclient/src/offline.ts
offline.ts
⎇
Raw
1import type { AudioFile, Directory, FileListing } from "music-server-shared/types";
2import type { AppOptions, Playlist, PlaylistItem } from "./types";
3import { joinPath, toast } from "./utils";
4import { createSignal, type ResourceActions, type Signal } from "solid-js";
5import { AsyncSemaphore } from "./semaphore";
6
7interface Progress {
8 bytes: number;
9}
10
11export type SignalMapEntry = undefined | Progress | "DirectoryProgress" | "CachedFile";
12export const signalMap = new Map<string, Signal<SignalMapEntry>>();
13
14function updateOfflineFileTree(path: string, duration: number): void {
15 const basePath = path.split("/").slice(0, -1).join("/");
16 const name = path.split("/").toReversed()[0];
17 const node = getDirInTree(basePath, true);
18
19 const existingFileIndex = node.files.findIndex((f) => f.name === name);
20
21 const file = { name: name, duration: duration };
22 if (existingFileIndex >= 0) {
23 node.files[existingFileIndex] = file;
24 } else {
25 node.files.push(file);
26 }
27}
28
29export async function loadDataFromDB() {
30 try {
31 const db = await openDB();
32 const files = await db.listFiles(FileMetadataStore);
33 for (const file of files) {
34 signalMap.set(file.path, createSignal<SignalMapEntry>("CachedFile"));
35 updateOfflineFileTree(file.path, file.duration);
36 }
37 } catch (e) {
38 toast("Loading offline files failed", "error", e);
39 console.error(e);
40 }
41}
42
43interface DirectoryNode {
44 directories: Record<string, DirectoryNode>;
45 files: AudioFile[];
46}
47
48const directoryTree: DirectoryNode = {
49 directories: {},
50 files: [],
51};
52
53// get a DirectoryNode from the directory tree, and optionally create it if it doesn't exist yet
54function getDirInTree(path: string, create: true): DirectoryNode;
55function getDirInTree(path: string, create: false): DirectoryNode | null;
56function getDirInTree(path: string, create: boolean): DirectoryNode | null {
57 const parts = path.split("/").filter(Boolean);
58 let current = directoryTree;
59
60 for (const part of parts) {
61 if (!current.directories[part]) {
62 if (!create) return null;
63 current.directories[part] = { directories: {}, files: [] };
64 }
65 current = current.directories[part];
66 }
67 return current;
68}
69
70function deletePath(path: string): void {
71 const basePath = path.split("/").slice(0, -1).join("/");
72 const node = getDirInTree(basePath, false);
73 if (!node) return;
74
75 const name = path.split("/").toReversed()[0];
76 delete node.directories[name];
77 node.files = node.files.filter((file) => file.name !== name);
78}
79
80type FileEntry = { name: string } & ({ type: "Directory" } | { type: "File"; duration: number });
81const directoryCache = new Map<string, FileEntry[]>();
82export const clearDirectoryCache = (path: string) => directoryCache.delete(path);
83
84async function requestFileList(dir: string, recursive: boolean): Promise<FileListing | undefined> {
85 const response = await fetch(`/list/${dir}?recursive=${recursive}`).catch(() => {});
86 if (response?.status === 404) {
87 toast(`Directory not found: ${dir}`, "error");
88 return;
89 }
90 if (!response || !response.ok) {
91 toast(
92 `Fetching files for ${dir === "" ? "root" : dir} failed${response ? `: ${response.statusText}` : ""}`,
93 "error",
94 );
95 return;
96 }
97
98 const data: Directory = await response.json();
99 return data.files;
100}
101
102function saveFileListToCache(basePath: string, files: FileListing, recursive: boolean) {
103 const converted: FileEntry[] = files.map((item) => {
104 if ("files" in item) {
105 if (recursive) saveFileListToCache(joinPath(basePath, item.name), item.files, true);
106 return { ...item, type: "Directory" };
107 }
108 return { ...item, type: "File" };
109 });
110 directoryCache.set(basePath, converted);
111}
112
113function cachedToDirectory(fullPath: string, entries: FileEntry[] | undefined, recursive: boolean): FileListing | null {
114 if (!entries) return null;
115 const result = [];
116 for (const item of entries) {
117 const newPath = joinPath(fullPath, item.name);
118 let children = undefined;
119 if (item.type !== "File") {
120 children = recursive ? cachedToDirectory(newPath, directoryCache.get(newPath), true) : [];
121 if (!children) return null;
122 result.push({ name: item.name, files: children });
123 } else {
124 result.push({ name: item.name, duration: item.duration });
125 }
126 }
127 return result;
128}
129
130function listOfflineFiles(dir: string, recursive: boolean): FileListing {
131 const node = getDirInTree(dir, false);
132 if (!node) return [];
133 const entries: FileListing = [];
134
135 entries.push(...node.files);
136
137 if (recursive) {
138 for (const [name, childNode] of Object.entries(node.directories)) {
139 const directoryEntry: Directory = {
140 name,
141 files: childNode.files,
142 };
143 directoryEntry.files = listOfflineFiles(joinPath(dir, name), true);
144
145 entries.push(directoryEntry);
146 }
147 } else {
148 entries.push(...Object.keys(node.directories).map((dir) => ({ name: dir, files: [] })));
149 }
150
151 return entries.sort((a, b) => a.name.localeCompare(b.name));
152}
153
154export async function listFiles(dir: string, offline: boolean, recursive: boolean): Promise<FileListing> {
155 if (offline) {
156 const res = listOfflineFiles(dir, recursive);
157 return res;
158 }
159 const cached = cachedToDirectory(dir, directoryCache.get(dir), recursive);
160 if (cached !== null) return cached;
161 const files = await requestFileList(dir, recursive);
162 if (!files) return [];
163 saveFileListToCache(dir, files, recursive);
164 return files;
165}
166
167export async function getCover(dir: string, offline: boolean) {
168 const db = await openDB();
169 const dbResult = await db.get(CoverStore, dir);
170 if (dbResult !== undefined) {
171 const cached = dbResult as StoredCover;
172 return cached.data === null ? undefined : URL.createObjectURL(new Blob([cached.data]));
173 }
174
175 if (offline) return undefined;
176
177 const response = await fetch(`/cover/${dir}`, { method: "GET" }).catch(() => {});
178 if (!response || response.status === 404) {
179 db.put(CoverStore, { path: dir, data: null });
180 return undefined;
181 }
182 const blob = await response.blob();
183 db.put(CoverStore, { path: dir, data: new Uint8Array(await blob.arrayBuffer()) });
184 return URL.createObjectURL(blob);
185}
186
187export const clearCoverCache = async (path: string) => {
188 const db = await openDB();
189 return db.delete(CoverStore, path);
190};
191
192export async function getSrc(path: string, options: AppOptions) {
193 const db = await openDB();
194 const dbResult = (await db.get(FileBlobStore, path)) as StoredFileBlob;
195 if (dbResult) return URL.createObjectURL(new Blob([dbResult.data]));
196
197 if (options.enableTranscoding) {
198 return `${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}`;
199 }
200 return joinPath("/download", path);
201}
202
203export async function loadPlaylists() {
204 const db = await openDB();
205 return await db.listPlaylists(PlaylistStore);
206}
207
208export async function savePlaylist(playlist: Playlist, overwrite: boolean) {
209 const db = await openDB();
210 playlist.items = playlist.items.map((item) => {
211 //some less space in the db by omitting ids
212 const { id, ...itemWithoutId } = item as PlaylistItem;
213 return itemWithoutId;
214 });
215 if (overwrite) await db.put(PlaylistStore, playlist);
216 else await db.add(PlaylistStore, playlist);
217}
218
219export async function deletePlaylist(name: string) {
220 const db = await openDB();
221 await db.delete(PlaylistStore, name);
222}
223
224export async function deleteFile(
225 item: AudioFile | Directory,
226 basePath: string,
227 fetchFiles: ResourceActions<FileListing>,
228 listing?: FileListing,
229) {
230 console.log(item);
231 const fullPath = joinPath(basePath, item.name);
232 const signal = signalMap.get(fullPath);
233 if (!signal) return;
234
235 if (!("duration" in item)) {
236 const newBasePath = joinPath(basePath, item.name);
237 const result = listing ?? listOfflineFiles(newBasePath, true);
238 for (const childItem of result) {
239 await deleteFile(childItem, newBasePath, fetchFiles, "files" in childItem ? childItem.files : undefined);
240 }
241 }
242
243 deletePath(fullPath);
244 const db = await openDB();
245 await db.delete(FileBlobStore, fullPath);
246 await db.delete(FileMetadataStore, fullPath);
247 signalMap.get(fullPath)?.[1](undefined);
248 fetchFiles.refetch();
249 console.log("here");
250}
251
252const downloadSemaphore = new AsyncSemaphore(1);
253
254async function downloadFile(
255 path: string,
256 progressSignal: Signal<SignalMapEntry>,
257 options: AppOptions,
258 duration: number,
259 fetchFiles: ResourceActions<FileListing>,
260): Promise<void> {
261 const [getProgress, setProgress] = progressSignal;
262 const progress = getProgress();
263 if (typeof progress === "object" && "bytes" in progress) return; // download already in progress
264
265 const db = await openDB();
266 try {
267 setProgress({ bytes: 0 });
268
269 downloadSemaphore.setMaxConcurrency(options.enableTranscoding ? options.maxTranscodes : options.maxDownloads);
270 const chunks: Uint8Array[] = [];
271 try {
272 await downloadSemaphore.acquire();
273 let res: Response;
274 if (options.enableTranscoding) {
275 res = await fetch(`${joinPath("/transcode", path)}?bitrate=${options.bitrate}&format=${options.format}`);
276 } else {
277 res = await fetch(joinPath("/download", path));
278 }
279 if (!res.ok || !res.body) throw res.statusText;
280 const reader = res.body.getReader();
281 let received = 0;
282
283 let lastUpdate = 0;
284 while (true) {
285 const { done, value } = await reader.read();
286 if (done) break;
287 chunks.push(value);
288 received += value.length;
289
290 const now = Date.now();
291 if (now - lastUpdate < 250) continue;
292 lastUpdate = now;
293 setProgress({ bytes: received });
294 }
295 } finally {
296 downloadSemaphore.release();
297 }
298
299 const data = concatChunks(chunks);
300
301 await db.put(FileBlobStore, {
302 path,
303 data,
304 } as StoredFileBlob);
305 await db.put(FileMetadataStore, {
306 path,
307 duration,
308 } as StoredFileMetadata);
309 updateOfflineFileTree(path, duration);
310 setProgress("CachedFile");
311 fetchFiles.refetch();
312 } catch (e) {
313 toast(`Downloading ${path} failed`, "error", e);
314 setProgress(undefined);
315 }
316}
317
318export async function download(
319 item: AudioFile | Directory,
320 basePath: string,
321 options: AppOptions,
322 fetchFiles: ResourceActions<FileListing>,
323 listing?: FileListing,
324) {
325 const fullPath = joinPath(basePath, item.name);
326 let signal = signalMap.get(fullPath);
327 if (!signal) {
328 signal = createSignal<SignalMapEntry>();
329 signalMap.set(fullPath, signal);
330 }
331 const [, setProgress] = signal;
332 if ("duration" in item) {
333 await downloadFile(fullPath, signal, options, item.duration, fetchFiles);
334 } else {
335 getCover(fullPath, false);
336 setProgress("DirectoryProgress");
337 try {
338 const result = listing ?? (await listFiles(fullPath, false, true));
339 const promises: Promise<unknown>[] = [];
340 for (const childItem of result) {
341 promises.push(
342 download(childItem, fullPath, options, fetchFiles, "files" in childItem ? childItem.files : undefined),
343 );
344 }
345 await Promise.allSettled(promises);
346 } finally {
347 setProgress(undefined);
348 }
349 }
350}
351
352interface StoredFileBlob {
353 path: string;
354 data: Uint8Array;
355}
356
357interface StoredFileMetadata {
358 path: string;
359 duration: number;
360}
361
362interface StoredCover {
363 path: string;
364 data: Uint8Array | null;
365}
366
367//separate blobs and metadata, so we don't have to load all file contents at startup, which would consume huge amounts of ram, and be very slow
368const FileBlobStore = "fileblobs";
369const FileMetadataStore = "filemetadata";
370const CoverStore = "covers";
371const PlaylistStore = "playlists";
372
373async function openDB() {
374 return new Promise<DatabaseWrapper>((resolve, reject) => {
375 const req = indexedDB.open("music-files");
376 req.onerror = () => reject(req.error);
377 req.onupgradeneeded = () => {
378 const db = req.result;
379 if (!db.objectStoreNames.contains(FileBlobStore)) db.createObjectStore(FileBlobStore, { keyPath: "path" });
380 if (!db.objectStoreNames.contains(FileMetadataStore))
381 db.createObjectStore(FileMetadataStore, { keyPath: "path" });
382 if (!db.objectStoreNames.contains(CoverStore)) db.createObjectStore(CoverStore, { keyPath: "path" });
383 if (!db.objectStoreNames.contains(PlaylistStore)) db.createObjectStore(PlaylistStore, { keyPath: "name" });
384 };
385 req.onsuccess = () => resolve(wrapDB(req.result));
386 });
387}
388
389interface DatabaseWrapper {
390 get(store: typeof CoverStore | typeof FileBlobStore, key: string): Promise<unknown>;
391 put(store: typeof FileBlobStore, value: StoredFileBlob): Promise<void>;
392 put(store: typeof FileMetadataStore, value: StoredFileMetadata): Promise<void>;
393 put(store: typeof CoverStore, value: StoredCover): Promise<void>;
394 put(store: typeof PlaylistStore, value: Playlist): Promise<void>;
395 add(store: typeof PlaylistStore, value: Playlist): Promise<void>;
396 delete(
397 store: typeof FileBlobStore | typeof CoverStore | typeof FileMetadataStore | typeof PlaylistStore,
398 key: string,
399 ): Promise<void>;
400 clear(store: string): Promise<void>;
401 listFiles(store: typeof FileMetadataStore): Promise<StoredFileMetadata[]>;
402 listPlaylists(store: typeof PlaylistStore): Promise<Playlist[]>;
403}
404
405function wrapDB(db: IDBDatabase): DatabaseWrapper {
406 return {
407 get: (store: string, key: string) =>
408 new Promise((res, rej) => {
409 const tx = db.transaction(store, "readonly");
410 const req = tx.objectStore(store).get(key);
411 req.onsuccess = () => res(req.result);
412 req.onerror = () => rej(req.error);
413 }),
414 put: (store: string, value: unknown) =>
415 new Promise<void>((res, rej) => {
416 const tx = db.transaction(store, "readwrite");
417 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).put(value);
418 req.onsuccess = () => res();
419 req.onerror = () => rej(req.error);
420 }),
421 add: (store: string, value: unknown) =>
422 new Promise<void>((res, rej) => {
423 const tx = db.transaction(store, "readwrite");
424 const req: IDBRequest<IDBValidKey> = tx.objectStore(store).add(value);
425 req.onsuccess = () => res();
426 req.onerror = () => rej(req.error);
427 }),
428 delete: (store: string, key: string) =>
429 new Promise<void>((res, rej) => {
430 const tx = db.transaction(store, "readwrite");
431 const req = tx.objectStore(store).delete(key);
432 req.onsuccess = () => res();
433 req.onerror = () => rej(req.error);
434 }),
435 clear: (store: string) =>
436 new Promise<void>((res, rej) => {
437 const tx = db.transaction(store, "readwrite");
438 const req = tx.objectStore(store).clear();
439 req.onsuccess = () => res();
440 req.onerror = () => rej(req.error);
441 }),
442 listFiles: (store: typeof FileMetadataStore) =>
443 new Promise((res, rej) => {
444 const tx = db.transaction(store, "readonly");
445 const req = tx.objectStore(store).getAll();
446 req.onsuccess = () => res(req.result);
447 req.onerror = () => rej(req.error);
448 }),
449 listPlaylists: (store: typeof PlaylistStore) =>
450 new Promise((res, rej) => {
451 const tx = db.transaction(store, "readonly");
452 const req = tx.objectStore(store).getAll();
453 req.onsuccess = () => res(req.result);
454 req.onerror = () => rej(req.error);
455 }),
456 };
457}
458
459function concatChunks(chunks: Uint8Array[]): Uint8Array {
460 let total = 0;
461 for (const c of chunks) total += c.length;
462 const result = new Uint8Array(total);
463 let offset = 0;
464 for (const c of chunks) {
465 result.set(c, offset);
466 offset += c.length;
467 }
468 return result;
469}
470