improve performance when many items are downloading
Mwebclient/src/components/DownloadManager.tsx
@@ -1,10 +1,20 @@
import { createEffect, createSignal, For, onMount, Show, untrack, type Accessor, type Setter } from "solid-js";
import {
batch,
createEffect,
createMemo,
createSignal,
For,
Show,
untrack,
type Accessor,
type Setter,
} from "solid-js";
import "./DownloadManager.css";
import "./Dialog.css";
import "toastify-js/src/toastify.css";
import { formatBytes } from "../utils";
import { ClearSVG, RefreshSVG, StopSVG } from "./SVG";
import { downloadStatusMap, setDownloadStatusMap, type Progress } from "../offline";
import { type DownloadStatus, downloadStatusMap, setDownloadStatusMap, type Progress } from "../offline";
import { createStore } from "solid-js/store";
interface DownloadManagerProps {
@@ -26,26 +36,14 @@ export default function DownloadManager(props: DownloadManagerProps) {
props.setShowDownloadManager(false);
}
const [downloadItems, setDownloadItems] = createStore<Record<string, Progress | undefined>>({});
createEffect(() => {
const paths = Object.keys(downloadStatusMap);
untrack(() => {
// Update the downloadItems store with the current status of each path
for (const path of paths) {
const status = downloadStatusMap[path];
if (typeof status === "object" && "status" in status) setDownloadItems(path, status);
else setDownloadItems(path, undefined);
}
// Remove any paths that no longer exist in downloadStatusMap
for (const path of Object.keys(downloadItems)) {
if (!Object.keys(downloadStatusMap).includes(path)) {
setDownloadItems(path, undefined);
}
}
});
const downloadItems = createMemo(() => {
return Object.entries(downloadStatusMap).filter(([_path, status]) => isProgress(status));
});
function isProgress(status: DownloadStatus | undefined): status is Progress {
return status !== undefined && typeof status === "object" && "status" in status;
}
return (
<dialog
style={{ height: "90%" }}
@@ -60,19 +58,25 @@ export default function DownloadManager(props: DownloadManagerProps) {
<button
title="Retry all failed/aborted downloads"
onClick={() => {
for (const status of Object.values(downloadItems)) {
if (status && "retry" in status) status.retry();
}
batch(() => {
for (const [, status] of downloadItems()) {
if (isProgress(status) && "retry" in status) status.retry();
}
});
}}
>
<RefreshSVG />
</button>
<button
title="stop all running downloads"
onClick={() => {
for (const status of Object.values(downloadItems)) {
if (status && typeof status.status === "number") status.abort();
}
onClick={async () => {
batch(async () => {
for (const [, status] of downloadItems()) {
if (isProgress(status) && typeof status.status === "number") {
status.abort();
}
}
});
}}
>
<StopSVG />
@@ -80,10 +84,12 @@ export default function DownloadManager(props: DownloadManagerProps) {
<button
title="Clear all failed/aborted downloads"
onClick={() => {
for (const [path, status] of Object.entries(downloadItems)) {
if (status && (status.status === "Aborted" || status.status === "Failed"))
setDownloadStatusMap(path, undefined);
}
batch(() => {
for (const [path, status] of downloadItems()) {
if (isProgress(status) && (status.status === "Aborted" || status.status === "Failed"))
setDownloadStatusMap(path, undefined);
}
});
}}
>
Clear
@@ -94,9 +100,9 @@ export default function DownloadManager(props: DownloadManagerProps) {
</div>
<hr />
<div class="downloadmanager-content">
<For each={Object.entries(downloadItems)}>
<For each={downloadItems()}>
{([path, status]) => {
if (!status) return;
if (!isProgress(status)) return;
const [showReason, setShowReason] = createSignal(false);
return (
<div
Mwebclient/src/components/Playlist.tsx
@@ -8,12 +8,13 @@ import {
onMount,
Show,
on,
batch,
} from "solid-js";
import "./Playlist.css";
import { getItemName, type PlayerState, type PlaylistItem, type AppOptions } from "../types";
import { formatTime, LocalStorageValues, toast } from "../utils";
import { ClearSVG, DownloadSVG, DownSVG, DragSVG, LinkSVG, SaveSVG, SyncSVG, UpSVG } from "./SVG";
import { savePlaylist, download, getStoredFiles, getDownloadStatus } from "../offline";
import { savePlaylist, download, storedFilesLive } from "../offline";
import { DraggableList } from "./DraggableList";
import type { FlatFileListing } from "../App";
@@ -70,14 +71,17 @@ export default function Playlist(props: PlaylistProps) {
const itemsToSync: PlaylistItem[] = [];
const playlist = props.playlist();
const unique = playlist.filter((item, pos) => playlist.indexOf(item) === pos);
const storedFiles = await getStoredFiles();
const storedFiles = storedFilesLive();
for (const item of unique) {
if (!storedFiles.includes(item.path)) itemsToSync.push(item);
}
const downloadPromises = itemsToSync.map((item) => {
const basePath = item.path.split("/").slice(0, -1).join("/");
const fileName = item.path.split("/").toReversed()[0];
return download(fileName, item, basePath, props.options, props.fetchFiles);
let downloadPromises!: Promise<void>[];
batch(() => {
downloadPromises = itemsToSync.map((item) => {
const basePath = item.path.split("/").slice(0, -1).join("/");
const fileName = item.path.split("/").toReversed()[0];
return download(fileName, item, basePath, props.options, props.fetchFiles);
});
});
await Promise.allSettled(downloadPromises);
toast("Playlist sync completed", "success");
@@ -115,14 +119,9 @@ export default function Playlist(props: PlaylistProps) {
const [syncedCount, setSyncedCount] = createSignal(0);
createEffect(
on(
() => props.playlist().map((item) => getDownloadStatus(item.path)),
async () => {
const playlist = props.playlist();
const storedFiles = await getStoredFiles();
setSyncedCount(playlist.filter((item) => storedFiles.includes(item.path)).length);
},
),
on([props.playlist, storedFilesLive], async ([playlist, storedFiles]) => {
setSyncedCount(playlist.filter((item) => storedFiles.includes(item.path)).length);
}),
);
onMount(() => {
const savedPlaylistButtons = localStorage.getItem(LocalStorageValues.playlistButtons);
Mwebclient/src/offline.ts
@@ -8,12 +8,12 @@ import type {
import { parseMap } from "music-server-shared/mapconversion";
import type { AppOptions, Playlist } from "./types";
import { joinPath, toast } from "./utils";
import type { ResourceActions } from "solid-js";
import { batch, from, observable, untrack, type ResourceActions } from "solid-js";
import { AsyncSemaphore } from "./semaphore";
import { encodePath } from "music-server-shared/utils";
import type { FlatFileListing } from "./App";
import { createStore } from "solid-js/store";
import Dexie from "dexie";
import Dexie, { liveQuery } from "dexie";
interface StoredFile {
path: string;
@@ -54,8 +54,10 @@ export type DownloadStatus = Progress | LoadingChildren | "Synced" | undefined;
export const [downloadStatusMap, setDownloadStatusMap] = createStore<Record<string, DownloadStatus>>({});
//creates entry if it doesn't exist, so reactivity works
export function getDownloadStatus(path: string): DownloadStatus | undefined {
if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
export function getDownloadStatus(path: string): DownloadStatus {
untrack(() => {
if (!Object.keys(downloadStatusMap).includes(path)) setDownloadStatusMap(path, undefined);
});
return downloadStatusMap[path];
}
@@ -386,9 +388,11 @@ export async function deleteFile(
await db.files.bulkDelete(toDelete); //delete from database
deleteInOfflineTree(fullPath); //delete node from tree
for (const path of toDelete) {
setDownloadStatusMap(path, undefined); // set all children to unsynced
}
batch(() => {
for (const path of toDelete) {
setDownloadStatusMap(path, undefined); // set all children to unsynced
}
});
fetchFiles.refetch();
} finally {
stopProgress(fullPath, undefined);
@@ -406,19 +410,21 @@ async function downloadFile(
fetchFiles: ResourceActions<FlatFileListing>,
redownload: boolean,
): Promise<void> {
const retry = () => {
setDownloadStatusMap(path, undefined);
return downloadFile(path, options, item, fetchFiles, redownload);
};
const retry = () => downloadFile(path, options, item, fetchFiles, redownload);
const controller = new AbortController();
let abortResolve!: () => void;
const abortPromise = new Promise<void>((resolve) => {
abortResolve = resolve;
}).then(() => {
controller.abort();
const abortPromise = new Promise<void>((outerResolve) => {
abortResolve = () => {
controller.abort();
stopProgress(
path,
// if it was already synced, restore it
initialProgress === "Synced" ? "Synced" : { status: "Aborted", retry },
);
outerResolve();
};
});
// file already downloaded or download already in progress
const initialProgress = downloadStatusMap[path];
if (
@@ -472,20 +478,17 @@ async function downloadFile(
downloadSemaphore.release();
}
} catch (e) {
if (e && typeof e === "object" && "name" in e && e.name === "AbortError") return;
stopProgress(
path,
// if it was already synced, restore it
initialProgress === "Synced"
? "Synced"
: //if it was manually aborted, set as aborted
e && typeof e === "object" && "name" in e && e.name === "AbortError"
? { status: "Aborted", retry }
: //otherwise set as failed and set reason to the error message
{
status: "Failed",
reason: e ? e.toString() : "Unknown error",
retry,
},
: {
status: "Failed",
reason: e ? e.toString() : "Unknown error",
retry,
},
);
}
}
@@ -507,9 +510,12 @@ export async function download(
await getCover(fullPath, false);
//start downloads for all children to allow them to run in parallel, then wait for the results
const promises: Promise<unknown>[] = [];
for (const [childName, childItem] of await listFiles(fullPath, false, false)) {
promises.push(download(childName, childItem, fullPath, options, fetchFiles, redownload));
}
const listing = await listFiles(fullPath, false, false);
batch(() => {
for (const [childName, childItem] of listing) {
promises.push(download(childName, childItem, fullPath, options, fetchFiles, redownload));
}
});
await Promise.allSettled(promises);
} finally {
updateRecursiveStatus(fullPath, -1, true);
@@ -517,6 +523,7 @@ export async function download(
}
}
export async function getStoredFiles() {
return await db.files.toCollection().primaryKeys();
}
export const storedFilesLive = from(
liveQuery(async () => await db.files.toCollection().keys()),
[],
);