Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(resources): show the download's progress on each resource's row
Elian Doranaf0b58b1e36e
diff --git a/apps/client/src/widgets/dialogs/resource_download.css b/apps/client/src/widgets/dialogs/resource_download.cssindex 4f83a23e80..7a4a69ac99 100644--- a/apps/client/src/widgets/dialogs/resource_download.css+++ b/apps/client/src/widgets/dialogs/resource_download.css@@ -27,16 +27,13 @@}.resource-download-progress {- progress {- width: 100%;- }+ margin-top: 0.5em;.resource-download-progress-detail {display: flex;justify-content: space-between;gap: 1em;- color: var(--muted-text-color);- font-size: 0.9em;+ margin-top: 0.35em;font-variant-numeric: tabular-nums;}}diff --git a/apps/client/src/widgets/dialogs/resource_download.spec.tsx b/apps/client/src/widgets/dialogs/resource_download.spec.tsxindex 5a3974ec39..a035e125ec 100644--- a/apps/client/src/widgets/dialogs/resource_download.spec.tsx+++ b/apps/client/src/widgets/dialogs/resource_download.spec.tsx@@ -78,17 +78,43 @@ describe("ResourceDownloadDialog", () => {await click("resource_download.download");expect(mocks.post).toHaveBeenCalledExactlyOnceWith("resources/download",{ ids: [ "transcription.model" ] });- expect(container.querySelector("progress")?.value).toBe(200);+ expect(row?.querySelector(".resource-download-icon")?.classList).toContain("bx-loader-circle");+ expect(row?.querySelector("[role=progressbar]")?.getAttribute("aria-valuenow")).toBe("200");await send(progress({ receivedBytes: 600, file: "encoder.int8.onnx" }));- expect(container.querySelector("progress")?.value).toBe(600);- expect(text(".resource-download-progress")).toContain("encoder.int8.onnx");+ expect(row?.querySelector("[role=progressbar]")?.getAttribute("aria-valuenow")).toBe("600");+ expect(row?.querySelector(".resource-download-progress")?.textContent)+ .toContain("encoder.int8.onnx");await send(progress({ status: "succeeded", receivedBytes: 700 }));await hidden();expect(results).toEqual([ true ]);});+ it("marks each resource done as the download moves on, the one downloading with its bar", async () => {+ const vad = { ...STATUS.resources[0], id: "transcription.vad", title: "VAD", downloadedSize: 0 };+ mocks.post.mockResolvedValue(progress());+ await open({ ...STATUS, resources: [ ...STATUS.resources, vad ] });+ await click("resource_download.download");++ await send(progress({ resources: [+ { id: "transcription.model", receivedBytes: 700, totalBytes: 700, done: true },+ { id: "transcription.vad", receivedBytes: 100, totalBytes: 700, done: false }+ ] }));+ const [ first, second ] = container.querySelectorAll(".resource-download-item");+ expect(first.querySelector(".resource-download-icon")?.classList).toContain("bx-check-circle");+ expect(first.querySelector("[role=progressbar]")).toBeNull();+ expect(second.querySelector(".resource-download-icon")?.classList).toContain("bx-loader-circle");+ expect(second.querySelector("[role=progressbar]")?.getAttribute("aria-valuenow")).toBe("100");++ await send(progress({ status: "failed", error: "offline", resources: [+ { id: "transcription.model", receivedBytes: 700, totalBytes: 700, done: true },+ { id: "transcription.vad", receivedBytes: 100, totalBytes: 700, done: false }+ ] }));+ expect(second.querySelector(".resource-download-icon")?.classList).toContain("bx-circle");+ expect(container.querySelector("[role=progressbar]")).toBeNull();+ });+it("keeps an end reported before the request that started the download returns", async () => {let respond = (_value: ResourceDownloadProgress) => {};mocks.post.mockReturnValue(new Promise((resolve) => respond = resolve));@@ -165,13 +191,20 @@ describe("ResourceDownloadDialog", () => {});function progress(overrides: Partial<ResourceDownloadProgress> = {}): ResourceDownloadProgress {+ const receivedBytes = overrides.receivedBytes ?? 0;return {downloadId: "d1",resourceIds: [ "transcription.model" ],status: "downloading",- receivedBytes: 0,+ receivedBytes,totalBytes: 700,file: null,+ resources: [ {+ id: "transcription.model",+ receivedBytes,+ totalBytes: 700,+ done: overrides.status === "succeeded"+ } ],...overrides};}diff --git a/apps/client/src/widgets/dialogs/resource_download.tsx b/apps/client/src/widgets/dialogs/resource_download.tsxindex 053582f8de..a1b63f3f88 100644--- a/apps/client/src/widgets/dialogs/resource_download.tsx+++ b/apps/client/src/widgets/dialogs/resource_download.tsx@@ -1,7 +1,8 @@import "./resource_download.css";import type {- ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse, WebSocketMessage+ ResourceDownloadProgress, ResourceDownloadState, ResourceInfo, ResourcesStatusResponse,+ WebSocketMessage} from "@triliumnext/commons";import { useEffect, useRef, useState } from "preact/hooks";@@ -15,6 +16,7 @@ import { Card, CardSection } from "../react/Card";import { useTriliumEvent } from "../react/hooks";import Icon from "../react/Icon";import Modal from "../react/Modal";+import ProgressBar from "../react/ProgressBar";export interface ResourceDownloadDialogOptions {ids: string[];@@ -109,8 +111,9 @@ export default function ResourceDownloadDialog() {)}</>}>- {opts && <ResourceList status={opts.status} remaining={remaining} />}- {isDownloading && <DownloadProgress download={download} />}+ {opts && (+ <ResourceList status={opts.status} remaining={remaining} download={download} />+ )}{lacksSpace && (<ExtendedAdmonitiontype="caution"@@ -136,15 +139,32 @@ export default function ResourceDownloadDialog() {);}-function ResourceList({ status, remaining }: { status: ResourcesStatusResponse; remaining: number }) {+interface ResourceListProps {+ status: ResourcesStatusResponse;+ remaining: number;+ download: ResourceDownloadProgress | null;+}++function ResourceList({ status, remaining, download }: ResourceListProps) {const missing = status.resources.filter(({ installed }) => !installed);const unitOf = Math.max(remaining, ...missing.map(({ size }) => size));+ const activeId = download?.status === "downloading"+ ? download.resources.find(({ done }) => !done)?.id+ : undefined;return (<><p>{t("resource_download.intro")}</p><Card className="resource-download-list">- {missing.map((resource) => <ResourceItem key={resource.id} resource={resource} unitOf={unitOf} />)}+ {missing.map((resource) => (+ <ResourceItem+ key={resource.id}+ resource={resource}+ unitOf={unitOf}+ state={download?.resources.find(({ id }) => id === resource.id)}+ file={resource.id === activeId ? download?.file ?? null : undefined}+ />+ ))}<CardSection className="tn-card-option resource-download-total"><div className="tn-card-option-label"><span className="tn-card-option-title">{t("resource_download.total")}</span>@@ -165,11 +185,25 @@ function ResourceList({ status, remaining }: { status: ResourcesStatusResponse;);}-/** One resource as a settings row: its icon, what it is and where it comes from, and its size. */-function ResourceItem({ resource, unitOf }: { resource: ResourceInfo; unitOf: number }) {+interface ResourceItemProps {+ resource: ResourceInfo;+ unitOf: number;+ /** How far a download has got with the resource, once one has started. */+ state?: ResourceDownloadState;+ /** The file being downloaded, `null` before the first one, while this resource downloads. */+ file?: string | null;+}++/**+ * One resource as a settings row: its icon, what it is and where it comes from, and its size.+ * Once a download starts, the icon shows whether the resource is done, downloading or waiting, as+ * the setup's sync steps do, and the resource downloading carries its own progress bar.+ */+function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {+ const isActive = file !== undefined;return (<CardSection className="tn-card-option resource-download-item">- <Icon icon={resource.icon} className="resource-download-icon" />+ <Icon icon={getRowIcon(resource, state, isActive)} className="resource-download-icon" /><div className="tn-card-option-label"><span className="tn-card-option-title">{resource.title}</span><small className="tn-card-option-description">{resource.description}</small>@@ -186,6 +220,7 @@ function ResourceItem({ resource, unitOf }: { resource: ResourceInfo; unitOf: nu})}</>}</small>+ {isActive && state && <RowProgress state={state} file={file ?? null} />}</div><span className="tn-card-option-value resource-download-size">{formatSize(resource.size, 2, unitOf)}@@ -194,23 +229,34 @@ function ResourceItem({ resource, unitOf }: { resource: ResourceInfo; unitOf: nu);}-function DownloadProgress({ download }: { download: ResourceDownloadProgress }) {+function RowProgress({ state, file }: { state: ResourceDownloadState; file: string | null }) {return (<div className="resource-download-progress">- <progress value={download.receivedBytes} max={download.totalBytes} />- <div className="resource-download-progress-detail">- <span>{download.file && t("resource_download.current_file", { file: download.file })}</span>+ <ProgressBar value={state.receivedBytes} max={state.totalBytes} />+ <small className="tn-card-option-description resource-download-progress-detail">+ <span>{file && t("resource_download.current_file", { file })}</span><span>{t("resource_download.progress", {- received: formatSize(download.receivedBytes, 1),- total: formatSize(download.totalBytes)+ received: formatSize(state.receivedBytes, 2, state.totalBytes),+ total: formatSize(state.totalBytes)})}</span>- </div>+ </small></div>);}+function getRowIcon(resource: ResourceInfo, state: ResourceDownloadState | undefined,+ isActive: boolean) {+ if (!state) {+ return resource.icon;+ }+ if (state.done) {+ return "bx bx-check-circle";+ }+ return isActive ? "bx bx-loader-circle bx-spin" : "bx bx-circle";+}+/** The bytes still to download for the resources not installed. */function getRemainingSize(status: ResourcesStatusResponse) {return status.resourcesdiff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsindex 732fa1a5c2..b3d6f72641 100644--- a/apps/server/src/services/resources.spec.ts+++ b/apps/server/src/services/resources.spec.ts@@ -90,6 +90,17 @@ describe("ResourceManager", () => {file: null});expect(reports.some(({ file }) => file === "model.bin")).toBe(true);+ expect(started.resources).toEqual([+ { id: "test.model", receivedBytes: 0, totalBytes: MODEL.length + TOKENS.length, done: false }+ ]);+ expect(reports.some(({ resources }) => !resources[0].done && resources[0].receivedBytes > 0))+ .toBe(true);+ expect(finished.resources).toEqual([ {+ id: "test.model",+ receivedBytes: MODEL.length + TOKENS.length,+ totalBytes: MODEL.length + TOKENS.length,+ done: true+ } ]);const directory = resourceDirectory();expect(path.basename(directory)).toMatch(/^test\.model-[0-9a-f]{12}$/);expect(manager.getPath("test.model")).toBe(directory);diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsindex 87dd8fa4ad..8783163e5a 100644--- a/apps/server/src/services/resources.ts+++ b/apps/server/src/services/resources.ts@@ -116,7 +116,7 @@ export class ResourceManager {const missing = ids.map((id) => this.getDefinition(id)).filter(({ id }) => !this.getPath(id));const running = this.findDownload(ids);if (running) {- return running.progress;+ return copyProgress(running.progress);}const download: Download = {@@ -126,7 +126,10 @@ export class ResourceManager {status: missing.length ? "downloading" : "succeeded",receivedBytes: 0,totalBytes: sumSizes(missing.flatMap(({ files }) => files)),- file: null+ file: null,+ resources: missing.map(({ id, files }) => ({+ id, receivedBytes: 0, totalBytes: sumSizes(files), done: false+ }))},controller: new AbortController()};@@ -134,7 +137,7 @@ export class ResourceManager {this.downloads.set(download.progress.downloadId, download);void this.runDownload(download, missing);}- return { ...download.progress };+ return copyProgress(download.progress);}cancelDownload(downloadId: string) {@@ -147,28 +150,33 @@ export class ResourceManager {const report = (force: boolean) => {if (force || Date.now() - lastReported >= PROGRESS_INTERVAL_MS) {lastReported = Date.now();- this.broadcast({ ...progress });+ this.broadcast(copyProgress(progress));}};try {- for (const definition of definitions) {+ for (const [ index, definition ] of definitions.entries()) {+ const state = progress.resources[index];+ const receive = (bytes: number) => {+ progress.receivedBytes += bytes;+ state.receivedBytes += bytes;+ };const directory = this.getDirectory(definition);const verified = readVerifiedFiles(directory);for (const file of definition.files) {const destination = path.join(directory, file.path);if (isVerified(verified, destination, file)) {- progress.receivedBytes += file.size;+ receive(file.size);continue;}progress.file = path.basename(file.path);report(true);await fs.promises.mkdir(path.dirname(destination), { recursive: true });if (await hasContent(destination, file)) {- progress.receivedBytes += file.size;+ receive(file.size);} else {await downloadFile(file, destination, controller.signal, (bytes) => {- progress.receivedBytes += bytes;+ receive(bytes);report(false);});}@@ -176,6 +184,8 @@ export class ResourceManager {verified[file.path] = { size, mtimeMs };await writeVerifiedFiles(directory, verified);}+ state.done = true;+ report(true);getLog().info(`Resources: installed ${definition.id} in ${directory}.`);}progress.status = "succeeded";@@ -403,6 +413,11 @@ async function hasContent(destination: string, file: ResourceFile) {return hash.digest("hex") === file.sha256;}+/** A snapshot of `progress`, which the running download goes on changing. */+function copyProgress(progress: ResourceDownloadProgress): ResourceDownloadProgress {+ return { ...progress, resources: progress.resources.map((state) => ({ ...state })) };+}+function sumSizes(files: ResourceFile[]) {return files.reduce((total, { size }) => total + size, 0);}diff --git a/packages/commons/src/lib/resources.ts b/packages/commons/src/lib/resources.tsindex a2f63c3234..a006c330d1 100644--- a/packages/commons/src/lib/resources.ts+++ b/packages/commons/src/lib/resources.ts@@ -22,6 +22,14 @@ export interface ResourceInfo {installed: boolean;}+/** How far a download has got with one of the resources it fetches. */+export interface ResourceDownloadState {+ id: string;+ receivedBytes: number;+ totalBytes: number;+ done: boolean;+}+/** A download as it runs, and as the `resource-download` WebSocket message reports it. */export interface ResourceDownloadProgress {downloadId: string;@@ -33,6 +41,8 @@ export interface ResourceDownloadProgress {file: string | null;/** Why the download failed, when `status` is `failed`. */error?: string;+ /** Each resource the download fetches, in the order it fetches them. */+ resources: ResourceDownloadState[];}export interface ResourcesStatusResponse {