Skip to content

Build your personal knowledge base with Trilium Notes

feat(resources): download manager for large files features need at run time

Elian Dorane4ba59e6ae42
diff --git a/apps/client/src/components/app_context.ts b/apps/client/src/components/app_context.ts
index 92560a549c..fb9197bda5 100644
--- a/apps/client/src/components/app_context.ts
+++ b/apps/client/src/components/app_context.ts
@@ -24,6 +24,7 @@ import { ContentEmbedOpts } from "../widgets/dialogs/content_embed.jsx";
import type { ResolveOptions } from "../widgets/dialogs/delete_notes.js";
import { IconPickerOpts } from "../widgets/dialogs/icon_picker.jsx";
import type { InfoProps } from "../widgets/dialogs/info.jsx";
+import type { ResourceDownloadDialogOptions } from "../widgets/dialogs/resource_download.jsx";
import type { LightboxOptions } from "../widgets/dialogs/lightbox.jsx";
import type { MarkdownImportOpts } from "../widgets/dialogs/markdown_import.jsx";
import { ChooseNoteTypeCallback } from "../widgets/dialogs/note_type_chooser.jsx";
@@ -169,6 +170,7 @@ export type CommandMappings = {
showLightbox: LightboxOptions;
showToolCallDetails: { toolCall: ToolCall };
showConfirmDialog: ConfirmWithMessageOptions;
+ showResourceDownloadDialog: ResourceDownloadDialogOptions;
showRecentChanges: CommandData & { ancestorNoteId: string };
showDeletedNotes: CommandData & { ancestorNoteId?: string };
showImportDialog: CommandData & { noteId: string };
diff --git a/apps/client/src/layouts/layout_commons.tsx b/apps/client/src/layouts/layout_commons.tsx
index d4dccf00bf..d6be21df4d 100644
--- a/apps/client/src/layouts/layout_commons.tsx
+++ b/apps/client/src/layouts/layout_commons.tsx
@@ -40,6 +40,7 @@ export function applyModals(rootContainer: RootContainer) {
.child(<LazyDialog triggerEvents={["showPromptDialog"]} loader={() => import("../widgets/dialogs/prompt.js")} />)
.child(<LazyDialog triggerEvents={["showItemPickerDialog"]} loader={() => import("../widgets/dialogs/item_picker.js")} />)
.child(<LazyDialog triggerEvents={["showCpuArchWarning"]} loader={() => import("../widgets/dialogs/incorrect_cpu_arch.js")} />)
+ .child(<LazyDialog triggerEvents={["showResourceDownloadDialog"]} loader={() => import("../widgets/dialogs/resource_download.js")} />)
.child(<LazyDialog triggerEvents={["showOptions"]} loader={() => import("../widgets/dialogs/OptionsDialog.jsx")} />)
.child(<LazyDialog triggerEvents={["showContentLanguagesDialog"]} loader={() => import("../widgets/dialogs/content_languages.js")} />)
.child(<LazyDialog triggerEvents={["showOcrTextDialog"]} loader={() => import("../widgets/dialogs/ocr_text.js")} />)
diff --git a/apps/client/src/services/dialog.ts b/apps/client/src/services/dialog.ts
index 725b39831d..186d3a5ece 100644
--- a/apps/client/src/services/dialog.ts
+++ b/apps/client/src/services/dialog.ts
@@ -1,6 +1,8 @@
-import type { NotePickerDialogOptions } from "../widgets/dialogs/note_picker.js";
+import type { ResourcesStatusResponse } from "@triliumnext/commons";
import { Modal } from "bootstrap";
+import type { NotePickerDialogOptions } from "../widgets/dialogs/note_picker.js";
+
import appContext from "../components/app_context.js";
import type { ConfirmDeleteNoteBoxOptions, ConfirmDialogOptions, ConfirmDialogResult, ConfirmWithMessageOptions, MessageType } from "../widgets/dialogs/confirm.js";
import { InfoExtraProps } from "../widgets/dialogs/info.jsx";
@@ -9,6 +11,7 @@ import type { PromptDialogOptions } from "../widgets/dialogs/prompt.js";
import { focusSavedElement, saveFocusedElement } from "./focus.js";
import keyboardActionsService from "./keyboard_actions.js";
import type { NoteDeletionTarget } from "./note_deletion.js";
+import server from "./server.js";
/**
* @param declaredZIndex the z-index the dialog defines for itself, if any (see {@link raiseAboveStackedPopup}).
@@ -291,6 +294,22 @@ export async function prompt(props: PromptDialogOptions) {
return new Promise<string | null>((res) => appContext.triggerCommand("showPromptDialog", { ...props, callback: res }));
}
+/**
+ * Makes sure the resources `ids` are installed, asking to download those that are not and showing
+ * the download.
+ *
+ * @returns `true` once every resource is installed, `false` when the download was declined,
+ * canceled or failed.
+ */
+export async function requireResources(ids: string[]) {
+ const status = await server.get<ResourcesStatusResponse>(`resources?ids=${ids.map(encodeURIComponent).join(",")}`);
+ if (status.resources.every(({ installed }) => installed)) {
+ return true;
+ }
+ return new Promise<boolean>((res) =>
+ appContext.triggerCommand("showResourceDownloadDialog", { ids, status, callback: res }));
+}
+
export default {
info,
chooseNote,
@@ -298,5 +317,6 @@ export default {
confirmWithNoteDeletion,
confirmDeleteNoteBoxWithNote,
pickSingleItem,
- prompt
+ prompt,
+ requireResources
};
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json
index 644753cd45..9659766b3d 100644
--- a/apps/client/src/translations/en/translation.json
+++ b/apps/client/src/translations/en/translation.json
@@ -4606,5 +4606,20 @@
"offset": "{{offset}} into the transcription",
"cleanup": "Hide filler words",
"cleanup_tooltip": "Hides “uh”, “um” and stuttered words. The transcript itself is not changed."
+ },
+ "resource_download": {
+ "title": "Download required",
+ "intro": "This needs the files below, which do not come with Trilium. They are downloaded once and kept on this device.",
+ "partially_downloaded": "{{size}}, of which {{downloaded}} is already downloaded",
+ "sources": "From {{sources}}",
+ "total": "Download size: {{size}}",
+ "free_space": "Free space: {{size}}",
+ "not_enough_space": "There is not enough free space for this download.",
+ "progress": "{{received}} of {{total}}",
+ "current_file": "Downloading {{file}}…",
+ "failed": "The download failed: {{error}}",
+ "download": "Download",
+ "retry": "Retry",
+ "cancel": "Cancel"
}
}
diff --git a/apps/client/src/widgets/dialogs/resource_download.css b/apps/client/src/widgets/dialogs/resource_download.css
new file mode 100644
index 0000000000..0de5657505
--- /dev/null
+++ b/apps/client/src/widgets/dialogs/resource_download.css
@@ -0,0 +1,41 @@
+.modal.resource-download-dialog {
+ .resource-download-list {
+ padding-inline-start: 1.25em;
+
+ li + li {
+ margin-top: 0.75em;
+ }
+
+ p {
+ margin-bottom: 0.25em;
+ }
+ }
+
+ .resource-download-meta {
+ color: var(--muted-text-color);
+ font-size: 0.9em;
+ }
+
+ .resource-download-totals {
+ display: flex;
+ flex-wrap: wrap;
+ justify-content: space-between;
+ gap: 0.5em;
+ font-weight: 600;
+ }
+
+ .resource-download-progress {
+ progress {
+ width: 100%;
+ }
+
+ .resource-download-progress-detail {
+ display: flex;
+ justify-content: space-between;
+ gap: 1em;
+ color: var(--muted-text-color);
+ font-size: 0.9em;
+ font-variant-numeric: tabular-nums;
+ }
+ }
+}
diff --git a/apps/client/src/widgets/dialogs/resource_download.spec.tsx b/apps/client/src/widgets/dialogs/resource_download.spec.tsx
new file mode 100644
index 0000000000..e72da98566
--- /dev/null
+++ b/apps/client/src/widgets/dialogs/resource_download.spec.tsx
@@ -0,0 +1,161 @@
+import type { ResourceDownloadProgress, ResourcesStatusResponse, WebSocketMessage } from "@triliumnext/commons";
+import { act } from "preact/test-utils";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import Component from "../../components/component";
+import { renderInto } from "../../test/render";
+import { ParentComponent } from "../react/react_utils";
+import ResourceDownloadDialog from "./resource_download";
+
+const mocks = vi.hoisted(() => ({
+ post: vi.fn(),
+ handlers: new Set<(message: WebSocketMessage) => void>()
+}));
+
+vi.mock("../../services/i18n", () => ({
+ t: (key: string, values?: object) => (values ? `${key}:${JSON.stringify(values)}` : key)
+}));
+vi.mock("../../services/server", () => ({
+ default: { get: vi.fn(async () => []), post: mocks.post }
+}));
+vi.mock("../../services/ws", () => {
+ const ws = {
+ subscribeToMessages: (handler: (message: WebSocketMessage) => void) => mocks.handlers.add(handler),
+ unsubscribeToMessage: (handler: (message: WebSocketMessage) => void) => mocks.handlers.delete(handler)
+ };
+ return { default: ws, ...ws };
+});
+
+const STATUS: ResourcesStatusResponse = {
+ resources: [ {
+ id: "transcription.model",
+ title: "Speech recognition model",
+ description: "Transcribes meetings",
+ size: 700,
+ downloadedSize: 200,
+ sources: [ "huggingface.co", "github.com" ],
+ installed: false
+ } ],
+ freeBytes: 100,
+ download: null
+};
+
+describe("ResourceDownloadDialog", () => {
+ let host: Component;
+ let container: HTMLElement;
+ let results: boolean[];
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.handlers.clear();
+ results = [];
+ host = new Component();
+ container = renderInto(
+ <ParentComponent.Provider value={host}>
+ <ResourceDownloadDialog />
+ </ParentComponent.Provider>
+ );
+ });
+
+ it("describes what it downloads, and resolves once the download succeeds", async () => {
+ mocks.post.mockResolvedValue(progress({ receivedBytes: 200 }));
+ await open();
+
+ expect(text(".resource-download-list")).toContain("Speech recognition model");
+ expect(text(".resource-download-meta")).toContain("huggingface.co, github.com");
+ expect(text(".resource-download-totals")).toContain(`"size":"500 B"`);
+ expect(container.querySelector(".alert-warning")).not.toBeNull();
+
+ await click("resource_download.download");
+ expect(mocks.post).toHaveBeenCalledExactlyOnceWith("resources/download",
+ { ids: [ "transcription.model" ] });
+ expect(container.querySelector("progress")?.value).toBe(200);
+
+ await send(progress({ receivedBytes: 600, file: "encoder.int8.onnx" }));
+ expect(container.querySelector("progress")?.value).toBe(600);
+ expect(text(".resource-download-progress")).toContain("encoder.int8.onnx");
+
+ await send(progress({ status: "succeeded", receivedBytes: 700 }));
+ await hidden();
+ expect(results).toEqual([ true ]);
+ });
+
+ it("keeps an end reported before the request that started the download returns", async () => {
+ let respond = (_value: ResourceDownloadProgress) => {};
+ mocks.post.mockReturnValue(new Promise((resolve) => respond = resolve));
+ await open();
+
+ await click("resource_download.download");
+ await send(progress({ status: "failed", error: "checksum" }));
+ await act(async () => respond(progress()));
+
+ expect(text(".alert-danger")).toContain("checksum");
+ expect(findButton("resource_download.retry")).toBeDefined();
+ });
+
+ it("cancels the running download when closed, and resolves false", async () => {
+ mocks.post.mockResolvedValue(progress());
+ await open();
+ await click("resource_download.download");
+ expect(findButton("resource_download.download")).toBeUndefined();
+
+ await hidden();
+
+ expect(mocks.post).toHaveBeenLastCalledWith("resources/download/d1/cancel");
+ expect(results).toEqual([ false ]);
+ });
+
+ async function open() {
+ await act(async () => {
+ await host.handleEvent("showResourceDownloadDialog", {
+ ids: [ "transcription.model" ],
+ status: STATUS,
+ callback: (installed: boolean) => results.push(installed)
+ });
+ });
+ }
+
+ async function send(download: ResourceDownloadProgress) {
+ await act(async () => {
+ for (const handler of mocks.handlers) {
+ handler({ type: "resource-download", download });
+ }
+ });
+ }
+
+ async function click(label: string) {
+ const button = findButton(label);
+ expect(button).toBeDefined();
+ await act(async () => button?.click());
+ }
+
+ /** Bootstrap fires this when the dialog closes; the test sends it instead. */
+ async function hidden() {
+ const dialog = container.querySelector(".resource-download-dialog");
+ expect(dialog).not.toBeNull();
+ await act(async () => {
+ dialog?.dispatchEvent(new Event("hidden.bs.modal", { bubbles: true }));
+ });
+ }
+
+ function text(selector: string) {
+ return container.querySelector(selector)?.textContent ?? "";
+ }
+
+ function findButton(label: string) {
+ return [ ...container.querySelectorAll("button") ]
+ .find((candidate) => candidate.textContent?.includes(label));
+ }
+});
+
+function progress(overrides: Partial<ResourceDownloadProgress> = {}): ResourceDownloadProgress {
+ return {
+ downloadId: "d1",
+ resourceIds: [ "transcription.model" ],
+ status: "downloading",
+ receivedBytes: 0,
+ totalBytes: 700,
+ file: null,
+ ...overrides
+ };
+}
diff --git a/apps/client/src/widgets/dialogs/resource_download.tsx b/apps/client/src/widgets/dialogs/resource_download.tsx
new file mode 100644
index 0000000000..2a041b9d94
--- /dev/null
+++ b/apps/client/src/widgets/dialogs/resource_download.tsx
@@ -0,0 +1,171 @@
+import "./resource_download.css";
+
+import type {
+ ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse, WebSocketMessage
+} from "@triliumnext/commons";
+import { useEffect, useRef, useState } from "preact/hooks";
+
+import { t } from "../../services/i18n";
+import server from "../../services/server";
+import { formatSize } from "../../services/utils";
+import { subscribeToMessages, unsubscribeToMessage } from "../../services/ws";
+import Alert from "../react/Alert";
+import Button from "../react/Button";
+import { useTriliumEvent } from "../react/hooks";
+import Modal from "../react/Modal";
+
+export interface ResourceDownloadDialogOptions {
+ ids: string[];
+ status: ResourcesStatusResponse;
+ /** Called once the dialog closes, with whether every resource is now installed. */
+ callback: (installed: boolean) => void;
+}
+
+/**
+ * Asks to download the resources a feature needs, then shows the download until it finishes.
+ * Closing the dialog cancels a running download.
+ */
+export default function ResourceDownloadDialog() {
+ const [ opts, setOpts ] = useState<ResourceDownloadDialogOptions>();
+ const [ shown, setShown ] = useState(false);
+ const [ download, setDownload ] = useState<ResourceDownloadProgress | null>(null);
+ const downloadRef = useRef(download);
+ downloadRef.current = download;
+
+ useTriliumEvent("showResourceDownloadDialog", (newOpts) => {
+ setOpts(newOpts);
+ setDownload(newOpts.status.download);
+ setShown(true);
+ });
+
+ useEffect(() => {
+ if (!shown || !opts) {
+ return;
+ }
+ // Matched by resource rather than by download ID, which a fast download can report
+ // before the request that started it returns.
+ const handler = (message: WebSocketMessage) => {
+ if (message.type === "resource-download"
+ && message.download.resourceIds.some((id) => opts.ids.includes(id))) {
+ setDownload(message.download);
+ }
+ };
+ subscribeToMessages(handler);
+ return () => unsubscribeToMessage(handler);
+ }, [ shown, opts ]);
+
+ useEffect(() => {
+ if (download?.status === "succeeded") {
+ setShown(false);
+ }
+ }, [ download ]);
+
+ async function startDownload() {
+ if (!opts) {
+ return;
+ }
+ setDownload(null);
+ const started = await server.post<ResourceDownloadProgress>("resources/download", { ids: opts.ids });
+ setDownload((current) => current?.downloadId === started.downloadId ? current : started);
+ }
+
+ function cancelDownload() {
+ const running = downloadRef.current;
+ if (running?.status === "downloading") {
+ void server.post(`resources/download/${running.downloadId}/cancel`);
+ }
+ }
+
+ const isDownloading = download?.status === "downloading";
+ const isFailed = download?.status === "failed";
+
+ return (
+ <Modal
+ className="resource-download-dialog"
+ size="md"
+ title={t("resource_download.title")}
+ show={shown}
+ onHidden={() => {
+ cancelDownload();
+ opts?.callback(downloadRef.current?.status === "succeeded");
+ setShown(false);
+ }}
+ footer={<>
+ <Button text={t("resource_download.cancel")} onClick={() => setShown(false)} />
+ {!isDownloading && (
+ <Button
+ text={isFailed ? t("resource_download.retry") : t("resource_download.download")}
+ icon="bx bx-download"
+ kind="primary"
+ onClick={startDownload}
+ />
+ )}
+ </>}
+ >
+ {opts && <ResourceList status={opts.status} />}
+ {isDownloading && <DownloadProgress download={download} />}
+ {isFailed && (
+ <Alert type="danger">{t("resource_download.failed", { error: download.error })}</Alert>
+ )}
+ </Modal>
+ );
+}
+
+function ResourceList({ status }: { status: ResourcesStatusResponse }) {
+ const missing = status.resources.filter(({ installed }) => !installed);
+ const remaining = missing.reduce((total, { size, downloadedSize }) => total + size - downloadedSize, 0);
+
+ return (
+ <>
+ <p>{t("resource_download.intro")}</p>
+ <ul className="resource-download-list">
+ {missing.map((resource) => <ResourceItem key={resource.id} resource={resource} />)}
+ </ul>
+ <p className="resource-download-totals">
+ <span>{t("resource_download.total", { size: formatSize(remaining) })}</span>
+ {status.freeBytes !== null && (
+ <span>{t("resource_download.free_space", { size: formatSize(status.freeBytes) })}</span>
+ )}
+ </p>
+ {status.freeBytes !== null && status.freeBytes < remaining && (
+ <Alert type="warning">{t("resource_download.not_enough_space")}</Alert>
+ )}
+ </>
+ );
+}
+
+function ResourceItem({ resource }: { resource: ResourceInfo }) {
+ const size = formatSize(resource.size);
+ return (
+ <li>
+ <strong>{resource.title}</strong>
+ <p>{resource.description}</p>
+ <p className="resource-download-meta">
+ {resource.downloadedSize
+ ? t("resource_download.partially_downloaded", {
+ size, downloaded: formatSize(resource.downloadedSize)
+ })
+ : size}
+ {" · "}
+ {t("resource_download.sources", { sources: resource.sources.join(", ") })}
+ </p>
+ </li>
+ );
+}
+
+function DownloadProgress({ download }: { download: ResourceDownloadProgress }) {
+ return (
+ <div className="resource-download-progress">
+ <progress value={download.receivedBytes} max={download.totalBytes} />
+ <div className="resource-download-progress-detail">
+ <span>{download.file && t("resource_download.current_file", { file: download.file })}</span>
+ <span>
+ {t("resource_download.progress", {
+ received: formatSize(download.receivedBytes, 1),
+ total: formatSize(download.totalBytes)
+ })}
+ </span>
+ </div>
+ </div>
+ );
+}
diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.json
index c880021410..69dab37b4a 100644
--- a/apps/server/src/assets/translations/en/server.json
+++ b/apps/server/src/assets/translations/en/server.json
@@ -572,5 +572,11 @@
"save-failed": "The transcript could not be saved: {{- message}} The transcription keeps it and tries again with the next sentence.",
"capture-lost": "The transcription stopped because the audio could no longer be captured. What was transcribed until then is kept.",
"attachment-title": "Transcript"
+ },
+ "resources": {
+ "unknown": "There is no resource named {{id}}.",
+ "http-error": "Downloading {{name}} failed with HTTP status {{status}}.",
+ "size-mismatch": "{{name}} does not have the expected size.",
+ "hash-mismatch": "{{name}} does not match its expected checksum, so it was discarded."
}
}
diff --git a/apps/server/src/routes/api/resources.ts b/apps/server/src/routes/api/resources.ts
new file mode 100644
index 0000000000..af6be6faaf
--- /dev/null
+++ b/apps/server/src/routes/api/resources.ts
@@ -0,0 +1,34 @@
+import type { ResourceDownloadProgress, ResourcesStatusResponse } from "@triliumnext/commons";
+import { ValidationError } from "@triliumnext/core";
+import type { Request } from "express";
+
+import resources from "../../services/resources.js";
+
+/** `GET /api/resources?ids=a,b`: whether each resource is installed, and the download that runs. */
+function getStatus(req: Request): Promise<ResourcesStatusResponse> {
+ return resources.getStatus(parseIds(req.query.ids));
+}
+
+/** `POST /api/resources/download` with `{ ids }`: starts downloading the resources not installed. */
+function startDownload(req: Request): ResourceDownloadProgress {
+ return resources.startDownload(parseIds(req.body?.ids));
+}
+
+/** `POST /api/resources/download/:downloadId/cancel`. */
+function cancelDownload(req: Request<{ downloadId: string }>) {
+ resources.cancelDownload(req.params.downloadId);
+}
+
+function parseIds(ids: unknown) {
+ const list = typeof ids === "string" ? ids.split(",") : ids;
+ if (!Array.isArray(list) || !list.length || !list.every((id) => typeof id === "string")) {
+ throw new ValidationError("Expected a list of resource IDs.");
+ }
+ return list as string[];
+}
+
+export default {
+ getStatus,
+ startDownload,
+ cancelDownload
+};
diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts
index 1e00258ead..8c33be5378 100644
--- a/apps/server/src/routes/routes.ts
+++ b/apps/server/src/routes/routes.ts
@@ -27,6 +27,7 @@ import metricsRoute from "./api/metrics.js";
import ocrRoute from "./api/ocr.js";
import onenoteImportRoute from "./api/onenote_import.js";
import recoveryCodes from './api/recovery_codes.js';
+import resourcesRoute from "./api/resources.js";
import senderRoute from "./api/sender.js";
import setupRestoreRoute from "./api/setup_restore.js";
import systemInfoRoute from "./api/system_info.js";
@@ -197,6 +198,10 @@ function register(app: express.Application) {
asyncApiRoute(PST, "/api/ocr/batch-process", ocrRoute.batchProcessOCR);
asyncApiRoute(GET, "/api/ocr/batch-progress", ocrRoute.getBatchProgress);
+ asyncApiRoute(GET, "/api/resources", resourcesRoute.getStatus);
+ apiRoute(PST, "/api/resources/download", resourcesRoute.startDownload);
+ apiRoute(PST, "/api/resources/download/:downloadId/cancel", resourcesRoute.cancelDownload);
+
app.use("", router);
}
diff --git a/apps/server/src/services/data_dir.spec.ts b/apps/server/src/services/data_dir.spec.ts
index 2f55e63c2e..e93b1f0c51 100644
--- a/apps/server/src/services/data_dir.spec.ts
+++ b/apps/server/src/services/data_dir.spec.ts
@@ -325,7 +325,7 @@ describe("data_dir.ts unit tests", () => {
});
describe("#getDataDirs()", () => {
- const envKeys: Omit<keyof ReturnType<typeof getDataDirs>, "TRILIUM_DATA_DIR">[] = [ "DOCUMENT_PATH", "BACKUP_DIR", "LOG_DIR", "ANONYMIZED_DB_DIR", "CONFIG_INI_PATH", "TMP_DIR", "OCR_CACHE_DIR" ];
+ const envKeys: Omit<keyof ReturnType<typeof getDataDirs>, "TRILIUM_DATA_DIR">[] = [ "DOCUMENT_PATH", "BACKUP_DIR", "LOG_DIR", "ANONYMIZED_DB_DIR", "CONFIG_INI_PATH", "TMP_DIR", "OCR_CACHE_DIR", "RESOURCES_DIR" ];
const setMockedEnv = (prefix: string | null) => {
envKeys.forEach((key) => {
diff --git a/apps/server/src/services/data_dir.ts b/apps/server/src/services/data_dir.ts
index d4c96811a7..7fbdffc39a 100644
--- a/apps/server/src/services/data_dir.ts
+++ b/apps/server/src/services/data_dir.ts
@@ -48,7 +48,8 @@ export function getDataDirs(TRILIUM_DATA_DIR: string) {
TMP_DIR: process.env.TRILIUM_TMP_DIR || pathJoin(TRILIUM_DATA_DIR, "tmp"),
ANONYMIZED_DB_DIR: process.env.TRILIUM_ANONYMIZED_DB_DIR || pathJoin(TRILIUM_DATA_DIR, "anonymized-db"),
CONFIG_INI_PATH: process.env.TRILIUM_CONFIG_INI_PATH || pathJoin(TRILIUM_DATA_DIR, "config.ini"),
- OCR_CACHE_DIR: process.env.TRILIUM_OCR_CACHE_DIR || pathJoin(TRILIUM_DATA_DIR, "ocr-cache")
+ OCR_CACHE_DIR: process.env.TRILIUM_OCR_CACHE_DIR || pathJoin(TRILIUM_DATA_DIR, "ocr-cache"),
+ RESOURCES_DIR: process.env.TRILIUM_RESOURCES_DIR || pathJoin(TRILIUM_DATA_DIR, "resources")
} as const;
createDirIfNotExisting(dataDirs.TMP_DIR, TRILIUM_DATA_DIR);
@@ -75,6 +76,7 @@ function checkExistingEntries(dataDirs: ReturnType<typeof getDataDirs>) {
[ dataDirs.BACKUP_DIR, R_OK | W_OK | X_OK ],
[ dataDirs.ANONYMIZED_DB_DIR, R_OK | W_OK | X_OK ],
[ dataDirs.OCR_CACHE_DIR, R_OK | W_OK | X_OK ],
+ [ dataDirs.RESOURCES_DIR, R_OK | W_OK | X_OK ],
[ dataDirs.CONFIG_INI_PATH, R_OK ],
[ inDataDir("session_secret.txt"), R_OK ]
];
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.ts
new file mode 100644
index 0000000000..0cd1573a1b
--- /dev/null
+++ b/apps/server/src/services/resources.spec.ts
@@ -0,0 +1,209 @@
+import type { ResourceDownloadProgress } from "@triliumnext/commons";
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
+
+import { type ResourceDefinition, type ResourceFile, ResourceManager } from "./resources.js";
+
+const MODEL = Buffer.alloc(300_000, "m");
+const TOKENS = Buffer.from("a\nb\nc\n");
+
+let server: http.Server;
+let baseUrl = "";
+const requested: string[] = [];
+/** Requests to `/stall`, answered with headers and no body until the test ends. */
+let stalled: http.ServerResponse[] = [];
+
+let baseDir = "";
+let broadcasts: ResourceDownloadProgress[] = [];
+/** The reports of the download {@link waitForEnd} last waited for. */
+let reports: ResourceDownloadProgress[] = [];
+let manager: ResourceManager;
+
+beforeAll(async () => {
+ server = http.createServer((req, res) => {
+ requested.push(req.url ?? "");
+ if (req.url === "/model.bin") {
+ res.end(MODEL);
+ } else if (req.url === "/tokens.txt") {
+ res.end(TOKENS);
+ } else if (req.url === "/stall") {
+ res.flushHeaders();
+ stalled.push(res);
+ } else {
+ res.statusCode = 404;
+ res.end();
+ }
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+});
+
+afterAll(() => new Promise<void>((resolve) => server.close(() => resolve())));
+
+beforeEach(() => {
+ baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-resources-"));
+ broadcasts = [];
+ requested.length = 0;
+ manager = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));
+});
+
+afterEach(() => {
+ for (const res of stalled) {
+ res.destroy();
+ }
+ stalled = [];
+ fs.rmSync(baseDir, { recursive: true, force: true });
+});
+
+describe("ResourceManager", () => {
+ it("downloads and verifies every file, reporting progress, then serves the directory", async () => {
+ manager.register(defineModel());
+ expect(manager.getPath("test.model")).toBeNull();
+
+ const started = manager.startDownload([ "test.model" ]);
+ expect(started).toMatchObject({
+ status: "downloading", receivedBytes: 0, totalBytes: MODEL.length + TOKENS.length
+ });
+ const finished = await waitForEnd();
+
+ expect(finished).toMatchObject({
+ downloadId: started.downloadId,
+ status: "succeeded",
+ receivedBytes: MODEL.length + TOKENS.length,
+ file: null
+ });
+ expect(reports.some(({ file }) => file === "model.bin")).toBe(true);
+ const directory = path.join(baseDir, "test.model", "1");
+ expect(manager.getPath("test.model")).toBe(directory);
+ expect(fs.readFileSync(path.join(directory, "nested", "model.bin")).equals(MODEL)).toBe(true);
+ expect(fs.readdirSync(path.join(directory, "nested"))).toEqual([ "model.bin" ]);
+
+ const { resources, download } = await manager.getStatus([ "test.model" ]);
+ expect(resources).toEqual([ {
+ id: "test.model",
+ title: "Test model",
+ description: "For tests",
+ size: MODEL.length + TOKENS.length,
+ downloadedSize: MODEL.length + TOKENS.length,
+ sources: [ "127.0.0.1" ],
+ installed: true
+ } ]);
+ expect(download).toBeNull();
+ expect(manager.startDownload([ "test.model" ]).status).toBe("succeeded");
+ });
+
+ it("discards a file that fails its checksum and keeps the verified ones for the next try", async () => {
+ manager.register(defineModel({ tokensHash: sha256(Buffer.from("tampered")) }));
+
+ manager.startDownload([ "test.model" ]);
+ const failed = await waitForEnd();
+
+ expect(failed.status).toBe("failed");
+ expect(failed.error).toBeTruthy();
+ expect(manager.getPath("test.model")).toBeNull();
+ const directory = path.join(baseDir, "test.model", "1");
+ expect(fs.readdirSync(directory)).toEqual([ "nested" ]);
+ const [ info ] = (await manager.getStatus([ "test.model" ])).resources;
+ expect(info).toMatchObject({ installed: false, downloadedSize: MODEL.length });
+
+ requested.length = 0;
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).receivedBytes).toBe(MODEL.length + TOKENS.length);
+ expect(requested).toEqual([ "/tokens.txt" ]);
+ });
+
+ it("fails on an HTTP error or a file of the wrong size", async () => {
+ manager.register(
+ defineModel({ id: "test.missing", tokensUrl: "/absent" }),
+ defineModel({ id: "test.short", tokensSize: TOKENS.length + 1 })
+ );
+
+ manager.startDownload([ "test.missing" ]);
+ expect((await waitForEnd()).status).toBe("failed");
+ manager.startDownload([ "test.short" ]);
+ expect((await waitForEnd()).status).toBe("failed");
+ expect(manager.getPath("test.missing")).toBeNull();
+ expect(manager.getPath("test.short")).toBeNull();
+ });
+
+ it("joins a running download, and cancels it without leaving a partial file", async () => {
+ manager.register(defineModel({ tokensUrl: "/stall" }));
+
+ const first = manager.startDownload([ "test.model" ]);
+ await waitFor(() => stalled.length > 0);
+ expect(manager.startDownload([ "test.model" ]).downloadId).toBe(first.downloadId);
+ expect((await manager.getStatus([ "test.model" ])).download?.downloadId).toBe(first.downloadId);
+
+ manager.cancelDownload(first.downloadId);
+ expect((await waitForEnd()).status).toBe("canceled");
+ const directory = path.join(baseDir, "test.model", "1");
+ expect(fs.readdirSync(directory)).toEqual([ "nested" ]);
+ expect((await manager.getStatus([ "test.model" ])).download).toBeNull();
+ });
+
+ it("prefers a local copy, and refuses invalid or unknown resources", () => {
+ manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });
+ expect(manager.getPath("test.model")).toBe("/hand/installed");
+
+ expect(() => manager.register(defineModel())).toThrow(/already registered/);
+ expect(() => manager.register(defineModel({ id: "Test Model" }))).toThrow(/invalid ID/);
+ expect(() => manager.register({ ...defineModel({ id: "test.escape" }), files: [ file("../outside", MODEL) ] }))
+ .toThrow(/outside its directory/);
+ expect(() => manager.getPath("test.unknown")).toThrow();
+ expect(() => manager.startDownload([ "test.unknown" ])).toThrow();
+ });
+});
+
+function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash = sha256(TOKENS),
+ tokensSize = TOKENS.length } = {}): ResourceDefinition {
+ return {
+ id,
+ version: "1",
+ title: () => "Test model",
+ description: () => "For tests",
+ files: [
+ file("nested/model.bin", MODEL),
+ { path: "tokens.txt", url: `${baseUrl}${tokensUrl}`, size: tokensSize, sha256: tokensHash }
+ ]
+ };
+}
+
+function file(filePath: string, content: Buffer): ResourceFile {
+ return {
+ path: filePath,
+ url: `${baseUrl}/${path.basename(filePath)}`,
+ size: content.length,
+ sha256: sha256(content)
+ };
+}
+
+function sha256(content: Buffer) {
+ return createHash("sha256").update(content).digest("hex");
+}
+
+/** Resolves with the last progress report once the download it belongs to has ended. */
+async function waitForEnd() {
+ await waitFor(() => broadcasts.at(-1)?.status !== "downloading" && broadcasts.length > 0);
+ const last = broadcasts.at(-1);
+ if (!last) {
+ throw new Error("No progress was reported.");
+ }
+ reports = broadcasts;
+ broadcasts = [];
+ return last;
+}
+
+async function waitFor(condition: () => boolean) {
+ const deadline = Date.now() + 5000;
+ while (!condition()) {
+ if (Date.now() > deadline) {
+ throw new Error("Timed out.");
+ }
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+}
diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.ts
new file mode 100644
index 0000000000..a51c4f1a62
--- /dev/null
+++ b/apps/server/src/services/resources.ts
@@ -0,0 +1,293 @@
+import type { ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse } from "@triliumnext/commons";
+import { getLog, utils, ValidationError, ws } from "@triliumnext/core";
+import { t } from "i18next";
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream as NodeReadableStream } from "node:stream/web";
+
+import dataDirs from "./data_dir.js";
+
+/** One file of a resource, downloaded from `url` and accepted only if it matches `size` and `sha256`. */
+export interface ResourceFile {
+ /** Where the file goes, relative to the resource's directory; `/` separates directories. */
+ path: string;
+ url: string;
+ size: number;
+ /** The SHA-256 of the file, in lowercase hexadecimal. */
+ sha256: string;
+}
+
+export interface ResourceDefinition {
+ /** Lowercase words separated by dots, the feature first: `transcription.model`. */
+ id: string;
+ /** Part of the resource's directory, so a new version downloads next to the old one. */
+ version: string;
+ title(): string;
+ description(): string;
+ files: ResourceFile[];
+ /**
+ * Returns a directory that holds the files outside the manager, such as a model installed by
+ * hand. While it returns one, the resource counts as installed there.
+ */
+ findLocalCopy?(): string | null;
+}
+
+interface Download {
+ progress: ResourceDownloadProgress;
+ controller: AbortController;
+}
+
+const ID_PATTERN = /^[a-z0-9-]+(\.[a-z0-9-]+)*$/;
+const VERSION_PATTERN = /^[\w.-]+$/;
+const PROGRESS_INTERVAL_MS = 250;
+/** Written into a resource's directory once every file in it is verified. */
+const COMPLETE_MARKER = ".complete";
+
+/**
+ * Downloads the resources features register into `baseDir`, one directory per resource and
+ * version. A file is written under a `.partial` name and renamed once its size and SHA-256 match,
+ * so a file under its own name is always verified, and an interrupted download keeps the files it
+ * finished.
+ */
+export class ResourceManager {
+ private readonly definitions = new Map<string, ResourceDefinition>();
+ private readonly downloads = new Map<string, Download>();
+
+ constructor(
+ private readonly baseDir: string,
+ private readonly broadcast: (progress: ResourceDownloadProgress) => void
+ ) {}
+
+ register(...definitions: ResourceDefinition[]) {
+ for (const definition of definitions) {
+ validateDefinition(definition);
+ if (this.definitions.has(definition.id)) {
+ throw new Error(`The resource ${definition.id} is already registered.`);
+ }
+ this.definitions.set(definition.id, definition);
+ }
+ }
+
+ /** Returns the directory holding the resource `id`, or `null` while it is not installed. */
+ getPath(id: string) {
+ const definition = this.getDefinition(id);
+ const localCopy = definition.findLocalCopy?.();
+ if (localCopy) {
+ return localCopy;
+ }
+ const directory = this.getDirectory(definition);
+ return fs.existsSync(path.join(directory, COMPLETE_MARKER)) ? directory : null;
+ }
+
+ async getStatus(ids: string[]): Promise<ResourcesStatusResponse> {
+ const definitions = ids.map((id) => this.getDefinition(id));
+ return {
+ resources: definitions.map((definition) => this.getInfo(definition)),
+ freeBytes: await this.getFreeBytes(),
+ download: this.findDownload(ids)?.progress ?? null
+ };
+ }
+
+ /**
+ * Starts downloading the resources `ids` that are not installed, reporting progress through
+ * `broadcast`. Joins a download that already fetches any of them.
+ *
+ * @returns The download as it starts; `succeeded` already when every resource is installed.
+ */
+ startDownload(ids: string[]): ResourceDownloadProgress {
+ const missing = ids.map((id) => this.getDefinition(id)).filter(({ id }) => !this.getPath(id));
+ const running = this.findDownload(ids);
+ if (running) {
+ return running.progress;
+ }
+
+ const download: Download = {
+ progress: {
+ downloadId: utils.randomString(12),
+ resourceIds: ids,
+ status: missing.length ? "downloading" : "succeeded",
+ receivedBytes: 0,
+ totalBytes: sumSizes(missing.flatMap(({ files }) => files)),
+ file: null
+ },
+ controller: new AbortController()
+ };
+ if (missing.length) {
+ this.downloads.set(download.progress.downloadId, download);
+ void this.runDownload(download, missing);
+ }
+ return { ...download.progress };
+ }
+
+ cancelDownload(downloadId: string) {
+ this.downloads.get(downloadId)?.controller.abort();
+ }
+
+ private async runDownload(download: Download, definitions: ResourceDefinition[]) {
+ const { progress, controller } = download;
+ let lastReported = 0;
+ const report = (force: boolean) => {
+ if (force || Date.now() - lastReported >= PROGRESS_INTERVAL_MS) {
+ lastReported = Date.now();
+ this.broadcast({ ...progress });
+ }
+ };
+
+ try {
+ for (const definition of definitions) {
+ const directory = this.getDirectory(definition);
+ for (const file of definition.files) {
+ const destination = path.join(directory, file.path);
+ if (isDownloaded(destination, file)) {
+ progress.receivedBytes += file.size;
+ continue;
+ }
+ progress.file = path.basename(file.path);
+ report(true);
+ await fs.promises.mkdir(path.dirname(destination), { recursive: true });
+ await downloadFile(file, destination, controller.signal, (bytes) => {
+ progress.receivedBytes += bytes;
+ report(false);
+ });
+ }
+ await fs.promises.writeFile(path.join(directory, COMPLETE_MARKER), "");
+ getLog().info(`Resources: installed ${definition.id} in ${directory}.`);
+ }
+ progress.status = "succeeded";
+ } catch (e) {
+ if (controller.signal.aborted) {
+ progress.status = "canceled";
+ } else {
+ progress.status = "failed";
+ progress.error = (e as Error)?.message ?? String(e);
+ getLog().error(`Resources: downloading ${progress.resourceIds.join(", ")} failed: ${progress.error}`);
+ }
+ } finally {
+ progress.file = null;
+ this.downloads.delete(progress.downloadId);
+ report(true);
+ }
+ }
+
+ private getDefinition(id: string) {
+ const definition = this.definitions.get(id);
+ if (!definition) {
+ throw new ValidationError(t("resources.unknown", { id }));
+ }
+ return definition;
+ }
+
+ private getDirectory(definition: ResourceDefinition) {
+ return path.join(this.baseDir, definition.id, definition.version);
+ }
+
+ private getInfo(definition: ResourceDefinition): ResourceInfo {
+ const directory = this.getDirectory(definition);
+ const downloaded = definition.files
+ .filter((file) => isDownloaded(path.join(directory, file.path), file));
+ return {
+ id: definition.id,
+ title: definition.title(),
+ description: definition.description(),
+ size: sumSizes(definition.files),
+ downloadedSize: sumSizes(downloaded),
+ sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],
+ installed: this.getPath(definition.id) !== null
+ };
+ }
+
+ private findDownload(ids: string[]) {
+ for (const download of this.downloads.values()) {
+ if (download.progress.resourceIds.some((id) => ids.includes(id))) {
+ return download;
+ }
+ }
+ return undefined;
+ }
+
+ private async getFreeBytes() {
+ const directory = fs.existsSync(this.baseDir) ? this.baseDir : path.dirname(this.baseDir);
+ try {
+ const { bavail, bsize } = await fs.promises.statfs(directory);
+ return bavail * bsize;
+ } catch {
+ return null;
+ }
+ }
+}
+
+/**
+ * Downloads `file` to `destination` through `<destination>.partial`, renaming it once its size and
+ * SHA-256 match. A failed or aborted download leaves no file behind.
+ */
+export async function downloadFile(file: ResourceFile, destination: string, signal: AbortSignal,
+ onProgress: (bytes: number) => void) {
+ const partial = `${destination}.partial`;
+ const name = path.basename(file.path);
+ try {
+ const response = await fetch(file.url, { signal });
+ if (!response.ok || !response.body) {
+ throw new Error(t("resources.http-error", { name, status: response.status }));
+ }
+
+ const hash = createHash("sha256");
+ let received = 0;
+ const meter = new Transform({
+ transform(chunk: Buffer, _encoding, callback) {
+ received += chunk.length;
+ if (received > file.size) {
+ callback(new Error(t("resources.size-mismatch", { name })));
+ return;
+ }
+ hash.update(chunk);
+ onProgress(chunk.length);
+ callback(null, chunk);
+ }
+ });
+ const body = Readable.fromWeb(response.body as NodeReadableStream<Uint8Array>);
+ await pipeline(body, meter, fs.createWriteStream(partial), { signal });
+
+ if (received !== file.size) {
+ throw new Error(t("resources.size-mismatch", { name }));
+ }
+ if (hash.digest("hex") !== file.sha256) {
+ throw new Error(t("resources.hash-mismatch", { name }));
+ }
+ await fs.promises.rename(partial, destination);
+ } catch (e) {
+ await fs.promises.rm(partial, { force: true });
+ throw e;
+ }
+}
+
+function validateDefinition({ id, version, files }: ResourceDefinition) {
+ if (!ID_PATTERN.test(id) || !VERSION_PATTERN.test(version)) {
+ throw new Error(`The resource ${id} has an invalid ID or version.`);
+ }
+ for (const file of files) {
+ const segments = file.path.split("/");
+ if (segments.some((segment) => !segment || segment === "." || segment === "..")) {
+ throw new Error(`The resource ${id} has a file outside its directory: ${file.path}.`);
+ }
+ }
+}
+
+/** Whether `destination` is a finished download of `file`: only a verified file gets its name. */
+function isDownloaded(destination: string, file: ResourceFile) {
+ try {
+ return fs.statSync(destination).size === file.size;
+ } catch {
+ return false;
+ }
+}
+
+function sumSizes(files: ResourceFile[]) {
+ return files.reduce((total, { size }) => total + size, 0);
+}
+
+export default new ResourceManager(dataDirs.RESOURCES_DIR, (download) => {
+ ws.sendMessageToAllClients({ type: "resource-download", download });
+});
diff --git a/packages/commons/src/index.ts b/packages/commons/src/index.ts
index 6a22860188..cae1313b4c 100644
--- a/packages/commons/src/index.ts
+++ b/packages/commons/src/index.ts
@@ -56,5 +56,6 @@ export * from "./lib/marked_extensions.js";
// is imported by virtually every client module. Import it via its subpath instead,
// e.g. "@triliumnext/commons/src/lib/markdown_renderer".
export * from "./lib/task_states.js";
+export * from "./lib/resources.js";
export * from "./lib/transcript.js";
export * from "./lib/keyboard_shortcut_display.js";
diff --git a/packages/commons/src/lib/resources.ts b/packages/commons/src/lib/resources.ts
new file mode 100644
index 0000000000..65876ce195
--- /dev/null
+++ b/packages/commons/src/lib/resources.ts
@@ -0,0 +1,40 @@
+/**
+ * Resources are large files a feature needs at run time but that do not ship with the app, such as
+ * the speech recognition model. The backend declares each one with its source and SHA-256, and
+ * downloads it into the data directory on request; the client asks for resources by ID only.
+ */
+
+/** A resource as `GET /api/resources` reports it. */
+export interface ResourceInfo {
+ id: string;
+ title: string;
+ description: string;
+ /** The size of all its files, in bytes. */
+ size: number;
+ /** The bytes of its files already downloaded and verified, kept from an interrupted download. */
+ downloadedSize: number;
+ /** The host names its files are downloaded from. */
+ sources: string[];
+ installed: boolean;
+}
+
+/** A download as it runs, and as the `resource-download` WebSocket message reports it. */
+export interface ResourceDownloadProgress {
+ downloadId: string;
+ resourceIds: string[];
+ status: "downloading" | "succeeded" | "failed" | "canceled";
+ receivedBytes: number;
+ totalBytes: number;
+ /** The name of the file being downloaded. */
+ file: string | null;
+ /** Why the download failed, when `status` is `failed`. */
+ error?: string;
+}
+
+export interface ResourcesStatusResponse {
+ resources: ResourceInfo[];
+ /** The free space where resources are stored, in bytes, or `null` when it cannot be read. */
+ freeBytes: number | null;
+ /** The download that already fetches some of the resources, if one runs. */
+ download: ResourceDownloadProgress | null;
+}
diff --git a/packages/commons/src/lib/ws_api.ts b/packages/commons/src/lib/ws_api.ts
index c93f1993ee..08cfe9aedf 100644
--- a/packages/commons/src/lib/ws_api.ts
+++ b/packages/commons/src/lib/ws_api.ts
@@ -1,4 +1,5 @@
import type { LlmStreamChunk } from "./llm_api.js";
+import type { ResourceDownloadProgress } from "./resources.js";
export interface EntityChange {
id?: number | null;
@@ -225,4 +226,7 @@ export type WebSocketMessage = AllTaskDefinitions | {
*/
type: "llm-stream-end";
streamId: string;
+} | {
+ type: "resource-download";
+ download: ResourceDownloadProgress;
}