Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(resources): download manager for large files features need at run time
Elian Dorane4ba59e6ae42
diff --git a/apps/client/src/components/app_context.ts b/apps/client/src/components/app_context.tsindex 92560a549c..fb9197bda5 100644--- a/apps/client/src/components/app_context.ts+++ b/apps/client/src/components/app_context.ts@@ -24,6 +24,7 @@ import { ContentEmbedOpts } from "../widgets/dialogs/content_embed.jsx";import type { ResolveOptions } from "../widgets/dialogs/delete_notes.js";import { IconPickerOpts } from "../widgets/dialogs/icon_picker.jsx";import type { InfoProps } from "../widgets/dialogs/info.jsx";+import type { ResourceDownloadDialogOptions } from "../widgets/dialogs/resource_download.jsx";import type { LightboxOptions } from "../widgets/dialogs/lightbox.jsx";import type { MarkdownImportOpts } from "../widgets/dialogs/markdown_import.jsx";import { ChooseNoteTypeCallback } from "../widgets/dialogs/note_type_chooser.jsx";@@ -169,6 +170,7 @@ export type CommandMappings = {showLightbox: LightboxOptions;showToolCallDetails: { toolCall: ToolCall };showConfirmDialog: ConfirmWithMessageOptions;+ showResourceDownloadDialog: ResourceDownloadDialogOptions;showRecentChanges: CommandData & { ancestorNoteId: string };showDeletedNotes: CommandData & { ancestorNoteId?: string };showImportDialog: CommandData & { noteId: string };diff --git a/apps/client/src/layouts/layout_commons.tsx b/apps/client/src/layouts/layout_commons.tsxindex d4dccf00bf..d6be21df4d 100644--- a/apps/client/src/layouts/layout_commons.tsx+++ b/apps/client/src/layouts/layout_commons.tsx@@ -40,6 +40,7 @@ export function applyModals(rootContainer: RootContainer) {.child(<LazyDialog triggerEvents={["showPromptDialog"]} loader={() => import("../widgets/dialogs/prompt.js")} />).child(<LazyDialog triggerEvents={["showItemPickerDialog"]} loader={() => import("../widgets/dialogs/item_picker.js")} />).child(<LazyDialog triggerEvents={["showCpuArchWarning"]} loader={() => import("../widgets/dialogs/incorrect_cpu_arch.js")} />)+ .child(<LazyDialog triggerEvents={["showResourceDownloadDialog"]} loader={() => import("../widgets/dialogs/resource_download.js")} />).child(<LazyDialog triggerEvents={["showOptions"]} loader={() => import("../widgets/dialogs/OptionsDialog.jsx")} />).child(<LazyDialog triggerEvents={["showContentLanguagesDialog"]} loader={() => import("../widgets/dialogs/content_languages.js")} />).child(<LazyDialog triggerEvents={["showOcrTextDialog"]} loader={() => import("../widgets/dialogs/ocr_text.js")} />)diff --git a/apps/client/src/services/dialog.ts b/apps/client/src/services/dialog.tsindex 725b39831d..186d3a5ece 100644--- a/apps/client/src/services/dialog.ts+++ b/apps/client/src/services/dialog.ts@@ -1,6 +1,8 @@-import type { NotePickerDialogOptions } from "../widgets/dialogs/note_picker.js";+import type { ResourcesStatusResponse } from "@triliumnext/commons";import { Modal } from "bootstrap";+import type { NotePickerDialogOptions } from "../widgets/dialogs/note_picker.js";+import appContext from "../components/app_context.js";import type { ConfirmDeleteNoteBoxOptions, ConfirmDialogOptions, ConfirmDialogResult, ConfirmWithMessageOptions, MessageType } from "../widgets/dialogs/confirm.js";import { InfoExtraProps } from "../widgets/dialogs/info.jsx";@@ -9,6 +11,7 @@ import type { PromptDialogOptions } from "../widgets/dialogs/prompt.js";import { focusSavedElement, saveFocusedElement } from "./focus.js";import keyboardActionsService from "./keyboard_actions.js";import type { NoteDeletionTarget } from "./note_deletion.js";+import server from "./server.js";/*** @param declaredZIndex the z-index the dialog defines for itself, if any (see {@link raiseAboveStackedPopup}).@@ -291,6 +294,22 @@ export async function prompt(props: PromptDialogOptions) {return new Promise<string | null>((res) => appContext.triggerCommand("showPromptDialog", { ...props, callback: res }));}+/**+ * Makes sure the resources `ids` are installed, asking to download those that are not and showing+ * the download.+ *+ * @returns `true` once every resource is installed, `false` when the download was declined,+ * canceled or failed.+ */+export async function requireResources(ids: string[]) {+ const status = await server.get<ResourcesStatusResponse>(`resources?ids=${ids.map(encodeURIComponent).join(",")}`);+ if (status.resources.every(({ installed }) => installed)) {+ return true;+ }+ return new Promise<boolean>((res) =>+ appContext.triggerCommand("showResourceDownloadDialog", { ids, status, callback: res }));+}+export default {info,chooseNote,@@ -298,5 +317,6 @@ export default {confirmWithNoteDeletion,confirmDeleteNoteBoxWithNote,pickSingleItem,- prompt+ prompt,+ requireResources};diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.jsonindex 644753cd45..9659766b3d 100644--- a/apps/client/src/translations/en/translation.json+++ b/apps/client/src/translations/en/translation.json@@ -4606,5 +4606,20 @@"offset": "{{offset}} into the transcription","cleanup": "Hide filler words","cleanup_tooltip": "Hides “uh”, “um” and stuttered words. The transcript itself is not changed."+ },+ "resource_download": {+ "title": "Download required",+ "intro": "This needs the files below, which do not come with Trilium. They are downloaded once and kept on this device.",+ "partially_downloaded": "{{size}}, of which {{downloaded}} is already downloaded",+ "sources": "From {{sources}}",+ "total": "Download size: {{size}}",+ "free_space": "Free space: {{size}}",+ "not_enough_space": "There is not enough free space for this download.",+ "progress": "{{received}} of {{total}}",+ "current_file": "Downloading {{file}}…",+ "failed": "The download failed: {{error}}",+ "download": "Download",+ "retry": "Retry",+ "cancel": "Cancel"}}diff --git a/apps/client/src/widgets/dialogs/resource_download.css b/apps/client/src/widgets/dialogs/resource_download.cssnew file mode 100644index 0000000000..0de5657505--- /dev/null+++ b/apps/client/src/widgets/dialogs/resource_download.css@@ -0,0 +1,41 @@+.modal.resource-download-dialog {+ .resource-download-list {+ padding-inline-start: 1.25em;++ li + li {+ margin-top: 0.75em;+ }++ p {+ margin-bottom: 0.25em;+ }+ }++ .resource-download-meta {+ color: var(--muted-text-color);+ font-size: 0.9em;+ }++ .resource-download-totals {+ display: flex;+ flex-wrap: wrap;+ justify-content: space-between;+ gap: 0.5em;+ font-weight: 600;+ }++ .resource-download-progress {+ progress {+ width: 100%;+ }++ .resource-download-progress-detail {+ display: flex;+ justify-content: space-between;+ gap: 1em;+ color: var(--muted-text-color);+ font-size: 0.9em;+ font-variant-numeric: tabular-nums;+ }+ }+}diff --git a/apps/client/src/widgets/dialogs/resource_download.spec.tsx b/apps/client/src/widgets/dialogs/resource_download.spec.tsxnew file mode 100644index 0000000000..e72da98566--- /dev/null+++ b/apps/client/src/widgets/dialogs/resource_download.spec.tsx@@ -0,0 +1,161 @@+import type { ResourceDownloadProgress, ResourcesStatusResponse, WebSocketMessage } from "@triliumnext/commons";+import { act } from "preact/test-utils";+import { beforeEach, describe, expect, it, vi } from "vitest";++import Component from "../../components/component";+import { renderInto } from "../../test/render";+import { ParentComponent } from "../react/react_utils";+import ResourceDownloadDialog from "./resource_download";++const mocks = vi.hoisted(() => ({+ post: vi.fn(),+ handlers: new Set<(message: WebSocketMessage) => void>()+}));++vi.mock("../../services/i18n", () => ({+ t: (key: string, values?: object) => (values ? `${key}:${JSON.stringify(values)}` : key)+}));+vi.mock("../../services/server", () => ({+ default: { get: vi.fn(async () => []), post: mocks.post }+}));+vi.mock("../../services/ws", () => {+ const ws = {+ subscribeToMessages: (handler: (message: WebSocketMessage) => void) => mocks.handlers.add(handler),+ unsubscribeToMessage: (handler: (message: WebSocketMessage) => void) => mocks.handlers.delete(handler)+ };+ return { default: ws, ...ws };+});++const STATUS: ResourcesStatusResponse = {+ resources: [ {+ id: "transcription.model",+ title: "Speech recognition model",+ description: "Transcribes meetings",+ size: 700,+ downloadedSize: 200,+ sources: [ "huggingface.co", "github.com" ],+ installed: false+ } ],+ freeBytes: 100,+ download: null+};++describe("ResourceDownloadDialog", () => {+ let host: Component;+ let container: HTMLElement;+ let results: boolean[];++ beforeEach(() => {+ vi.clearAllMocks();+ mocks.handlers.clear();+ results = [];+ host = new Component();+ container = renderInto(+ <ParentComponent.Provider value={host}>+ <ResourceDownloadDialog />+ </ParentComponent.Provider>+ );+ });++ it("describes what it downloads, and resolves once the download succeeds", async () => {+ mocks.post.mockResolvedValue(progress({ receivedBytes: 200 }));+ await open();++ expect(text(".resource-download-list")).toContain("Speech recognition model");+ expect(text(".resource-download-meta")).toContain("huggingface.co, github.com");+ expect(text(".resource-download-totals")).toContain(`"size":"500 B"`);+ expect(container.querySelector(".alert-warning")).not.toBeNull();++ await click("resource_download.download");+ expect(mocks.post).toHaveBeenCalledExactlyOnceWith("resources/download",+ { ids: [ "transcription.model" ] });+ expect(container.querySelector("progress")?.value).toBe(200);++ await send(progress({ receivedBytes: 600, file: "encoder.int8.onnx" }));+ expect(container.querySelector("progress")?.value).toBe(600);+ expect(text(".resource-download-progress")).toContain("encoder.int8.onnx");++ await send(progress({ status: "succeeded", receivedBytes: 700 }));+ await hidden();+ expect(results).toEqual([ true ]);+ });++ it("keeps an end reported before the request that started the download returns", async () => {+ let respond = (_value: ResourceDownloadProgress) => {};+ mocks.post.mockReturnValue(new Promise((resolve) => respond = resolve));+ await open();++ await click("resource_download.download");+ await send(progress({ status: "failed", error: "checksum" }));+ await act(async () => respond(progress()));++ expect(text(".alert-danger")).toContain("checksum");+ expect(findButton("resource_download.retry")).toBeDefined();+ });++ it("cancels the running download when closed, and resolves false", async () => {+ mocks.post.mockResolvedValue(progress());+ await open();+ await click("resource_download.download");+ expect(findButton("resource_download.download")).toBeUndefined();++ await hidden();++ expect(mocks.post).toHaveBeenLastCalledWith("resources/download/d1/cancel");+ expect(results).toEqual([ false ]);+ });++ async function open() {+ await act(async () => {+ await host.handleEvent("showResourceDownloadDialog", {+ ids: [ "transcription.model" ],+ status: STATUS,+ callback: (installed: boolean) => results.push(installed)+ });+ });+ }++ async function send(download: ResourceDownloadProgress) {+ await act(async () => {+ for (const handler of mocks.handlers) {+ handler({ type: "resource-download", download });+ }+ });+ }++ async function click(label: string) {+ const button = findButton(label);+ expect(button).toBeDefined();+ await act(async () => button?.click());+ }++ /** Bootstrap fires this when the dialog closes; the test sends it instead. */+ async function hidden() {+ const dialog = container.querySelector(".resource-download-dialog");+ expect(dialog).not.toBeNull();+ await act(async () => {+ dialog?.dispatchEvent(new Event("hidden.bs.modal", { bubbles: true }));+ });+ }++ function text(selector: string) {+ return container.querySelector(selector)?.textContent ?? "";+ }++ function findButton(label: string) {+ return [ ...container.querySelectorAll("button") ]+ .find((candidate) => candidate.textContent?.includes(label));+ }+});++function progress(overrides: Partial<ResourceDownloadProgress> = {}): ResourceDownloadProgress {+ return {+ downloadId: "d1",+ resourceIds: [ "transcription.model" ],+ status: "downloading",+ receivedBytes: 0,+ totalBytes: 700,+ file: null,+ ...overrides+ };+}diff --git a/apps/client/src/widgets/dialogs/resource_download.tsx b/apps/client/src/widgets/dialogs/resource_download.tsxnew file mode 100644index 0000000000..2a041b9d94--- /dev/null+++ b/apps/client/src/widgets/dialogs/resource_download.tsx@@ -0,0 +1,171 @@+import "./resource_download.css";++import type {+ ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse, WebSocketMessage+} from "@triliumnext/commons";+import { useEffect, useRef, useState } from "preact/hooks";++import { t } from "../../services/i18n";+import server from "../../services/server";+import { formatSize } from "../../services/utils";+import { subscribeToMessages, unsubscribeToMessage } from "../../services/ws";+import Alert from "../react/Alert";+import Button from "../react/Button";+import { useTriliumEvent } from "../react/hooks";+import Modal from "../react/Modal";++export interface ResourceDownloadDialogOptions {+ ids: string[];+ status: ResourcesStatusResponse;+ /** Called once the dialog closes, with whether every resource is now installed. */+ callback: (installed: boolean) => void;+}++/**+ * Asks to download the resources a feature needs, then shows the download until it finishes.+ * Closing the dialog cancels a running download.+ */+export default function ResourceDownloadDialog() {+ const [ opts, setOpts ] = useState<ResourceDownloadDialogOptions>();+ const [ shown, setShown ] = useState(false);+ const [ download, setDownload ] = useState<ResourceDownloadProgress | null>(null);+ const downloadRef = useRef(download);+ downloadRef.current = download;++ useTriliumEvent("showResourceDownloadDialog", (newOpts) => {+ setOpts(newOpts);+ setDownload(newOpts.status.download);+ setShown(true);+ });++ useEffect(() => {+ if (!shown || !opts) {+ return;+ }+ // Matched by resource rather than by download ID, which a fast download can report+ // before the request that started it returns.+ const handler = (message: WebSocketMessage) => {+ if (message.type === "resource-download"+ && message.download.resourceIds.some((id) => opts.ids.includes(id))) {+ setDownload(message.download);+ }+ };+ subscribeToMessages(handler);+ return () => unsubscribeToMessage(handler);+ }, [ shown, opts ]);++ useEffect(() => {+ if (download?.status === "succeeded") {+ setShown(false);+ }+ }, [ download ]);++ async function startDownload() {+ if (!opts) {+ return;+ }+ setDownload(null);+ const started = await server.post<ResourceDownloadProgress>("resources/download", { ids: opts.ids });+ setDownload((current) => current?.downloadId === started.downloadId ? current : started);+ }++ function cancelDownload() {+ const running = downloadRef.current;+ if (running?.status === "downloading") {+ void server.post(`resources/download/${running.downloadId}/cancel`);+ }+ }++ const isDownloading = download?.status === "downloading";+ const isFailed = download?.status === "failed";++ return (+ <Modal+ className="resource-download-dialog"+ size="md"+ title={t("resource_download.title")}+ show={shown}+ onHidden={() => {+ cancelDownload();+ opts?.callback(downloadRef.current?.status === "succeeded");+ setShown(false);+ }}+ footer={<>+ <Button text={t("resource_download.cancel")} onClick={() => setShown(false)} />+ {!isDownloading && (+ <Button+ text={isFailed ? t("resource_download.retry") : t("resource_download.download")}+ icon="bx bx-download"+ kind="primary"+ onClick={startDownload}+ />+ )}+ </>}+ >+ {opts && <ResourceList status={opts.status} />}+ {isDownloading && <DownloadProgress download={download} />}+ {isFailed && (+ <Alert type="danger">{t("resource_download.failed", { error: download.error })}</Alert>+ )}+ </Modal>+ );+}++function ResourceList({ status }: { status: ResourcesStatusResponse }) {+ const missing = status.resources.filter(({ installed }) => !installed);+ const remaining = missing.reduce((total, { size, downloadedSize }) => total + size - downloadedSize, 0);++ return (+ <>+ <p>{t("resource_download.intro")}</p>+ <ul className="resource-download-list">+ {missing.map((resource) => <ResourceItem key={resource.id} resource={resource} />)}+ </ul>+ <p className="resource-download-totals">+ <span>{t("resource_download.total", { size: formatSize(remaining) })}</span>+ {status.freeBytes !== null && (+ <span>{t("resource_download.free_space", { size: formatSize(status.freeBytes) })}</span>+ )}+ </p>+ {status.freeBytes !== null && status.freeBytes < remaining && (+ <Alert type="warning">{t("resource_download.not_enough_space")}</Alert>+ )}+ </>+ );+}++function ResourceItem({ resource }: { resource: ResourceInfo }) {+ const size = formatSize(resource.size);+ return (+ <li>+ <strong>{resource.title}</strong>+ <p>{resource.description}</p>+ <p className="resource-download-meta">+ {resource.downloadedSize+ ? t("resource_download.partially_downloaded", {+ size, downloaded: formatSize(resource.downloadedSize)+ })+ : size}+ {" · "}+ {t("resource_download.sources", { sources: resource.sources.join(", ") })}+ </p>+ </li>+ );+}++function DownloadProgress({ download }: { download: ResourceDownloadProgress }) {+ return (+ <div className="resource-download-progress">+ <progress value={download.receivedBytes} max={download.totalBytes} />+ <div className="resource-download-progress-detail">+ <span>{download.file && t("resource_download.current_file", { file: download.file })}</span>+ <span>+ {t("resource_download.progress", {+ received: formatSize(download.receivedBytes, 1),+ total: formatSize(download.totalBytes)+ })}+ </span>+ </div>+ </div>+ );+}diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.jsonindex c880021410..69dab37b4a 100644--- a/apps/server/src/assets/translations/en/server.json+++ b/apps/server/src/assets/translations/en/server.json@@ -572,5 +572,11 @@"save-failed": "The transcript could not be saved: {{- message}} The transcription keeps it and tries again with the next sentence.","capture-lost": "The transcription stopped because the audio could no longer be captured. What was transcribed until then is kept.","attachment-title": "Transcript"+ },+ "resources": {+ "unknown": "There is no resource named {{id}}.",+ "http-error": "Downloading {{name}} failed with HTTP status {{status}}.",+ "size-mismatch": "{{name}} does not have the expected size.",+ "hash-mismatch": "{{name}} does not match its expected checksum, so it was discarded."}}diff --git a/apps/server/src/routes/api/resources.ts b/apps/server/src/routes/api/resources.tsnew file mode 100644index 0000000000..af6be6faaf--- /dev/null+++ b/apps/server/src/routes/api/resources.ts@@ -0,0 +1,34 @@+import type { ResourceDownloadProgress, ResourcesStatusResponse } from "@triliumnext/commons";+import { ValidationError } from "@triliumnext/core";+import type { Request } from "express";++import resources from "../../services/resources.js";++/** `GET /api/resources?ids=a,b`: whether each resource is installed, and the download that runs. */+function getStatus(req: Request): Promise<ResourcesStatusResponse> {+ return resources.getStatus(parseIds(req.query.ids));+}++/** `POST /api/resources/download` with `{ ids }`: starts downloading the resources not installed. */+function startDownload(req: Request): ResourceDownloadProgress {+ return resources.startDownload(parseIds(req.body?.ids));+}++/** `POST /api/resources/download/:downloadId/cancel`. */+function cancelDownload(req: Request<{ downloadId: string }>) {+ resources.cancelDownload(req.params.downloadId);+}++function parseIds(ids: unknown) {+ const list = typeof ids === "string" ? ids.split(",") : ids;+ if (!Array.isArray(list) || !list.length || !list.every((id) => typeof id === "string")) {+ throw new ValidationError("Expected a list of resource IDs.");+ }+ return list as string[];+}++export default {+ getStatus,+ startDownload,+ cancelDownload+};diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.tsindex 1e00258ead..8c33be5378 100644--- a/apps/server/src/routes/routes.ts+++ b/apps/server/src/routes/routes.ts@@ -27,6 +27,7 @@ import metricsRoute from "./api/metrics.js";import ocrRoute from "./api/ocr.js";import onenoteImportRoute from "./api/onenote_import.js";import recoveryCodes from './api/recovery_codes.js';+import resourcesRoute from "./api/resources.js";import senderRoute from "./api/sender.js";import setupRestoreRoute from "./api/setup_restore.js";import systemInfoRoute from "./api/system_info.js";@@ -197,6 +198,10 @@ function register(app: express.Application) {asyncApiRoute(PST, "/api/ocr/batch-process", ocrRoute.batchProcessOCR);asyncApiRoute(GET, "/api/ocr/batch-progress", ocrRoute.getBatchProgress);+ asyncApiRoute(GET, "/api/resources", resourcesRoute.getStatus);+ apiRoute(PST, "/api/resources/download", resourcesRoute.startDownload);+ apiRoute(PST, "/api/resources/download/:downloadId/cancel", resourcesRoute.cancelDownload);+app.use("", router);}diff --git a/apps/server/src/services/data_dir.spec.ts b/apps/server/src/services/data_dir.spec.tsindex 2f55e63c2e..e93b1f0c51 100644--- a/apps/server/src/services/data_dir.spec.ts+++ b/apps/server/src/services/data_dir.spec.ts@@ -325,7 +325,7 @@ describe("data_dir.ts unit tests", () => {});describe("#getDataDirs()", () => {- const envKeys: Omit<keyof ReturnType<typeof getDataDirs>, "TRILIUM_DATA_DIR">[] = [ "DOCUMENT_PATH", "BACKUP_DIR", "LOG_DIR", "ANONYMIZED_DB_DIR", "CONFIG_INI_PATH", "TMP_DIR", "OCR_CACHE_DIR" ];+ const envKeys: Omit<keyof ReturnType<typeof getDataDirs>, "TRILIUM_DATA_DIR">[] = [ "DOCUMENT_PATH", "BACKUP_DIR", "LOG_DIR", "ANONYMIZED_DB_DIR", "CONFIG_INI_PATH", "TMP_DIR", "OCR_CACHE_DIR", "RESOURCES_DIR" ];const setMockedEnv = (prefix: string | null) => {envKeys.forEach((key) => {diff --git a/apps/server/src/services/data_dir.ts b/apps/server/src/services/data_dir.tsindex d4c96811a7..7fbdffc39a 100644--- a/apps/server/src/services/data_dir.ts+++ b/apps/server/src/services/data_dir.ts@@ -48,7 +48,8 @@ export function getDataDirs(TRILIUM_DATA_DIR: string) {TMP_DIR: process.env.TRILIUM_TMP_DIR || pathJoin(TRILIUM_DATA_DIR, "tmp"),ANONYMIZED_DB_DIR: process.env.TRILIUM_ANONYMIZED_DB_DIR || pathJoin(TRILIUM_DATA_DIR, "anonymized-db"),CONFIG_INI_PATH: process.env.TRILIUM_CONFIG_INI_PATH || pathJoin(TRILIUM_DATA_DIR, "config.ini"),- OCR_CACHE_DIR: process.env.TRILIUM_OCR_CACHE_DIR || pathJoin(TRILIUM_DATA_DIR, "ocr-cache")+ OCR_CACHE_DIR: process.env.TRILIUM_OCR_CACHE_DIR || pathJoin(TRILIUM_DATA_DIR, "ocr-cache"),+ RESOURCES_DIR: process.env.TRILIUM_RESOURCES_DIR || pathJoin(TRILIUM_DATA_DIR, "resources")} as const;createDirIfNotExisting(dataDirs.TMP_DIR, TRILIUM_DATA_DIR);@@ -75,6 +76,7 @@ function checkExistingEntries(dataDirs: ReturnType<typeof getDataDirs>) {[ dataDirs.BACKUP_DIR, R_OK | W_OK | X_OK ],[ dataDirs.ANONYMIZED_DB_DIR, R_OK | W_OK | X_OK ],[ dataDirs.OCR_CACHE_DIR, R_OK | W_OK | X_OK ],+ [ dataDirs.RESOURCES_DIR, R_OK | W_OK | X_OK ],[ dataDirs.CONFIG_INI_PATH, R_OK ],[ inDataDir("session_secret.txt"), R_OK ]];diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsnew file mode 100644index 0000000000..0cd1573a1b--- /dev/null+++ b/apps/server/src/services/resources.spec.ts@@ -0,0 +1,209 @@+import type { ResourceDownloadProgress } from "@triliumnext/commons";+import { createHash } from "node:crypto";+import fs from "node:fs";+import http from "node:http";+import type { AddressInfo } from "node:net";+import os from "node:os";+import path from "node:path";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";++import { type ResourceDefinition, type ResourceFile, ResourceManager } from "./resources.js";++const MODEL = Buffer.alloc(300_000, "m");+const TOKENS = Buffer.from("a\nb\nc\n");++let server: http.Server;+let baseUrl = "";+const requested: string[] = [];+/** Requests to `/stall`, answered with headers and no body until the test ends. */+let stalled: http.ServerResponse[] = [];++let baseDir = "";+let broadcasts: ResourceDownloadProgress[] = [];+/** The reports of the download {@link waitForEnd} last waited for. */+let reports: ResourceDownloadProgress[] = [];+let manager: ResourceManager;++beforeAll(async () => {+ server = http.createServer((req, res) => {+ requested.push(req.url ?? "");+ if (req.url === "/model.bin") {+ res.end(MODEL);+ } else if (req.url === "/tokens.txt") {+ res.end(TOKENS);+ } else if (req.url === "/stall") {+ res.flushHeaders();+ stalled.push(res);+ } else {+ res.statusCode = 404;+ res.end();+ }+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+});++afterAll(() => new Promise<void>((resolve) => server.close(() => resolve())));++beforeEach(() => {+ baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-resources-"));+ broadcasts = [];+ requested.length = 0;+ manager = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));+});++afterEach(() => {+ for (const res of stalled) {+ res.destroy();+ }+ stalled = [];+ fs.rmSync(baseDir, { recursive: true, force: true });+});++describe("ResourceManager", () => {+ it("downloads and verifies every file, reporting progress, then serves the directory", async () => {+ manager.register(defineModel());+ expect(manager.getPath("test.model")).toBeNull();++ const started = manager.startDownload([ "test.model" ]);+ expect(started).toMatchObject({+ status: "downloading", receivedBytes: 0, totalBytes: MODEL.length + TOKENS.length+ });+ const finished = await waitForEnd();++ expect(finished).toMatchObject({+ downloadId: started.downloadId,+ status: "succeeded",+ receivedBytes: MODEL.length + TOKENS.length,+ file: null+ });+ expect(reports.some(({ file }) => file === "model.bin")).toBe(true);+ const directory = path.join(baseDir, "test.model", "1");+ expect(manager.getPath("test.model")).toBe(directory);+ expect(fs.readFileSync(path.join(directory, "nested", "model.bin")).equals(MODEL)).toBe(true);+ expect(fs.readdirSync(path.join(directory, "nested"))).toEqual([ "model.bin" ]);++ const { resources, download } = await manager.getStatus([ "test.model" ]);+ expect(resources).toEqual([ {+ id: "test.model",+ title: "Test model",+ description: "For tests",+ size: MODEL.length + TOKENS.length,+ downloadedSize: MODEL.length + TOKENS.length,+ sources: [ "127.0.0.1" ],+ installed: true+ } ]);+ expect(download).toBeNull();+ expect(manager.startDownload([ "test.model" ]).status).toBe("succeeded");+ });++ it("discards a file that fails its checksum and keeps the verified ones for the next try", async () => {+ manager.register(defineModel({ tokensHash: sha256(Buffer.from("tampered")) }));++ manager.startDownload([ "test.model" ]);+ const failed = await waitForEnd();++ expect(failed.status).toBe("failed");+ expect(failed.error).toBeTruthy();+ expect(manager.getPath("test.model")).toBeNull();+ const directory = path.join(baseDir, "test.model", "1");+ expect(fs.readdirSync(directory)).toEqual([ "nested" ]);+ const [ info ] = (await manager.getStatus([ "test.model" ])).resources;+ expect(info).toMatchObject({ installed: false, downloadedSize: MODEL.length });++ requested.length = 0;+ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).receivedBytes).toBe(MODEL.length + TOKENS.length);+ expect(requested).toEqual([ "/tokens.txt" ]);+ });++ it("fails on an HTTP error or a file of the wrong size", async () => {+ manager.register(+ defineModel({ id: "test.missing", tokensUrl: "/absent" }),+ defineModel({ id: "test.short", tokensSize: TOKENS.length + 1 })+ );++ manager.startDownload([ "test.missing" ]);+ expect((await waitForEnd()).status).toBe("failed");+ manager.startDownload([ "test.short" ]);+ expect((await waitForEnd()).status).toBe("failed");+ expect(manager.getPath("test.missing")).toBeNull();+ expect(manager.getPath("test.short")).toBeNull();+ });++ it("joins a running download, and cancels it without leaving a partial file", async () => {+ manager.register(defineModel({ tokensUrl: "/stall" }));++ const first = manager.startDownload([ "test.model" ]);+ await waitFor(() => stalled.length > 0);+ expect(manager.startDownload([ "test.model" ]).downloadId).toBe(first.downloadId);+ expect((await manager.getStatus([ "test.model" ])).download?.downloadId).toBe(first.downloadId);++ manager.cancelDownload(first.downloadId);+ expect((await waitForEnd()).status).toBe("canceled");+ const directory = path.join(baseDir, "test.model", "1");+ expect(fs.readdirSync(directory)).toEqual([ "nested" ]);+ expect((await manager.getStatus([ "test.model" ])).download).toBeNull();+ });++ it("prefers a local copy, and refuses invalid or unknown resources", () => {+ manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });+ expect(manager.getPath("test.model")).toBe("/hand/installed");++ expect(() => manager.register(defineModel())).toThrow(/already registered/);+ expect(() => manager.register(defineModel({ id: "Test Model" }))).toThrow(/invalid ID/);+ expect(() => manager.register({ ...defineModel({ id: "test.escape" }), files: [ file("../outside", MODEL) ] }))+ .toThrow(/outside its directory/);+ expect(() => manager.getPath("test.unknown")).toThrow();+ expect(() => manager.startDownload([ "test.unknown" ])).toThrow();+ });+});++function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash = sha256(TOKENS),+ tokensSize = TOKENS.length } = {}): ResourceDefinition {+ return {+ id,+ version: "1",+ title: () => "Test model",+ description: () => "For tests",+ files: [+ file("nested/model.bin", MODEL),+ { path: "tokens.txt", url: `${baseUrl}${tokensUrl}`, size: tokensSize, sha256: tokensHash }+ ]+ };+}++function file(filePath: string, content: Buffer): ResourceFile {+ return {+ path: filePath,+ url: `${baseUrl}/${path.basename(filePath)}`,+ size: content.length,+ sha256: sha256(content)+ };+}++function sha256(content: Buffer) {+ return createHash("sha256").update(content).digest("hex");+}++/** Resolves with the last progress report once the download it belongs to has ended. */+async function waitForEnd() {+ await waitFor(() => broadcasts.at(-1)?.status !== "downloading" && broadcasts.length > 0);+ const last = broadcasts.at(-1);+ if (!last) {+ throw new Error("No progress was reported.");+ }+ reports = broadcasts;+ broadcasts = [];+ return last;+}++async function waitFor(condition: () => boolean) {+ const deadline = Date.now() + 5000;+ while (!condition()) {+ if (Date.now() > deadline) {+ throw new Error("Timed out.");+ }+ await new Promise((resolve) => setTimeout(resolve, 5));+ }+}diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsnew file mode 100644index 0000000000..a51c4f1a62--- /dev/null+++ b/apps/server/src/services/resources.ts@@ -0,0 +1,293 @@+import type { ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse } from "@triliumnext/commons";+import { getLog, utils, ValidationError, ws } from "@triliumnext/core";+import { t } from "i18next";+import { createHash } from "node:crypto";+import fs from "node:fs";+import path from "node:path";+import { Readable, Transform } from "node:stream";+import { pipeline } from "node:stream/promises";+import type { ReadableStream as NodeReadableStream } from "node:stream/web";++import dataDirs from "./data_dir.js";++/** One file of a resource, downloaded from `url` and accepted only if it matches `size` and `sha256`. */+export interface ResourceFile {+ /** Where the file goes, relative to the resource's directory; `/` separates directories. */+ path: string;+ url: string;+ size: number;+ /** The SHA-256 of the file, in lowercase hexadecimal. */+ sha256: string;+}++export interface ResourceDefinition {+ /** Lowercase words separated by dots, the feature first: `transcription.model`. */+ id: string;+ /** Part of the resource's directory, so a new version downloads next to the old one. */+ version: string;+ title(): string;+ description(): string;+ files: ResourceFile[];+ /**+ * Returns a directory that holds the files outside the manager, such as a model installed by+ * hand. While it returns one, the resource counts as installed there.+ */+ findLocalCopy?(): string | null;+}++interface Download {+ progress: ResourceDownloadProgress;+ controller: AbortController;+}++const ID_PATTERN = /^[a-z0-9-]+(\.[a-z0-9-]+)*$/;+const VERSION_PATTERN = /^[\w.-]+$/;+const PROGRESS_INTERVAL_MS = 250;+/** Written into a resource's directory once every file in it is verified. */+const COMPLETE_MARKER = ".complete";++/**+ * Downloads the resources features register into `baseDir`, one directory per resource and+ * version. A file is written under a `.partial` name and renamed once its size and SHA-256 match,+ * so a file under its own name is always verified, and an interrupted download keeps the files it+ * finished.+ */+export class ResourceManager {+ private readonly definitions = new Map<string, ResourceDefinition>();+ private readonly downloads = new Map<string, Download>();++ constructor(+ private readonly baseDir: string,+ private readonly broadcast: (progress: ResourceDownloadProgress) => void+ ) {}++ register(...definitions: ResourceDefinition[]) {+ for (const definition of definitions) {+ validateDefinition(definition);+ if (this.definitions.has(definition.id)) {+ throw new Error(`The resource ${definition.id} is already registered.`);+ }+ this.definitions.set(definition.id, definition);+ }+ }++ /** Returns the directory holding the resource `id`, or `null` while it is not installed. */+ getPath(id: string) {+ const definition = this.getDefinition(id);+ const localCopy = definition.findLocalCopy?.();+ if (localCopy) {+ return localCopy;+ }+ const directory = this.getDirectory(definition);+ return fs.existsSync(path.join(directory, COMPLETE_MARKER)) ? directory : null;+ }++ async getStatus(ids: string[]): Promise<ResourcesStatusResponse> {+ const definitions = ids.map((id) => this.getDefinition(id));+ return {+ resources: definitions.map((definition) => this.getInfo(definition)),+ freeBytes: await this.getFreeBytes(),+ download: this.findDownload(ids)?.progress ?? null+ };+ }++ /**+ * Starts downloading the resources `ids` that are not installed, reporting progress through+ * `broadcast`. Joins a download that already fetches any of them.+ *+ * @returns The download as it starts; `succeeded` already when every resource is installed.+ */+ startDownload(ids: string[]): ResourceDownloadProgress {+ const missing = ids.map((id) => this.getDefinition(id)).filter(({ id }) => !this.getPath(id));+ const running = this.findDownload(ids);+ if (running) {+ return running.progress;+ }++ const download: Download = {+ progress: {+ downloadId: utils.randomString(12),+ resourceIds: ids,+ status: missing.length ? "downloading" : "succeeded",+ receivedBytes: 0,+ totalBytes: sumSizes(missing.flatMap(({ files }) => files)),+ file: null+ },+ controller: new AbortController()+ };+ if (missing.length) {+ this.downloads.set(download.progress.downloadId, download);+ void this.runDownload(download, missing);+ }+ return { ...download.progress };+ }++ cancelDownload(downloadId: string) {+ this.downloads.get(downloadId)?.controller.abort();+ }++ private async runDownload(download: Download, definitions: ResourceDefinition[]) {+ const { progress, controller } = download;+ let lastReported = 0;+ const report = (force: boolean) => {+ if (force || Date.now() - lastReported >= PROGRESS_INTERVAL_MS) {+ lastReported = Date.now();+ this.broadcast({ ...progress });+ }+ };++ try {+ for (const definition of definitions) {+ const directory = this.getDirectory(definition);+ for (const file of definition.files) {+ const destination = path.join(directory, file.path);+ if (isDownloaded(destination, file)) {+ progress.receivedBytes += file.size;+ continue;+ }+ progress.file = path.basename(file.path);+ report(true);+ await fs.promises.mkdir(path.dirname(destination), { recursive: true });+ await downloadFile(file, destination, controller.signal, (bytes) => {+ progress.receivedBytes += bytes;+ report(false);+ });+ }+ await fs.promises.writeFile(path.join(directory, COMPLETE_MARKER), "");+ getLog().info(`Resources: installed ${definition.id} in ${directory}.`);+ }+ progress.status = "succeeded";+ } catch (e) {+ if (controller.signal.aborted) {+ progress.status = "canceled";+ } else {+ progress.status = "failed";+ progress.error = (e as Error)?.message ?? String(e);+ getLog().error(`Resources: downloading ${progress.resourceIds.join(", ")} failed: ${progress.error}`);+ }+ } finally {+ progress.file = null;+ this.downloads.delete(progress.downloadId);+ report(true);+ }+ }++ private getDefinition(id: string) {+ const definition = this.definitions.get(id);+ if (!definition) {+ throw new ValidationError(t("resources.unknown", { id }));+ }+ return definition;+ }++ private getDirectory(definition: ResourceDefinition) {+ return path.join(this.baseDir, definition.id, definition.version);+ }++ private getInfo(definition: ResourceDefinition): ResourceInfo {+ const directory = this.getDirectory(definition);+ const downloaded = definition.files+ .filter((file) => isDownloaded(path.join(directory, file.path), file));+ return {+ id: definition.id,+ title: definition.title(),+ description: definition.description(),+ size: sumSizes(definition.files),+ downloadedSize: sumSizes(downloaded),+ sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],+ installed: this.getPath(definition.id) !== null+ };+ }++ private findDownload(ids: string[]) {+ for (const download of this.downloads.values()) {+ if (download.progress.resourceIds.some((id) => ids.includes(id))) {+ return download;+ }+ }+ return undefined;+ }++ private async getFreeBytes() {+ const directory = fs.existsSync(this.baseDir) ? this.baseDir : path.dirname(this.baseDir);+ try {+ const { bavail, bsize } = await fs.promises.statfs(directory);+ return bavail * bsize;+ } catch {+ return null;+ }+ }+}++/**+ * Downloads `file` to `destination` through `<destination>.partial`, renaming it once its size and+ * SHA-256 match. A failed or aborted download leaves no file behind.+ */+export async function downloadFile(file: ResourceFile, destination: string, signal: AbortSignal,+ onProgress: (bytes: number) => void) {+ const partial = `${destination}.partial`;+ const name = path.basename(file.path);+ try {+ const response = await fetch(file.url, { signal });+ if (!response.ok || !response.body) {+ throw new Error(t("resources.http-error", { name, status: response.status }));+ }++ const hash = createHash("sha256");+ let received = 0;+ const meter = new Transform({+ transform(chunk: Buffer, _encoding, callback) {+ received += chunk.length;+ if (received > file.size) {+ callback(new Error(t("resources.size-mismatch", { name })));+ return;+ }+ hash.update(chunk);+ onProgress(chunk.length);+ callback(null, chunk);+ }+ });+ const body = Readable.fromWeb(response.body as NodeReadableStream<Uint8Array>);+ await pipeline(body, meter, fs.createWriteStream(partial), { signal });++ if (received !== file.size) {+ throw new Error(t("resources.size-mismatch", { name }));+ }+ if (hash.digest("hex") !== file.sha256) {+ throw new Error(t("resources.hash-mismatch", { name }));+ }+ await fs.promises.rename(partial, destination);+ } catch (e) {+ await fs.promises.rm(partial, { force: true });+ throw e;+ }+}++function validateDefinition({ id, version, files }: ResourceDefinition) {+ if (!ID_PATTERN.test(id) || !VERSION_PATTERN.test(version)) {+ throw new Error(`The resource ${id} has an invalid ID or version.`);+ }+ for (const file of files) {+ const segments = file.path.split("/");+ if (segments.some((segment) => !segment || segment === "." || segment === "..")) {+ throw new Error(`The resource ${id} has a file outside its directory: ${file.path}.`);+ }+ }+}++/** Whether `destination` is a finished download of `file`: only a verified file gets its name. */+function isDownloaded(destination: string, file: ResourceFile) {+ try {+ return fs.statSync(destination).size === file.size;+ } catch {+ return false;+ }+}++function sumSizes(files: ResourceFile[]) {+ return files.reduce((total, { size }) => total + size, 0);+}++export default new ResourceManager(dataDirs.RESOURCES_DIR, (download) => {+ ws.sendMessageToAllClients({ type: "resource-download", download });+});diff --git a/packages/commons/src/index.ts b/packages/commons/src/index.tsindex 6a22860188..cae1313b4c 100644--- a/packages/commons/src/index.ts+++ b/packages/commons/src/index.ts@@ -56,5 +56,6 @@ export * from "./lib/marked_extensions.js";// is imported by virtually every client module. Import it via its subpath instead,// e.g. "@triliumnext/commons/src/lib/markdown_renderer".export * from "./lib/task_states.js";+export * from "./lib/resources.js";export * from "./lib/transcript.js";export * from "./lib/keyboard_shortcut_display.js";diff --git a/packages/commons/src/lib/resources.ts b/packages/commons/src/lib/resources.tsnew file mode 100644index 0000000000..65876ce195--- /dev/null+++ b/packages/commons/src/lib/resources.ts@@ -0,0 +1,40 @@+/**+ * Resources are large files a feature needs at run time but that do not ship with the app, such as+ * the speech recognition model. The backend declares each one with its source and SHA-256, and+ * downloads it into the data directory on request; the client asks for resources by ID only.+ */++/** A resource as `GET /api/resources` reports it. */+export interface ResourceInfo {+ id: string;+ title: string;+ description: string;+ /** The size of all its files, in bytes. */+ size: number;+ /** The bytes of its files already downloaded and verified, kept from an interrupted download. */+ downloadedSize: number;+ /** The host names its files are downloaded from. */+ sources: string[];+ installed: boolean;+}++/** A download as it runs, and as the `resource-download` WebSocket message reports it. */+export interface ResourceDownloadProgress {+ downloadId: string;+ resourceIds: string[];+ status: "downloading" | "succeeded" | "failed" | "canceled";+ receivedBytes: number;+ totalBytes: number;+ /** The name of the file being downloaded. */+ file: string | null;+ /** Why the download failed, when `status` is `failed`. */+ error?: string;+}++export interface ResourcesStatusResponse {+ resources: ResourceInfo[];+ /** The free space where resources are stored, in bytes, or `null` when it cannot be read. */+ freeBytes: number | null;+ /** The download that already fetches some of the resources, if one runs. */+ download: ResourceDownloadProgress | null;+}diff --git a/packages/commons/src/lib/ws_api.ts b/packages/commons/src/lib/ws_api.tsindex c93f1993ee..08cfe9aedf 100644--- a/packages/commons/src/lib/ws_api.ts+++ b/packages/commons/src/lib/ws_api.ts@@ -1,4 +1,5 @@import type { LlmStreamChunk } from "./llm_api.js";+import type { ResourceDownloadProgress } from "./resources.js";export interface EntityChange {id?: number | null;@@ -225,4 +226,7 @@ export type WebSocketMessage = AllTaskDefinitions | {*/type: "llm-stream-end";streamId: string;+} | {+ type: "resource-download";+ download: ResourceDownloadProgress;}