Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
perf(spreadsheet): render xlsx exports on the backend
Elian Doran6fa5296b00d7
diff --git a/apps/client/src/widgets/type_widgets/spreadsheet/export.tsx b/apps/client/src/widgets/type_widgets/spreadsheet/export.tsxindex 292d728f8e..68559430ad 100644--- a/apps/client/src/widgets/type_widgets/spreadsheet/export.tsx+++ b/apps/client/src/widgets/type_widgets/spreadsheet/export.tsx@@ -1,10 +1,10 @@-import type { ResolvedImage } from "@triliumnext/commons/src/lib/spreadsheet/render_to_xlsx";import { FUniver } from "@univerjs/presets";import { MutableRef } from "preact/hooks";import NoteContext from "../../../components/note_context";import FNote from "../../../entities/fnote";import { t } from "../../../services/i18n";+import server from "../../../services/server";import toast from "../../../services/toast";import utils from "../../../services/utils";import { useTriliumEvent } from "../../react/hooks";@@ -19,9 +19,9 @@ const UTF8_BOM = "\uFEFF";* Exports the spreadsheet when the `exportXlsx` / `exportCsv` events fire for this note* context. The events are raised from the note actions menu and the floating buttons (the* same surfaces the PNG/SVG exports use), so they work regardless of Univer's own toolbar- * (which is hidden in read-only mode). The conversions run client-side via the exporters in- * `@triliumnext/commons`, dynamically imported so their dependencies (e.g. exceljs) are only- * fetched on export (and never enter the standalone/core bundles).+ * (which is hidden in read-only mode). CSV is converted client-side via the exporters in+ * `@triliumnext/commons`, dynamically imported so they are only fetched on export; XLSX is+ * rendered by the backend, which keeps exceljs out of the client bundle entirely.*/export default function useSpreadsheetExport(apiRef: MutableRef<FUniver | undefined>, note: FNote, noteContext: NoteContext | null | undefined) {useTriliumEvent("exportXlsx", ({ ntxId }) => {@@ -39,10 +39,11 @@ async function exportToXlsx(univerAPI: FUniver | undefined, note: FNote) {if (json == null) return;try {- // Dynamic import keeps exceljs out of the main bundle (and out of standalone/core).- const { renderSpreadsheetToXlsx } = await import("@triliumnext/commons/src/lib/spreadsheet/render_to_xlsx");- const buffer = await renderSpreadsheetToXlsx(json, { resolveImage: resolveSpreadsheetImage });- await download(note, "xlsx", new Blob([buffer as BlobPart], { type: XLSX_MIME }));+ // The backend renders the workbook and resolves its attachment images, so the client never+ // loads exceljs. It answers with base64, which is what the data-URL download wants anyway.+ const body = { content: json };+ const { base64 } = await server.post<{ base64: string }>("spreadsheet/xlsx", body);+ utils.triggerDownload(downloadName(note, "xlsx"), `data:${XLSX_MIME};base64,${base64}`);} catch (e) {console.error("[spreadsheet-export] xlsx failed", e);toast.showError(t("spreadsheet.export-failed"));@@ -95,7 +96,11 @@ async function download(note: FNote, extension: string, blob: Blob) {// awaits gets silently blocked once the user-activation is consumed, whereas the// data-URL path used by downloadAsPng/Svg works.const dataUrl = await blobToDataUrl(blob);- utils.triggerDownload(`${note.title || "spreadsheet"}.${extension}`, dataUrl);+ utils.triggerDownload(downloadName(note, extension), dataUrl);+}++function downloadName(note: FNote, extension: string) {+ return `${note.title || "spreadsheet"}.${extension}`;}function blobToDataUrl(blob: Blob): Promise<string> {@@ -106,45 +111,3 @@ function blobToDataUrl(blob: Blob): Promise<string> {reader.readAsDataURL(blob);});}--/**- * Resolves a spreadsheet image source to embeddable bytes for the XLSX exporter. Images are stored- * as `api/attachments/<id>/image/...` URLs (fetched here) or inline `data:` URLs; both are reduced- * to a base64 payload and an exceljs-supported extension. Returns null (image skipped) on a fetch- * failure or an unsupported format — exceljs only embeds png/jpeg/gif.- */-async function resolveSpreadsheetImage(source: string): Promise<ResolvedImage | null> {- try {- const dataUrl = source.startsWith("data:") ? source : await fetchAsDataUrl(source);- if (!dataUrl) return null;-- // Parse `data:<mime>;base64,<payload>` with plain string ops — a regex with a nested- // quantifier over the `;` parameters can backtrack exponentially (ReDoS).- const comma = dataUrl.indexOf(",");- if (comma < 0) return null;- const header = dataUrl.slice(0, comma);- if (!/;base64$/i.test(header)) return null; // only base64 payloads-- const mime = header.slice("data:".length).split(";")[0];- const extension = imageExtensionForMime(mime);- return extension ? { base64: dataUrl.slice(comma + 1), extension } : null;- } catch {- return null;- }-}--async function fetchAsDataUrl(url: string): Promise<string | null> {- const response = await fetch(url);- if (!response.ok) return null;- return blobToDataUrl(await response.blob());-}--function imageExtensionForMime(mime: string): ResolvedImage["extension"] | null {- switch (mime.toLowerCase()) {- case "image/png": return "png";- case "image/jpeg":- case "image/jpg": return "jpeg";- case "image/gif": return "gif";- default: return null; // svg/webp/bmp etc. — exceljs can't embed these- }-}diff --git a/apps/client/vite.config.mts b/apps/client/vite.config.mtsindex 4d78bf6dae..5175a11777 100644--- a/apps/client/vite.config.mts+++ b/apps/client/vite.config.mts@@ -85,11 +85,7 @@ export default defineConfig(() => ({optimizeDeps: {include: ["ckeditor5",- "mathlive",- // Pre-bundle so the first spreadsheet XLSX export (which dynamically imports- // exceljs) doesn't trigger an on-demand re-optimization + dev-server reload- // that aborts the export.- "exceljs"+ "mathlive"]},build: {diff --git a/packages/trilium-core/src/routes/api/spreadsheet.spec.ts b/packages/trilium-core/src/routes/api/spreadsheet.spec.tsnew file mode 100644index 0000000000..bcd586fe7b--- /dev/null+++ b/packages/trilium-core/src/routes/api/spreadsheet.spec.ts@@ -0,0 +1,135 @@+import ExcelJS from "exceljs";+import { beforeAll, describe, expect, it } from "vitest";++import { decodeBase64 } from "../../services/utils/binary";+import { createTextNote } from "../../test/api_fixtures";+import { CoreApiTester } from "../../test/api_tester";++/**+ * Drives the shared spreadsheet render route through {@link CoreApiTester} (no Express), so this+ * spec runs under both the node and standalone (WASM) suites.+ */+let api: CoreApiTester;++/** A 1x1 transparent PNG, small enough to be written out in full. */+const PIXEL_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAA"+ + "C0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";++describe("Spreadsheet API (core)", () => {+ beforeAll(() => {+ api = CoreApiTester.build();+ });++ it("renders a workbook to a readable xlsx, and rejects a non-string content", async () => {+ const cellData = { "0": { "0": { v: "Hello", t: 1 }, "1": { v: 42, t: 2 } } };+ const ws = (await renderWorkbook(workbookJson(cellData))).getWorksheet("Ledger");++ expect(ws?.getCell("A1").value).toBe("Hello");+ expect(ws?.getCell("B1").value).toBe(42);++ const rejected = await api.post("/api/spreadsheet/xlsx", {+ body: { content: { not: "a string" } }+ });+ expect(rejected.status).toBe(400);+ });++ it("embeds an image stored as an attachment", async () => {+ const source = await uploadPixelPng("Attachment image export");+ const wb = await renderWorkbook(workbookJson(cellWithDrawing(source)));++ expect(wb.worksheets[0].getImages().length).toBe(1);+ expect(wb.model.media[0]?.extension).toBe("png");+ });++ it("embeds an inline data URL", async () => {+ const source = `data:image/png;base64,${PIXEL_PNG_BASE64}`;+ const wb = await renderWorkbook(workbookJson(cellWithDrawing(source)));++ expect(wb.worksheets[0].getImages().length).toBe(1);+ expect(wb.model.media[0]?.extension).toBe("png");+ });++ it("skips a drawing it can't turn into embeddable bytes", async () => {+ // A vanished attachment, a format exceljs can't embed, and a source that is neither an+ // attachment URL nor a data URL.+ const sources = [+ "api/attachments/nosuchattachment/image/x.png",+ "data:image/svg+xml;base64,PHN2Zy8+",+ "https://example.com/photo.png"+ ];++ for (const source of sources) {+ const wb = await renderWorkbook(workbookJson(cellWithDrawing(source)));+ expect(wb.worksheets[0].getImages().length).toBe(0);+ }+ });+});++/** Posts the workbook JSON to the route and reads the answer back as a real workbook. */+async function renderWorkbook(json: string): Promise<ExcelJS.Workbook> {+ const res = await api.post<{ base64: string }>("/api/spreadsheet/xlsx", {+ body: { content: json }+ });+ expect(res.status).toBe(200);++ // The decoder can hand back a view into a larger pooled buffer, so copy the bytes into an+ // ArrayBuffer of their own before exceljs unzips it.+ const bytes = decodeBase64(res.body.base64);+ const wb = new ExcelJS.Workbook();+ await wb.xlsx.load(new Uint8Array(bytes).buffer);+ return wb;+}++/** Wraps cell data into a complete single-sheet workbook payload. */+function workbookJson(cellData: Record<string, Record<string, unknown>>): string {+ return JSON.stringify({+ version: 1,+ workbook: {+ sheetOrder: ["s1"],+ styles: {},+ sheets: {+ s1: {+ id: "s1",+ name: "Ledger",+ hidden: 0,+ mergeData: [],+ cellData,+ rowData: {},+ columnData: {}+ }+ }+ }+ });+}++/** Cell data holding one cell image pointing at `source`. */+function cellWithDrawing(source: string) {+ const drawing = { drawingId: "d1", source, transform: { width: 10, height: 10 } };++ return {+ "0": {+ "0": {+ p: { drawings: { d1: drawing }, drawingsOrder: ["d1"] }+ }+ }+ };+}++/** Uploads the pixel PNG and returns the `api/attachments/...` URL it is served from. */+async function uploadPixelPng(title: string): Promise<string> {+ const { noteId } = await createTextNote(api, { title });++ const res = await api.post<{ uploaded: boolean; url: string }>(+ `/api/notes/${noteId}/attachments/upload`,+ {+ file: {+ originalname: "pixel.png",+ mimetype: "image/png",+ buffer: decodeBase64(PIXEL_PNG_BASE64)+ }+ }+ );+ expect(res.body.uploaded).toBe(true);++ return res.body.url;+}diff --git a/packages/trilium-core/src/routes/api/spreadsheet.ts b/packages/trilium-core/src/routes/api/spreadsheet.tsnew file mode 100644index 0000000000..f4023234d3--- /dev/null+++ b/packages/trilium-core/src/routes/api/spreadsheet.ts@@ -0,0 +1,94 @@+import type { ResolvedImage } from "@triliumnext/commons/src/lib/spreadsheet/render_to_xlsx.js";+import type { Request } from "express";++import becca from "../../becca/becca.js";+import { ValidationError } from "../../errors.js";+import { encodeBase64 } from "../../services/utils/binary.js";++// A drawing whose image lives in an attachment, as `persistence.tsx` rewrites it on save.+const ATTACHMENT_IMAGE_SOURCE = /attachments\/([A-Za-z0-9_]+)\/image\//;++/**+ * Renders a spreadsheet workbook to an `.xlsx` file, returned base64-encoded so the client can+ * hand it straight to a data-URL download. The client posts the live editor state instead of a+ * `noteId`, so an export includes edits the debounced save has not flushed yet.+ *+ * `exceljs` is dynamically imported, which both keeps it off the startup path and keeps it out of+ * the client bundle: rendering here is the only reason the browser used to need the library.+ */+async function renderXlsx(req: Request) {+ const { content } = req.body;++ if (typeof content !== "string") {+ throw new ValidationError("'content' must be the workbook JSON.");+ }++ const { renderSpreadsheetToXlsx } = await import(+ "@triliumnext/commons/src/lib/spreadsheet/render_to_xlsx.js"+ );+ const buffer = await renderSpreadsheetToXlsx(content, { resolveImage });++ return { base64: encodeBase64(new Uint8Array(buffer)) };+}++/**+ * Resolves a drawing's `source` to embeddable bytes. Attachment URLs are read straight from becca;+ * inline `data:` URLs already carry their payload. Returns null (image skipped) when the attachment+ * is gone, its content is unavailable (a protected note outside a protected session), or its format+ * is one exceljs cannot embed.+ */+async function resolveImage(source: string): Promise<ResolvedImage | null> {+ if (source.startsWith("data:")) {+ return resolveDataUrl(source);+ }++ const match = ATTACHMENT_IMAGE_SOURCE.exec(source);+ const attachment = match ? becca.getAttachment(match[1]) : null;+ if (!attachment) {+ return null;+ }++ const extension = imageExtensionForMime(attachment.mime);+ if (!extension) {+ return null;+ }++ try {+ return { base64: encodeBase64(attachment.getContent()), extension };+ } catch {+ return null;+ }+}++/**+ * Splits a `data:<mime>;base64,<payload>` URL with plain string ops — a regex over the `;`+ * parameters can backtrack exponentially.+ */+function resolveDataUrl(dataUrl: string): ResolvedImage | null {+ const comma = dataUrl.indexOf(",");+ if (comma < 0) {+ return null;+ }++ const header = dataUrl.slice(0, comma);+ if (!/;base64$/i.test(header)) {+ return null;+ }++ const extension = imageExtensionForMime(header.slice("data:".length).split(";")[0]);+ return extension ? { base64: dataUrl.slice(comma + 1), extension } : null;+}++function imageExtensionForMime(mime: string): ResolvedImage["extension"] | null {+ switch (mime.toLowerCase()) {+ case "image/png": return "png";+ case "image/jpeg":+ case "image/jpg": return "jpeg";+ case "image/gif": return "gif";+ default: return null; // svg/webp/bmp etc. — exceljs can't embed these+ }+}++export default {+ renderXlsx+};diff --git a/packages/trilium-core/src/routes/index.ts b/packages/trilium-core/src/routes/index.tsindex 0488a9254a..4d5d8bf54c 100644--- a/packages/trilium-core/src/routes/index.ts+++ b/packages/trilium-core/src/routes/index.ts@@ -38,6 +38,7 @@ import loginApiRoute from "./api/login";import fontsRoute from "./api/fonts";import ocrRoute from "./api/ocr";import linkEmbedRoute from "./api/link_embed";+import spreadsheetRoute from "./api/spreadsheet";import llmRoute from "./api/llm";// TODO: Deduplicate with routes.ts@@ -169,6 +170,8 @@ export function buildSharedApiRoutes({ route, asyncRoute, asyncRouteWithoutTrans// signature, and a query string ends up in every access log along the way.asyncApiRoute(PST, "/api/link-embed/metadata", linkEmbedRoute.getMetadata);+ asyncApiRoute(PST, "/api/spreadsheet/xlsx", spreadsheetRoute.renderXlsx);+// group of the services below are meant to be executed from the outside// Not transactional: a status read needs no transaction, and one is unopenable during the moment// a restore has the database detached — which is exactly when the wizard is polling hardest.