Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(llm): let the LLM create and edit SVG image notes
Elian Doran1cfc8a9ed162
diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsxindex 57a7d2bdcc..e422b310bf 100644--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx@@ -401,6 +401,27 @@ describe("ToolCallCard", () => {expect(json?.querySelector(".llm-chat-written")).toBeNull();});+ it("previews an SVG a call wrote as an image, never as live markup", () => {+ mocks.notes = { s: { type: "image", mime: "image/svg+xml" } };+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><script>alert(1)</script><circle r=\"4\"/></svg>";+ const target = renderCard([+ { id: "1", toolName: "create_note", input: { parentNoteId: "root", title: "Dot", content: svg, type: "image" } },+ { id: "2", toolName: "get_note", input: { noteId: "s" } },+ { id: "3", toolName: "set_note_content", input: { noteId: "s", content: svg } },+ { id: "4", toolName: "append_to_note", input: { noteId: "s", content: "<rect/>" } }+ ]);+ const [ created, , rewritten, appended ] = [ ...(target.querySelector(".llm-chat-tool-calls")?.children ?? []) ];++ for (const call of [ created, rewritten ]) {+ const img = call?.querySelector<HTMLImageElement>(".llm-chat-written img.llm-chat-written-svg");+ expect(img).not.toBeNull();+ expect(decodeURIComponent(img?.getAttribute("src") ?? "")).toBe(`data:image/svg+xml;charset=utf-8,${svg}`);+ }+ expect(target.querySelector("svg, script")).toBeNull();+ expect(appended).not.toBeUndefined();+ expect(appended?.querySelector(".llm-chat-written")).toBeNull();+ });+it("shows what set_note_content wrote by the type and mime it changed the note to", () => {mocks.notes = { t: { type: "text", mime: "text/html" }, c: { type: "code", mime: "text/x-python" } };// A call in between keeps the two from grouping into one section.diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.cssindex 8b8f05a5aa..be666b515a 100644--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css@@ -132,3 +132,9 @@padding-left: 0.5rem;border-left: 2px solid rgb(60, 160, 60);}++.llm-chat-written-svg {+ display: block;+ max-width: 100%;+ max-height: 280px;+}diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsxindex b59cdb7afc..82c941fb62 100644--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx@@ -17,6 +17,7 @@ import { isFailedToolCall, type ToolCall } from "./llm_chat_types.js";const HELP_NOTE_PREFIX = "_help_";const CONTENT_PREVIEW_SOURCE_LENGTH = 1000;+const SVG_MIME = "image/svg+xml";/** What a finished call shows: a short summary beside its label, and the view it folds open to. */export interface ToolCallView {@@ -35,7 +36,7 @@ export function getToolCallView(toolCall: ToolCall): ToolCallView | null {const { type, mime, content } = toolCall.input;if (typeof type !== "string" || typeof content !== "string") return null;const body = <WrittenContent type={type} mime={typeof mime === "string" ? mime : undefined} content={content} />;- return hasWrittenContentView(type, content) ? { body } : null;+ return hasWrittenContentView(type, typeof mime === "string" ? mime : undefined, content) ? { body } : null;}if (toolCall.toolName === "set_note_content" || toolCall.toolName === "append_to_note") {@@ -254,7 +255,8 @@ function NoteResultRow({ noteId, preview, nested, onLinkClick, children }: {/*** The content a tool wrote into a note, shown the way its type reads: text as rendered Markdown, code- * and diagrams as a code block, a web view as its URL. JSON types (canvas, mind map) have no view.+ * and diagrams as a code block, a web view as its URL, an SVG as an image. JSON types (canvas, mind+ * map) have no view. The SVG loads through an `<img>`, which runs none of its scripts.*/function WrittenContent({ type, mime, content, appended }: { type: string; mime?: string; content: string; appended?: boolean }) {const noteType = type !== "text" && findNoteType(type, mime);@@ -269,6 +271,9 @@ function WrittenContent({ type, mime, content, appended }: { type: string; mime?{type === "webView" && (<a className="tn-link external" href={content} target="_blank" rel="noopener noreferrer">{content}</a>)}+ {type === "image" && (+ <img className="llm-chat-written-svg" src={`data:${SVG_MIME};charset=utf-8,${encodeURIComponent(content)}`} alt="" />+ )}</div></div>);@@ -289,13 +294,18 @@ function NoteWrittenContent({ noteId, content, appended, type, mime }: {if (!note) return null;const writtenType = type ?? note.type;const writtenMime = mime ?? (writtenType === note.type ? note.mime : undefined);- if (!hasWrittenContentView(writtenType, content)) return null;+ if (!hasWrittenContentView(writtenType, writtenMime, content, appended)) return null;return <WrittenContent type={writtenType} mime={writtenMime} content={content} appended={appended} />;}-function hasWrittenContentView(type: string, content: string): boolean {+/**+ * An image the LLM writes is always an SVG, so one without a mime is too. Markup appended to an SVG+ * has no picture of its own.+ */+function hasWrittenContentView(type: string, mime: string | undefined, content: string, appended = false): boolean {if (!content.trim()) return false;if (type === "webView") return /^https?:\/\//i.test(content.trim());+ if (type === "image") return !appended && (mime ?? SVG_MIME) === SVG_MIME;return [ "text", "code", "mermaid", "search" ].includes(type);}diff --git a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.htmlindex 3ce947fa92..5560ab2160 100644--- a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html+++ b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html@@ -159,7 +159,13 @@ class="admonition note">saved to be able to revert any unwanted changes.</li></ul></li>- <li>Create a new note</li>+ <li>Create a new note+ <ul>+ <li>Asked for a drawing, the LLM can create an SVG image note, or turn an+ existing note into one. It can write only SVG images, not other formats+ such as PNG.</li>+ </ul>+ </li><li>Rename or delete a note.</li></ul></li>diff --git a/docs/User Guide/User Guide/AI.md b/docs/User Guide/User Guide/AI.mdindex 4824b95130..1e8b5c3344 100644--- a/docs/User Guide/User Guide/AI.md+++ b/docs/User Guide/User Guide/AI.md@@ -118,6 +118,7 @@ Here are a few tools that Trilium provides for the LLM:* When re-writing a note, the LLM can also change its type (for example from a text note to a code note) or the language of a code note, rewriting the content to match.* Whenever the AI makes a change, a [revision](Basic%20Concepts%20and%20Features/Notes/Note%20Revisions.md) is saved to be able to revert any unwanted changes.* Create a new note+ * Asked for a drawing, the LLM can create an SVG image note, or turn an existing note into one. It can write only SVG images, not other formats such as PNG.* Rename or delete a note.* At attribute level:* Get the full list of attributes, or a specific attribute.diff --git a/packages/trilium-core/src/services/llm/tools/helpers.spec.ts b/packages/trilium-core/src/services/llm/tools/helpers.spec.tsindex 8cc67467d6..c94bd3c66a 100644--- a/packages/trilium-core/src/services/llm/tools/helpers.spec.ts+++ b/packages/trilium-core/src/services/llm/tools/helpers.spec.ts@@ -23,6 +23,7 @@ import {getDocNoteHtml,getNoteContentForLlm,getNoteMeta,+ getSvgImageError,registerDocNoteHtmlReader,setNoteContentFromLlm} from "./helpers.js";@@ -358,3 +359,25 @@ describe("applyTextEdits", () => {expect(result).not.toHaveProperty("content");});});++describe("getSvgImageError", () => {+ it("accepts SVG source, with or without a prolog, doctype or comment", () => {+ expect(getSvgImageError("image/svg+xml", "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>")).toBeUndefined();+ expect(getSvgImageError("image/svg+xml", [+ "<?xml version=\"1.0\"?>",+ "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" \"x.dtd\">",+ "<!-- logo -->",+ "<SVG viewBox=\"0 0 1 1\">",+ " <rect/>",+ "</SVG>\n"+ ].join("\n"))).toBeUndefined();+ });++ it("rejects another mime, other markup and a truncated SVG", () => {+ expect(getSvgImageError("image/png", "<svg></svg>")).toBe("Only SVG images are supported; use mime 'image/svg+xml'");+ const notSvg = "Image content must be complete SVG source: an <svg> root element and its closing </svg>";+ expect(getSvgImageError("image/svg+xml", "<svgx></svgx>")).toBe(notSvg);+ expect(getSvgImageError("image/svg+xml", "Here is your drawing: <svg></svg>")).toBe(notSvg);+ expect(getSvgImageError("image/svg+xml", "<svg><rect/>")).toBe(notSvg);+ });+});diff --git a/packages/trilium-core/src/services/llm/tools/helpers.ts b/packages/trilium-core/src/services/llm/tools/helpers.tsindex c5190524f3..5574e31f43 100644--- a/packages/trilium-core/src/services/llm/tools/helpers.ts+++ b/packages/trilium-core/src/services/llm/tools/helpers.ts@@ -15,11 +15,19 @@ const ATTACHMENT_PREVIEW_MAX_LENGTH = 200;/** Skip expensive content loading/conversion for notes larger than this. */const CONTENT_PREVIEW_SIZE_THRESHOLD = 10_000;-/** Note types the LLM can create or change a note to; each one stores string content. */+/**+ * Note types the LLM can create or change a note to; each one stores string content. An `image` must+ * be an SVG ({@link getSvgImageError}).+ */export const LLM_NOTE_TYPES = [- "text", "code", "render", "book", "mermaid", "canvas", "webView", "relationMap", "search", "mindMap"+ "text", "code", "render", "book", "mermaid", "canvas", "webView", "relationMap", "search", "mindMap", "image"] as const;+export const SVG_MIME = "image/svg+xml";++/** An optional XML prolog, doctype and comments, then an `<svg>` root element through to `</svg>`. */+const SVG_SOURCE_RE = /^\s*(?:<\?xml[\s\S]*?\?>\s*)?(?:(?:<!--[\s\S]*?-->|<!DOCTYPE[^>]*>)\s*)*<svg[\s>/][\s\S]*<\/svg>\s*$/i;+/** Note IDs that must not be deleted, moved, or cloned by the LLM. */export const PROTECTED_SYSTEM_NOTES = new Set(["root", "_hidden", "_share", "_lbRoot", "_globalNoteMap"]);@@ -96,6 +104,20 @@ export function setNoteContentFromLlm(note: BNote, content: string) {}}+/**+ * Why the LLM cannot write `content` into an image note of `mime`, or `undefined` if it can. The LLM+ * writes only SVG images, and a truncated reply must not leave a broken image behind.+ */+export function getSvgImageError(mime: string, content: string): string | undefined {+ if (mime !== SVG_MIME) {+ return `Only SVG images are supported; use mime '${SVG_MIME}'`;+ }+ if (!SVG_SOURCE_RE.test(content)) {+ return "Image content must be complete SVG source: an <svg> root element and its closing </svg>";+ }+ return undefined;+}+/** A single find-and-replace edit applied to note content. */export interface TextEdit {oldText: string;diff --git a/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts b/packages/trilium-core/src/services/llm/tools/note_tools.spec.tsindex 7cdfc550e2..babf8139a1 100644--- a/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts+++ b/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts@@ -119,6 +119,27 @@ describe("note_tools — write tools return post-write content", () => {expect(note.setContent).not.toHaveBeenCalled();});+ it("turns a note into an SVG image only with SVG content", () => {+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><rect/></svg>";+ const note = buildNote({ id: "code5", type: "code", mime: "text/xml", content: svg });+ withMutableContent(note, svg);+ note.save = vi.fn() as typeof note.save;++ expect(getTool("set_note_content").execute({ noteId: "code5", content: "<rect/>", type: "image" }))+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });+ expect(getTool("set_note_content").execute({ noteId: "code5", content: svg, type: "image", mime: "image/png" }))+ .toEqual({ error: "Only SVG images are supported; use mime 'image/svg+xml'" });+ expect(note.type).toBe("code");+ expect(note.setContent).not.toHaveBeenCalled();++ expect(getTool("set_note_content").execute({ noteId: "code5", content: svg, type: "image" }))+ .toMatchObject({ success: true, type: "image", mime: "image/svg+xml", content: svg });++ // An SVG image stays one, so a rewrite without a type is checked too.+ expect(getTool("set_note_content").execute({ noteId: "code5", content: "<svg>" }))+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });+ });+it("omits type and mime from the result when neither changed", () => {const note = buildNote({ id: "code4", type: "code", mime: "text/plain", content: "old" });withMutableContent(note, "old");@@ -145,6 +166,15 @@ describe("note_tools — write tools return post-write content", () => {expect(result.success).toBe(true);expect(result.content).toBe("first line\nsecond line");});++ it("refuses to append to an SVG image", () => {+ const note = buildNote({ id: "svg1", type: "image", mime: "image/svg+xml", content: "<svg></svg>" });+ withMutableContent(note, "<svg></svg>");++ expect(getTool("append_to_note").execute({ noteId: "svg1", content: "<rect/>" }))+ .toMatchObject({ error: expect.stringContaining("Cannot append to an SVG image") });+ expect(note.setContent).not.toHaveBeenCalled();+ });});describe("edit_note_content", () => {@@ -166,6 +196,19 @@ describe("note_tools — write tools return post-write content", () => {expect(result.content).toBe("const x = 42;\nconst y = 2;");});+ it("edits an SVG image, refusing edits that leave it incomplete", () => {+ const svg = "<svg><rect fill=\"red\"/></svg>";+ const note = buildNote({ id: "svg2", type: "image", mime: "image/svg+xml", content: svg });+ withMutableContent(note, svg);++ expect(getTool("edit_note_content").execute({ noteId: "svg2", edits: [{ oldText: "</svg>", newText: "" }] }))+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });+ expect(note.setContent).not.toHaveBeenCalled();++ expect(getTool("edit_note_content").execute({ noteId: "svg2", edits: [{ oldText: "red", newText: "blue" }] }))+ .toMatchObject({ success: true, content: "<svg><rect fill=\"blue\"/></svg>" });+ });+it("rejects text notes and returns no content field", () => {const note = buildNote({ id: "text2", type: "text", content: "<p>hi</p>" });withMutableContent(note, "<p>hi</p>");@@ -324,6 +367,29 @@ describe("note_tools — write tools return post-write content", () => {parentNoteId: "tparent", title: "T", content: "x", type: "text"})).toEqual({ error: "Failed to create note" });});++ it("creates an SVG image note, and refuses other images or content that is not SVG", async () => {+ buildNote({ id: "iparent", title: "IParent" });+ const { default: noteService } = await import("../../../services/notes.js");+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><circle r=\"4\"/></svg>";++ const created = buildNote({ id: "inew", type: "image", mime: "image/svg+xml", content: svg });+ withMutableContent(created, svg);+ vi.mocked(noteService.createNewNote).mockReturnValueOnce({ note: created } as ReturnType<typeof noteService.createNewNote>);+ const ok = getTool("create_note").execute({ parentNoteId: "iparent", title: "Dot", content: svg, type: "image" });+ expect(ok).toMatchObject({ success: true, type: "image", content: svg });+ expect(noteService.createNewNote).toHaveBeenCalledWith(expect.objectContaining({+ type: "image", mime: "image/svg+xml", content: svg+ }));++ expect(getTool("create_note").execute({+ parentNoteId: "iparent", title: "x", content: svg, type: "image", mime: "image/png"+ })).toEqual({ error: "Only SVG images are supported; use mime 'image/svg+xml'" });+ expect(getTool("create_note").execute({+ parentNoteId: "iparent", title: "x", content: "<svg><circle", type: "image"+ })).toMatchObject({ error: expect.stringContaining("complete SVG source") });+ expect(noteService.createNewNote).toHaveBeenCalledOnce();+ });});describe("search_notes", () => {diff --git a/packages/trilium-core/src/services/llm/tools/note_tools.ts b/packages/trilium-core/src/services/llm/tools/note_tools.tsindex 9b980823f6..1dd02f9b04 100644--- a/packages/trilium-core/src/services/llm/tools/note_tools.ts+++ b/packages/trilium-core/src/services/llm/tools/note_tools.ts@@ -11,7 +11,7 @@ import SearchContext from "../../../services/search/search_context.js";import TaskContext from "../../../services/task_context.js";import { z } from "zod";-import { applyTextEdits, getContentPreview, getNoteContentForLlm, getNoteMeta, LLM_NOTE_TYPES, PROTECTED_SYSTEM_NOTES, setNoteContentFromLlm,TOOL_LIMITS } from "./helpers.js";+import { applyTextEdits, getContentPreview, getNoteContentForLlm, getNoteMeta, getSvgImageError, LLM_NOTE_TYPES, PROTECTED_SYSTEM_NOTES, setNoteContentFromLlm, SVG_MIME, TOOL_LIMITS } from "./helpers.js";import { defineTools } from "./tool_registry.js";export const noteTools = defineTools({@@ -129,11 +129,17 @@ export const noteTools = defineTools({const typeChanged = newType !== note.type;let newMime = mime ?? note.mime;if (typeChanged && !mime) {- newMime = newType === "code" ? "" : noteTypesService.getDefaultMimeForNoteType(newType);+ newMime = newType === "image" ? SVG_MIME+ : newType === "code" ? ""+ : noteTypesService.getDefaultMimeForNoteType(newType);}if (!newMime) {return { error: "mime is required when changing a note to code" };}+ const svgError = newType === "image" ? getSvgImageError(newMime, content) : undefined;+ if (svgError) {+ return { error: svgError };+ }const mimeChanged = newMime !== note.mime;note.saveRevision({ source: "llm" });@@ -171,6 +177,9 @@ export const noteTools = defineTools({if (!note.hasStringContent()) {return { error: `Cannot update content for note type: ${note.type}` };}+ if (note.type === "image") {+ return { error: "Cannot append to an SVG image; use edit_note_content or set_note_content instead." };+ }const existingContent = note.getContent();if (typeof existingContent !== "string") {@@ -244,6 +253,10 @@ export const noteTools = defineTools({if (!result.ok) {return { error: result.error };}+ const svgError = note.type === "image" ? getSvgImageError(note.mime, result.content) : undefined;+ if (svgError) {+ return { error: svgError };+ }note.saveRevision({ source: "llm" });note.setContent(result.content);@@ -270,6 +283,7 @@ export const noteTools = defineTools({"- 'relationMap': visual map of note relations (JSON content)","- 'search': saved search (content is the search query)","- 'mindMap': mind map (JSON content)",+ "- 'image': an SVG drawing (content is complete SVG source, mime 'image/svg+xml'; no other image formats). For flowcharts, sequence, class or state diagrams prefer 'mermaid'.","Common mime values for code notes:","'text/javascript' (plain JavaScript, not executed as a Trilium script),","'application/javascript;env=frontend' (JavaScript, Trilium frontend script),",@@ -280,7 +294,7 @@ export const noteTools = defineTools({inputSchema: z.object({parentNoteId: z.string().describe("The ID of the parent note. Use 'root' for top-level notes."),title: z.string().describe("The title of the new note"),- content: z.string().describe("The content of the note (Markdown for text notes, plain text for code notes, empty string for render notes)"),+ content: z.string().describe("The content of the note (Markdown for text notes, plain text for code notes, SVG source for image notes, empty string for render notes)"),type: z.enum(LLM_NOTE_TYPES).describe("The type of note to create."),mime: z.string().optional().describe("MIME type, REQUIRED for code notes (e.g. 'application/javascript;env=backend', 'text/jsx'). Ignored for other types.")}),@@ -289,6 +303,13 @@ export const noteTools = defineTools({if (type === "code" && !mime) {return { error: "mime is required when creating code notes" };}+ if (type === "image") {+ mime ??= SVG_MIME;+ const svgError = getSvgImageError(mime, content);+ if (svgError) {+ return { error: svgError };+ }+ }const parentNote = becca.getNote(parentNoteId);if (!parentNote) {