Skip to content

Build your personal knowledge base with Trilium Notes

feat(llm): let the LLM create and edit SVG image notes

Elian Doran1cfc8a9ed162
diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx
index 57a7d2bdcc..e422b310bf 100644
--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx
+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallCard.spec.tsx
@@ -401,6 +401,27 @@ describe("ToolCallCard", () => {
expect(json?.querySelector(".llm-chat-written")).toBeNull();
});
+ it("previews an SVG a call wrote as an image, never as live markup", () => {
+ mocks.notes = { s: { type: "image", mime: "image/svg+xml" } };
+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><script>alert(1)</script><circle r=\"4\"/></svg>";
+ const target = renderCard([
+ { id: "1", toolName: "create_note", input: { parentNoteId: "root", title: "Dot", content: svg, type: "image" } },
+ { id: "2", toolName: "get_note", input: { noteId: "s" } },
+ { id: "3", toolName: "set_note_content", input: { noteId: "s", content: svg } },
+ { id: "4", toolName: "append_to_note", input: { noteId: "s", content: "<rect/>" } }
+ ]);
+ const [ created, , rewritten, appended ] = [ ...(target.querySelector(".llm-chat-tool-calls")?.children ?? []) ];
+
+ for (const call of [ created, rewritten ]) {
+ const img = call?.querySelector<HTMLImageElement>(".llm-chat-written img.llm-chat-written-svg");
+ expect(img).not.toBeNull();
+ expect(decodeURIComponent(img?.getAttribute("src") ?? "")).toBe(`data:image/svg+xml;charset=utf-8,${svg}`);
+ }
+ expect(target.querySelector("svg, script")).toBeNull();
+ expect(appended).not.toBeUndefined();
+ expect(appended?.querySelector(".llm-chat-written")).toBeNull();
+ });
+
it("shows what set_note_content wrote by the type and mime it changed the note to", () => {
mocks.notes = { t: { type: "text", mime: "text/html" }, c: { type: "code", mime: "text/x-python" } };
// A call in between keeps the two from grouping into one section.
diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css
index 8b8f05a5aa..be666b515a 100644
--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css
+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.css
@@ -132,3 +132,9 @@
padding-left: 0.5rem;
border-left: 2px solid rgb(60, 160, 60);
}
+
+.llm-chat-written-svg {
+ display: block;
+ max-width: 100%;
+ max-height: 280px;
+}
diff --git a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx
index b59cdb7afc..82c941fb62 100644
--- a/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx
+++ b/apps/client/src/widgets/type_widgets/llm_chat/ToolCallViews.tsx
@@ -17,6 +17,7 @@ import { isFailedToolCall, type ToolCall } from "./llm_chat_types.js";
const HELP_NOTE_PREFIX = "_help_";
const CONTENT_PREVIEW_SOURCE_LENGTH = 1000;
+const SVG_MIME = "image/svg+xml";
/** What a finished call shows: a short summary beside its label, and the view it folds open to. */
export interface ToolCallView {
@@ -35,7 +36,7 @@ export function getToolCallView(toolCall: ToolCall): ToolCallView | null {
const { type, mime, content } = toolCall.input;
if (typeof type !== "string" || typeof content !== "string") return null;
const body = <WrittenContent type={type} mime={typeof mime === "string" ? mime : undefined} content={content} />;
- return hasWrittenContentView(type, content) ? { body } : null;
+ return hasWrittenContentView(type, typeof mime === "string" ? mime : undefined, content) ? { body } : null;
}
if (toolCall.toolName === "set_note_content" || toolCall.toolName === "append_to_note") {
@@ -254,7 +255,8 @@ function NoteResultRow({ noteId, preview, nested, onLinkClick, children }: {
/**
* The content a tool wrote into a note, shown the way its type reads: text as rendered Markdown, code
- * and diagrams as a code block, a web view as its URL. JSON types (canvas, mind map) have no view.
+ * and diagrams as a code block, a web view as its URL, an SVG as an image. JSON types (canvas, mind
+ * map) have no view. The SVG loads through an `<img>`, which runs none of its scripts.
*/
function WrittenContent({ type, mime, content, appended }: { type: string; mime?: string; content: string; appended?: boolean }) {
const noteType = type !== "text" && findNoteType(type, mime);
@@ -269,6 +271,9 @@ function WrittenContent({ type, mime, content, appended }: { type: string; mime?
{type === "webView" && (
<a className="tn-link external" href={content} target="_blank" rel="noopener noreferrer">{content}</a>
)}
+ {type === "image" && (
+ <img className="llm-chat-written-svg" src={`data:${SVG_MIME};charset=utf-8,${encodeURIComponent(content)}`} alt="" />
+ )}
</div>
</div>
);
@@ -289,13 +294,18 @@ function NoteWrittenContent({ noteId, content, appended, type, mime }: {
if (!note) return null;
const writtenType = type ?? note.type;
const writtenMime = mime ?? (writtenType === note.type ? note.mime : undefined);
- if (!hasWrittenContentView(writtenType, content)) return null;
+ if (!hasWrittenContentView(writtenType, writtenMime, content, appended)) return null;
return <WrittenContent type={writtenType} mime={writtenMime} content={content} appended={appended} />;
}
-function hasWrittenContentView(type: string, content: string): boolean {
+/**
+ * An image the LLM writes is always an SVG, so one without a mime is too. Markup appended to an SVG
+ * has no picture of its own.
+ */
+function hasWrittenContentView(type: string, mime: string | undefined, content: string, appended = false): boolean {
if (!content.trim()) return false;
if (type === "webView") return /^https?:\/\//i.test(content.trim());
+ if (type === "image") return !appended && (mime ?? SVG_MIME) === SVG_MIME;
return [ "text", "code", "mermaid", "search" ].includes(type);
}
diff --git a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html
index 3ce947fa92..5560ab2160 100644
--- a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html
+++ b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/AI.html
@@ -159,7 +159,13 @@ class="admonition note">
saved to be able to revert any unwanted changes.</li>
</ul>
</li>
- <li>Create a new note</li>
+ <li>Create a new note
+ <ul>
+ <li>Asked for a drawing, the LLM can create an SVG image note, or turn an
+ existing note into one. It can write only SVG images, not other formats
+ such as PNG.</li>
+ </ul>
+ </li>
<li>Rename or delete a note.</li>
</ul>
</li>
diff --git a/docs/User Guide/User Guide/AI.md b/docs/User Guide/User Guide/AI.md
index 4824b95130..1e8b5c3344 100644
--- a/docs/User Guide/User Guide/AI.md
+++ b/docs/User Guide/User Guide/AI.md
@@ -118,6 +118,7 @@ Here are a few tools that Trilium provides for the LLM:
* When re-writing a note, the LLM can also change its type (for example from a text note to a code note) or the language of a code note, rewriting the content to match.
* Whenever the AI makes a change, a [revision](Basic%20Concepts%20and%20Features/Notes/Note%20Revisions.md) is saved to be able to revert any unwanted changes.
* Create a new note
+ * Asked for a drawing, the LLM can create an SVG image note, or turn an existing note into one. It can write only SVG images, not other formats such as PNG.
* Rename or delete a note.
* At attribute level:
* Get the full list of attributes, or a specific attribute.
diff --git a/packages/trilium-core/src/services/llm/tools/helpers.spec.ts b/packages/trilium-core/src/services/llm/tools/helpers.spec.ts
index 8cc67467d6..c94bd3c66a 100644
--- a/packages/trilium-core/src/services/llm/tools/helpers.spec.ts
+++ b/packages/trilium-core/src/services/llm/tools/helpers.spec.ts
@@ -23,6 +23,7 @@ import {
getDocNoteHtml,
getNoteContentForLlm,
getNoteMeta,
+ getSvgImageError,
registerDocNoteHtmlReader,
setNoteContentFromLlm
} from "./helpers.js";
@@ -358,3 +359,25 @@ describe("applyTextEdits", () => {
expect(result).not.toHaveProperty("content");
});
});
+
+describe("getSvgImageError", () => {
+ it("accepts SVG source, with or without a prolog, doctype or comment", () => {
+ expect(getSvgImageError("image/svg+xml", "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>")).toBeUndefined();
+ expect(getSvgImageError("image/svg+xml", [
+ "<?xml version=\"1.0\"?>",
+ "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" \"x.dtd\">",
+ "<!-- logo -->",
+ "<SVG viewBox=\"0 0 1 1\">",
+ " <rect/>",
+ "</SVG>\n"
+ ].join("\n"))).toBeUndefined();
+ });
+
+ it("rejects another mime, other markup and a truncated SVG", () => {
+ expect(getSvgImageError("image/png", "<svg></svg>")).toBe("Only SVG images are supported; use mime 'image/svg+xml'");
+ const notSvg = "Image content must be complete SVG source: an <svg> root element and its closing </svg>";
+ expect(getSvgImageError("image/svg+xml", "<svgx></svgx>")).toBe(notSvg);
+ expect(getSvgImageError("image/svg+xml", "Here is your drawing: <svg></svg>")).toBe(notSvg);
+ expect(getSvgImageError("image/svg+xml", "<svg><rect/>")).toBe(notSvg);
+ });
+});
diff --git a/packages/trilium-core/src/services/llm/tools/helpers.ts b/packages/trilium-core/src/services/llm/tools/helpers.ts
index c5190524f3..5574e31f43 100644
--- a/packages/trilium-core/src/services/llm/tools/helpers.ts
+++ b/packages/trilium-core/src/services/llm/tools/helpers.ts
@@ -15,11 +15,19 @@ const ATTACHMENT_PREVIEW_MAX_LENGTH = 200;
/** Skip expensive content loading/conversion for notes larger than this. */
const CONTENT_PREVIEW_SIZE_THRESHOLD = 10_000;
-/** Note types the LLM can create or change a note to; each one stores string content. */
+/**
+ * Note types the LLM can create or change a note to; each one stores string content. An `image` must
+ * be an SVG ({@link getSvgImageError}).
+ */
export const LLM_NOTE_TYPES = [
- "text", "code", "render", "book", "mermaid", "canvas", "webView", "relationMap", "search", "mindMap"
+ "text", "code", "render", "book", "mermaid", "canvas", "webView", "relationMap", "search", "mindMap", "image"
] as const;
+export const SVG_MIME = "image/svg+xml";
+
+/** An optional XML prolog, doctype and comments, then an `<svg>` root element through to `</svg>`. */
+const SVG_SOURCE_RE = /^\s*(?:<\?xml[\s\S]*?\?>\s*)?(?:(?:<!--[\s\S]*?-->|<!DOCTYPE[^>]*>)\s*)*<svg[\s>/][\s\S]*<\/svg>\s*$/i;
+
/** Note IDs that must not be deleted, moved, or cloned by the LLM. */
export const PROTECTED_SYSTEM_NOTES = new Set(["root", "_hidden", "_share", "_lbRoot", "_globalNoteMap"]);
@@ -96,6 +104,20 @@ export function setNoteContentFromLlm(note: BNote, content: string) {
}
}
+/**
+ * Why the LLM cannot write `content` into an image note of `mime`, or `undefined` if it can. The LLM
+ * writes only SVG images, and a truncated reply must not leave a broken image behind.
+ */
+export function getSvgImageError(mime: string, content: string): string | undefined {
+ if (mime !== SVG_MIME) {
+ return `Only SVG images are supported; use mime '${SVG_MIME}'`;
+ }
+ if (!SVG_SOURCE_RE.test(content)) {
+ return "Image content must be complete SVG source: an <svg> root element and its closing </svg>";
+ }
+ return undefined;
+}
+
/** A single find-and-replace edit applied to note content. */
export interface TextEdit {
oldText: string;
diff --git a/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts b/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts
index 7cdfc550e2..babf8139a1 100644
--- a/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts
+++ b/packages/trilium-core/src/services/llm/tools/note_tools.spec.ts
@@ -119,6 +119,27 @@ describe("note_tools — write tools return post-write content", () => {
expect(note.setContent).not.toHaveBeenCalled();
});
+ it("turns a note into an SVG image only with SVG content", () => {
+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><rect/></svg>";
+ const note = buildNote({ id: "code5", type: "code", mime: "text/xml", content: svg });
+ withMutableContent(note, svg);
+ note.save = vi.fn() as typeof note.save;
+
+ expect(getTool("set_note_content").execute({ noteId: "code5", content: "<rect/>", type: "image" }))
+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });
+ expect(getTool("set_note_content").execute({ noteId: "code5", content: svg, type: "image", mime: "image/png" }))
+ .toEqual({ error: "Only SVG images are supported; use mime 'image/svg+xml'" });
+ expect(note.type).toBe("code");
+ expect(note.setContent).not.toHaveBeenCalled();
+
+ expect(getTool("set_note_content").execute({ noteId: "code5", content: svg, type: "image" }))
+ .toMatchObject({ success: true, type: "image", mime: "image/svg+xml", content: svg });
+
+ // An SVG image stays one, so a rewrite without a type is checked too.
+ expect(getTool("set_note_content").execute({ noteId: "code5", content: "<svg>" }))
+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });
+ });
+
it("omits type and mime from the result when neither changed", () => {
const note = buildNote({ id: "code4", type: "code", mime: "text/plain", content: "old" });
withMutableContent(note, "old");
@@ -145,6 +166,15 @@ describe("note_tools — write tools return post-write content", () => {
expect(result.success).toBe(true);
expect(result.content).toBe("first line\nsecond line");
});
+
+ it("refuses to append to an SVG image", () => {
+ const note = buildNote({ id: "svg1", type: "image", mime: "image/svg+xml", content: "<svg></svg>" });
+ withMutableContent(note, "<svg></svg>");
+
+ expect(getTool("append_to_note").execute({ noteId: "svg1", content: "<rect/>" }))
+ .toMatchObject({ error: expect.stringContaining("Cannot append to an SVG image") });
+ expect(note.setContent).not.toHaveBeenCalled();
+ });
});
describe("edit_note_content", () => {
@@ -166,6 +196,19 @@ describe("note_tools — write tools return post-write content", () => {
expect(result.content).toBe("const x = 42;\nconst y = 2;");
});
+ it("edits an SVG image, refusing edits that leave it incomplete", () => {
+ const svg = "<svg><rect fill=\"red\"/></svg>";
+ const note = buildNote({ id: "svg2", type: "image", mime: "image/svg+xml", content: svg });
+ withMutableContent(note, svg);
+
+ expect(getTool("edit_note_content").execute({ noteId: "svg2", edits: [{ oldText: "</svg>", newText: "" }] }))
+ .toMatchObject({ error: expect.stringContaining("complete SVG source") });
+ expect(note.setContent).not.toHaveBeenCalled();
+
+ expect(getTool("edit_note_content").execute({ noteId: "svg2", edits: [{ oldText: "red", newText: "blue" }] }))
+ .toMatchObject({ success: true, content: "<svg><rect fill=\"blue\"/></svg>" });
+ });
+
it("rejects text notes and returns no content field", () => {
const note = buildNote({ id: "text2", type: "text", content: "<p>hi</p>" });
withMutableContent(note, "<p>hi</p>");
@@ -324,6 +367,29 @@ describe("note_tools — write tools return post-write content", () => {
parentNoteId: "tparent", title: "T", content: "x", type: "text"
})).toEqual({ error: "Failed to create note" });
});
+
+ it("creates an SVG image note, and refuses other images or content that is not SVG", async () => {
+ buildNote({ id: "iparent", title: "IParent" });
+ const { default: noteService } = await import("../../../services/notes.js");
+ const svg = "<svg xmlns=\"http://www.w3.org/2000/svg\"><circle r=\"4\"/></svg>";
+
+ const created = buildNote({ id: "inew", type: "image", mime: "image/svg+xml", content: svg });
+ withMutableContent(created, svg);
+ vi.mocked(noteService.createNewNote).mockReturnValueOnce({ note: created } as ReturnType<typeof noteService.createNewNote>);
+ const ok = getTool("create_note").execute({ parentNoteId: "iparent", title: "Dot", content: svg, type: "image" });
+ expect(ok).toMatchObject({ success: true, type: "image", content: svg });
+ expect(noteService.createNewNote).toHaveBeenCalledWith(expect.objectContaining({
+ type: "image", mime: "image/svg+xml", content: svg
+ }));
+
+ expect(getTool("create_note").execute({
+ parentNoteId: "iparent", title: "x", content: svg, type: "image", mime: "image/png"
+ })).toEqual({ error: "Only SVG images are supported; use mime 'image/svg+xml'" });
+ expect(getTool("create_note").execute({
+ parentNoteId: "iparent", title: "x", content: "<svg><circle", type: "image"
+ })).toMatchObject({ error: expect.stringContaining("complete SVG source") });
+ expect(noteService.createNewNote).toHaveBeenCalledOnce();
+ });
});
describe("search_notes", () => {
diff --git a/packages/trilium-core/src/services/llm/tools/note_tools.ts b/packages/trilium-core/src/services/llm/tools/note_tools.ts
index 9b980823f6..1dd02f9b04 100644
--- a/packages/trilium-core/src/services/llm/tools/note_tools.ts
+++ b/packages/trilium-core/src/services/llm/tools/note_tools.ts
@@ -11,7 +11,7 @@ import SearchContext from "../../../services/search/search_context.js";
import TaskContext from "../../../services/task_context.js";
import { z } from "zod";
-import { applyTextEdits, getContentPreview, getNoteContentForLlm, getNoteMeta, LLM_NOTE_TYPES, PROTECTED_SYSTEM_NOTES, setNoteContentFromLlm,TOOL_LIMITS } from "./helpers.js";
+import { applyTextEdits, getContentPreview, getNoteContentForLlm, getNoteMeta, getSvgImageError, LLM_NOTE_TYPES, PROTECTED_SYSTEM_NOTES, setNoteContentFromLlm, SVG_MIME, TOOL_LIMITS } from "./helpers.js";
import { defineTools } from "./tool_registry.js";
export const noteTools = defineTools({
@@ -129,11 +129,17 @@ export const noteTools = defineTools({
const typeChanged = newType !== note.type;
let newMime = mime ?? note.mime;
if (typeChanged && !mime) {
- newMime = newType === "code" ? "" : noteTypesService.getDefaultMimeForNoteType(newType);
+ newMime = newType === "image" ? SVG_MIME
+ : newType === "code" ? ""
+ : noteTypesService.getDefaultMimeForNoteType(newType);
}
if (!newMime) {
return { error: "mime is required when changing a note to code" };
}
+ const svgError = newType === "image" ? getSvgImageError(newMime, content) : undefined;
+ if (svgError) {
+ return { error: svgError };
+ }
const mimeChanged = newMime !== note.mime;
note.saveRevision({ source: "llm" });
@@ -171,6 +177,9 @@ export const noteTools = defineTools({
if (!note.hasStringContent()) {
return { error: `Cannot update content for note type: ${note.type}` };
}
+ if (note.type === "image") {
+ return { error: "Cannot append to an SVG image; use edit_note_content or set_note_content instead." };
+ }
const existingContent = note.getContent();
if (typeof existingContent !== "string") {
@@ -244,6 +253,10 @@ export const noteTools = defineTools({
if (!result.ok) {
return { error: result.error };
}
+ const svgError = note.type === "image" ? getSvgImageError(note.mime, result.content) : undefined;
+ if (svgError) {
+ return { error: svgError };
+ }
note.saveRevision({ source: "llm" });
note.setContent(result.content);
@@ -270,6 +283,7 @@ export const noteTools = defineTools({
"- 'relationMap': visual map of note relations (JSON content)",
"- 'search': saved search (content is the search query)",
"- 'mindMap': mind map (JSON content)",
+ "- 'image': an SVG drawing (content is complete SVG source, mime 'image/svg+xml'; no other image formats). For flowcharts, sequence, class or state diagrams prefer 'mermaid'.",
"Common mime values for code notes:",
"'text/javascript' (plain JavaScript, not executed as a Trilium script),",
"'application/javascript;env=frontend' (JavaScript, Trilium frontend script),",
@@ -280,7 +294,7 @@ export const noteTools = defineTools({
inputSchema: z.object({
parentNoteId: z.string().describe("The ID of the parent note. Use 'root' for top-level notes."),
title: z.string().describe("The title of the new note"),
- content: z.string().describe("The content of the note (Markdown for text notes, plain text for code notes, empty string for render notes)"),
+ content: z.string().describe("The content of the note (Markdown for text notes, plain text for code notes, SVG source for image notes, empty string for render notes)"),
type: z.enum(LLM_NOTE_TYPES).describe("The type of note to create."),
mime: z.string().optional().describe("MIME type, REQUIRED for code notes (e.g. 'application/javascript;env=backend', 'text/jsx'). Ignored for other types.")
}),
@@ -289,6 +303,13 @@ export const noteTools = defineTools({
if (type === "code" && !mime) {
return { error: "mime is required when creating code notes" };
}
+ if (type === "image") {
+ mime ??= SVG_MIME;
+ const svgError = getSvgImageError(mime, content);
+ if (svgError) {
+ return { error: svgError };
+ }
+ }
const parentNote = becca.getNote(parentNoteId);
if (!parentNote) {