Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: 对话框支持粘贴截图提问

shisonghongfd74b1f1ec17
diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json
index 0299b53b..dd08e040 100644
--- a/src-tauri/tauri.conf.json
+++ b/src-tauri/tauri.conf.json
@@ -22,7 +22,7 @@
}
],
"security": {
- "csp": "default-src 'self'; connect-src 'self' https: http:; img-src 'self' asset: https://asset.localhost; media-src 'self' asset: https://asset.localhost; style-src 'self' 'unsafe-inline'",
+ "csp": "default-src 'self'; connect-src 'self' https: http:; img-src 'self' asset: http://asset.localhost https://asset.localhost blob: data:; media-src 'self' asset: http://asset.localhost https://asset.localhost blob: data:; style-src 'self' 'unsafe-inline'",
"assetProtocol": {
"enable": true,
"scope": ["**"]
diff --git a/src/components/chat/chat-input.tsx b/src/components/chat/chat-input.tsx
index 28e4e997..7c1288e8 100644
--- a/src/components/chat/chat-input.tsx
+++ b/src/components/chat/chat-input.tsx
@@ -1,9 +1,18 @@
import { useEffect, useRef, useState, useCallback } from "react"
-import { FileSearch, Globe2, Send, Square } from "lucide-react"
+import { FileSearch, Globe2, ImagePlus, Send, Square, X } from "lucide-react"
import { useTranslation } from "react-i18next"
import { Button } from "@/components/ui/button"
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
import { isImeComposing } from "@/lib/keyboard-utils"
+import type { MessageImage } from "@/stores/chat-store"
+import {
+ MAX_IMAGE_BYTES,
+ MAX_IMAGE_MB,
+ MAX_IMAGES_PER_MESSAGE,
+ fileToMessageImage,
+ isAcceptedImageType,
+ messageImageToDataUrl,
+} from "@/lib/chat-image-utils"
export interface ChatSendOptions {
useWebSearch: boolean
@@ -11,7 +20,7 @@ export interface ChatSendOptions {
}
interface ChatInputProps {
- onSend: (text: string, options: ChatSendOptions) => void
+ onSend: (text: string, images: MessageImage[], options: ChatSendOptions) => void
onStop: () => void
isStreaming: boolean
anyTxtAvailable?: boolean
@@ -23,12 +32,92 @@ export function ChatInput({ onSend, onStop, isStreaming, anyTxtAvailable = true,
const [value, setValue] = useState("")
const [useWebSearch, setUseWebSearch] = useState(false)
const [useAnyTxtSearch, setUseAnyTxtSearch] = useState(false)
+ const [images, setImages] = useState<MessageImage[]>([])
+ const [imageError, setImageError] = useState<string | null>(null)
const textareaRef = useRef<HTMLTextAreaElement>(null)
+ const fileInputRef = useRef<HTMLInputElement>(null)
useEffect(() => {
if (!anyTxtAvailable) setUseAnyTxtSearch(false)
}, [anyTxtAvailable])
+ // Validate + decode a batch of files (from paste, drop, or the file
+ // picker) and append the accepted ones to `images`. Rejections set a
+ // transient error message rather than throwing — one bad file should
+ // never block the good ones in the same batch.
+ const addFiles = useCallback(
+ async (files: File[]) => {
+ const imageFiles = files.filter((f) => f.type.startsWith("image/"))
+ if (imageFiles.length === 0) return
+ let error: string | null = null
+ const accepted: MessageImage[] = []
+ // Read current count via the functional updater below; here we
+ // pre-compute remaining slots from the latest render's state.
+ let remaining = MAX_IMAGES_PER_MESSAGE - images.length
+ for (const file of imageFiles) {
+ if (remaining <= 0) {
+ error = t("chat.tooManyImages", { max: MAX_IMAGES_PER_MESSAGE })
+ break
+ }
+ if (!isAcceptedImageType(file.type)) {
+ error = t("chat.unsupportedImageType", { type: file.type || "?" })
+ continue
+ }
+ if (file.size > MAX_IMAGE_BYTES) {
+ error = t("chat.imageTooLarge", { max: MAX_IMAGE_MB, name: file.name || "image" })
+ continue
+ }
+ try {
+ accepted.push(await fileToMessageImage(file))
+ remaining -= 1
+ } catch {
+ error = t("chat.unsupportedImageType", { type: file.type || "?" })
+ }
+ }
+ if (accepted.length > 0) {
+ setImages((prev) => [...prev, ...accepted].slice(0, MAX_IMAGES_PER_MESSAGE))
+ }
+ setImageError(error)
+ },
+ [images.length, t],
+ )
+
+ const handlePaste = useCallback(
+ (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
+ const items = e.clipboardData?.items
+ if (!items) return
+ const files: File[] = []
+ for (const item of items) {
+ if (item.kind === "file" && item.type.startsWith("image/")) {
+ const file = item.getAsFile()
+ if (file) files.push(file)
+ }
+ }
+ if (files.length > 0) {
+ // Prevent the image's stray name/path from landing in the
+ // textarea as text on browsers that surface both.
+ e.preventDefault()
+ void addFiles(files)
+ }
+ },
+ [addFiles],
+ )
+
+ const handleFilePick = useCallback(
+ (e: React.ChangeEvent<HTMLInputElement>) => {
+ const files = e.target.files ? Array.from(e.target.files) : []
+ void addFiles(files)
+ // Reset so picking the same file again still fires onChange.
+ e.target.value = ""
+ },
+ [addFiles],
+ )
+
+ const removeImage = useCallback((index: number) => {
+ setImages((prev) => prev.filter((_, i) => i !== index))
+ setImageError(null)
+ }, [])
+
const handleInput = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
setValue(e.target.value)
const ta = e.target
@@ -38,13 +127,16 @@ export function ChatInput({ onSend, onStop, isStreaming, anyTxtAvailable = true,
const handleSend = useCallback(() => {
const trimmed = value.trim()
- if (!trimmed || isStreaming) return
- onSend(trimmed, { useWebSearch, useAnyTxtSearch })
+ // Allow image-only messages: send if there's text OR at least one image.
+ if ((!trimmed && images.length === 0) || isStreaming) return
+ onSend(trimmed, images, { useWebSearch, useAnyTxtSearch })
setValue("")
+ setImages([])
+ setImageError(null)
if (textareaRef.current) {
textareaRef.current.style.height = "auto"
}
- }, [value, isStreaming, onSend, useWebSearch, useAnyTxtSearch])
+ }, [value, images, isStreaming, onSend, useWebSearch, useAnyTxtSearch])
const handleKeyDown = useCallback(
(e: React.KeyboardEvent<HTMLTextAreaElement>) => {
@@ -71,20 +163,63 @@ export function ChatInput({ onSend, onStop, isStreaming, anyTxtAvailable = true,
return (
<div className="border-t bg-background/95 p-3">
<div className="rounded-lg border border-border/80 bg-card/80 p-2 shadow-sm ring-1 ring-black/5 focus-within:border-ring/60 focus-within:ring-ring/20 dark:ring-white/5">
+ {images.length > 0 && (
+ <div className="mb-2 flex flex-wrap gap-2 px-1">
+ {images.map((img, i) => (
+ <div key={i} className="group relative h-16 w-16 overflow-hidden rounded-md border border-border/70">
+ <img
+ src={messageImageToDataUrl(img)}
+ alt=""
+ className="h-full w-full object-cover"
+ />
+ <button
+ type="button"
+ onClick={() => removeImage(i)}
+ className="absolute right-0.5 top-0.5 rounded-full bg-background/80 p-0.5 text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-destructive group-hover:opacity-100"
+ title={t("chat.removeImage")}
+ >
+ <X className="h-3 w-3" />
+ </button>
+ </div>
+ ))}
+ </div>
+ )}
+ {imageError && (
+ <p className="mb-1 px-1 text-xs text-destructive">{imageError}</p>
+ )}
<textarea
ref={textareaRef}
value={value}
dir="auto"
onChange={handleInput}
onKeyDown={handleKeyDown}
+ onPaste={handlePaste}
placeholder={placeholder ?? "Type a message... (Enter to send, Shift+Enter for newline)"}
disabled={isStreaming}
rows={1}
className="block w-full resize-none border-0 bg-transparent px-2 py-2 text-sm leading-6 placeholder:text-muted-foreground focus:outline-none disabled:cursor-not-allowed disabled:opacity-50"
style={{ maxHeight: "120px", overflowY: "auto" }}
/>
+ <input
+ ref={fileInputRef}
+ type="file"
+ accept="image/png,image/jpeg,image/webp,image/gif"
+ multiple
+ className="hidden"
+ onChange={handleFilePick}
+ />
<div className="mt-1 flex items-center justify-between gap-3 border-t border-border/50 pt-2">
<div className="flex min-w-0 flex-wrap items-center gap-1.5">
+ <button
+ type="button"
+ onClick={() => fileInputRef.current?.click()}
+ disabled={isStreaming || images.length >= MAX_IMAGES_PER_MESSAGE}
+ className={searchToggleClass(false)}
+ title={t("chat.attachImage")}
+ >
+ <ImagePlus className="h-3.5 w-3.5" />
+ <span className="hidden sm:inline">{t("chat.attachImage")}</span>
+ </button>
<button
type="button"
aria-pressed={useWebSearch}
@@ -146,7 +281,7 @@ export function ChatInput({ onSend, onStop, isStreaming, anyTxtAvailable = true,
<Button
size="sm"
onClick={handleSend}
- disabled={!value.trim()}
+ disabled={!value.trim() && images.length === 0}
className="h-8 shrink-0 gap-1.5 rounded-md px-3"
title={t("chat.sendMessage")}
>
diff --git a/src/components/chat/chat-message.tsx b/src/components/chat/chat-message.tsx
index ab37ca0c..904dc0ea 100644
--- a/src/components/chat/chat-message.tsx
+++ b/src/components/chat/chat-message.tsx
@@ -20,7 +20,9 @@ import { convertLatexToUnicode } from "@/lib/latex-to-unicode"
import { normalizePath, getFileName } from "@/lib/path-utils"
import { makeQueryFileName } from "@/lib/wiki-filename"
import { hasUsableLlm } from "@/lib/has-usable-llm"
+import { messageImageToDataUrl } from "@/lib/chat-image-utils"
import { resolveMarkdownImageSrc } from "@/lib/markdown-image-resolver"
+import { transformImageEmbeds } from "@/lib/wikilink-transform"
import { findRawSourceForImage, imageUrlToAbsolute } from "@/lib/raw-source-resolver"
import { detectLanguage } from "@/lib/detect-language"
import { getHtmlLang, getTextDirection } from "@/lib/language-metadata"
@@ -90,19 +92,34 @@ function ChatMessageImpl({ message, isLastAssistant, onRegenerate }: ChatMessage
{isUser ? <User className="h-4 w-4" /> : <Bot className="h-4 w-4" />}
</div>
<div className="max-w-[80%] flex flex-col gap-1.5">
- <div
- className={`rounded-lg px-3 py-2 text-sm ${
- isUser
- ? "bg-primary text-primary-foreground"
- : "bg-muted text-foreground"
- }`}
- >
- {isUser ? (
- <p dir="auto" className="whitespace-pre-wrap break-words">{message.content}</p>
- ) : (
- <MarkdownContent content={message.content} />
- )}
- </div>
+ {isUser && message.images && message.images.length > 0 && (
+ <div className={`flex flex-wrap gap-1.5 ${isUser ? "justify-end" : ""}`}>
+ {message.images.map((img, i) => (
+ <img
+ key={i}
+ src={messageImageToDataUrl(img)}
+ alt=""
+ className="max-h-40 max-w-[180px] rounded-lg border border-border/40 object-contain"
+ loading="lazy"
+ />
+ ))}
+ </div>
+ )}
+ {(!isUser || message.content) && (
+ <div
+ className={`rounded-lg px-3 py-2 text-sm ${
+ isUser
+ ? "bg-primary text-primary-foreground"
+ : "bg-muted text-foreground"
+ }`}
+ >
+ {isUser ? (
+ <p dir="auto" className="whitespace-pre-wrap break-words">{message.content}</p>
+ ) : (
+ <MarkdownContent content={message.content} />
+ )}
+ </div>
+ )}
{isAssistant && <CitedReferencesPanel content={message.content} savedReferences={message.references} />}
{isAssistant && hovered && (
<div className="flex items-center gap-1">
@@ -899,6 +916,12 @@ function ThinkingBlock({ content }: { content: string }) {
function processContent(text: string): string {
let result = text
+ // Rewrite Obsidian image embeds (`![[…]]`) into standard markdown
+ // FIRST — before the `[[…]]` → wikilink conversion below, which
+ // would otherwise mangle the embed target into a broken
+ // `wikilink:` image. Same rule the wiki reader / raw preview use.
+ result = transformImageEmbeds(result)
+
// Wrap bare \begin{...}...\end{...} blocks with $$ for remark-math
result = result.replace(
/(?<!\$\$\s*)(\\begin\{[^}]+\}[\s\S]*?\\end\{[^}]+\})(?!\s*\$\$)/g,
diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx
index 0571ca27..59e7389b 100644
--- a/src/components/chat/chat-panel.tsx
+++ b/src/components/chat/chat-panel.tsx
@@ -4,7 +4,7 @@ import { BookOpen, Plus, Trash2, MessageSquare } from "lucide-react"
import { Button } from "@/components/ui/button"
import { ChatMessage, StreamingMessage, useSourceFiles } from "./chat-message"
import { ChatInput, type ChatSendOptions } from "./chat-input"
-import { useChatStore, chatMessagesToLLM, type MessageReference } from "@/stores/chat-store"
+import { useChatStore, chatMessagesToLLM, type MessageReference, type MessageImage } from "@/stores/chat-store"
import { useWikiStore } from "@/stores/wiki-store"
import { streamChat, type ChatMessage as LLMMessage } from "@/lib/llm-client"
import { executeIngestWrites } from "@/lib/ingest"
@@ -175,14 +175,18 @@ export function ChatPanel() {
}, [activeMessages, streamingContent])
const handleSend = useCallback(
- async (text: string, options: ChatSendOptions = { useWebSearch: false, useAnyTxtSearch: false }) => {
+ async (
+ text: string,
+ images: MessageImage[] = [],
+ options: ChatSendOptions = { useWebSearch: false, useAnyTxtSearch: false },
+ ) => {
// Auto-create a conversation if none is active
let convId = useChatStore.getState().activeConversationId
if (!convId) {
convId = createConversation()
}
- addMessage("user", text)
+ addMessage("user", text, images)
setStreaming(true)
// Build system prompt with wiki context using graph-enhanced retrieval
@@ -444,9 +448,31 @@ export function ChatPanel() {
const lastIdx = llmMessages.length - 1
const last = llmMessages[lastIdx]
if (last && last.role === "user") {
+ // The final user turn may now be either a plain string
+ // (text-only) or a ContentBlock[] (carries images). Prepend
+ // the language reminder to the textual part in both shapes —
+ // string-concat for the legacy form, and into the first text
+ // block (or a new leading text block) for the multimodal form.
+ const prefix = `[${langReminder}]\n\n`
+ let newContent: LLMMessage["content"]
+ if (typeof last.content === "string") {
+ newContent = `${prefix}${last.content}`
+ } else {
+ const blocks = [...last.content]
+ const firstTextIdx = blocks.findIndex((b) => b.type === "text")
+ if (firstTextIdx >= 0) {
+ const tb = blocks[firstTextIdx]
+ if (tb.type === "text") {
+ blocks[firstTextIdx] = { type: "text", text: `${prefix}${tb.text}` }
+ }
+ } else {
+ blocks.unshift({ type: "text", text: prefix })
+ }
+ newContent = blocks
+ }
llmMessages = [
...llmMessages.slice(0, lastIdx),
- { role: "user", content: `[${langReminder}]\n\n${last.content}` },
+ { role: "user", content: newContent },
]
}
}
@@ -529,7 +555,9 @@ export function ChatPanel() {
messages: s.messages.filter((m) => m.id !== lastUser.id),
}))
}
- handleSend(lastUserMsg.content)
+ // Re-send with the original text AND images so a regenerated turn
+ // keeps the same vision context.
+ handleSend(lastUserMsg.content, lastUserMsg.images ?? [])
}, [isStreaming, removeLastAssistantMessage, handleSend])
const handleWriteToWiki = useCallback(async () => {
diff --git a/src/i18n/en.json b/src/i18n/en.json
index 82a39268..d766c43c 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -139,7 +139,13 @@
"useAnyTxtSearch": "AnyTXT search",
"enableAnyTxtInSettings": "Enable AnyTXT in Settings → External Information Sources",
"sendMessage": "Send message",
- "stopGeneration": "Stop generation"
+ "stopGeneration": "Stop generation",
+ "attachImage": "Attach image",
+ "removeImage": "Remove image",
+ "imageMessage": "Image",
+ "imageTooLarge": "Image too large (max {{max}}MB): {{name}}",
+ "tooManyImages": "Too many images (max {{max}})",
+ "unsupportedImageType": "Unsupported image type: {{type}}"
},
"search": {
"title": "Search",
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index eeb4580c..38addaf7 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -139,7 +139,13 @@
"useAnyTxtSearch": "AnyTXT 搜索",
"enableAnyTxtInSettings": "请在设置 → 外部信息源中启用 AnyTXT",
"sendMessage": "发送消息",
- "stopGeneration": "停止生成"
+ "stopGeneration": "停止生成",
+ "attachImage": "添加图片",
+ "removeImage": "移除图片",
+ "imageMessage": "图片",
+ "imageTooLarge": "图片过大(最大 {{max}}MB):{{name}}",
+ "tooManyImages": "图片数量过多(最多 {{max}} 张)",
+ "unsupportedImageType": "不支持的图片类型:{{type}}"
},
"search": {
"title": "搜索",
diff --git a/src/lib/chat-image-utils.ts b/src/lib/chat-image-utils.ts
new file mode 100644
index 00000000..08e7f91c
--- /dev/null
+++ b/src/lib/chat-image-utils.ts
@@ -0,0 +1,45 @@
+import type { MessageImage } from "@/stores/chat-store"
+
+/** Image MIME types every vision-capable provider on our wire accepts. */
+export const ACCEPTED_IMAGE_TYPES = ["image/png", "image/jpeg", "image/webp", "image/gif"] as const
+
+/** Per-image size cap. base64 inflates bytes ~33% and chat history is
+ * persisted to JSON, so we keep this conservative. */
+export const MAX_IMAGE_BYTES = 5 * 1024 * 1024 // 5 MB
+export const MAX_IMAGE_MB = 5
+
+/** Max images attachable to a single message. */
+export const MAX_IMAGES_PER_MESSAGE = 5
+
+export function isAcceptedImageType(type: string): boolean {
+ return (ACCEPTED_IMAGE_TYPES as readonly string[]).includes(type)
+}
+
+/**
+ * Read a File/Blob into a {@link MessageImage} — raw base64 (no
+ * `data:` prefix; the provider translators add their own framing).
+ * FileReader's `readAsDataURL` gives us `data:<mime>;base64,<payload>`;
+ * we strip the prefix and keep the payload + the file's mediaType.
+ */
+export function fileToMessageImage(file: File | Blob): Promise<MessageImage> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader()
+ reader.onerror = () => reject(reader.error ?? new Error("Failed to read image"))
+ reader.onload = () => {
+ const result = reader.result
+ if (typeof result !== "string") {
+ reject(new Error("Unexpected FileReader result"))
+ return
+ }
+ const comma = result.indexOf(",")
+ const dataBase64 = comma >= 0 ? result.slice(comma + 1) : result
+ resolve({ mediaType: file.type || "image/png", dataBase64 })
+ }
+ reader.readAsDataURL(file)
+ })
+}
+
+/** Build the `data:` URL for rendering a stored image in an <img>. */
+export function messageImageToDataUrl(img: MessageImage): string {
+ return `data:${img.mediaType};base64,${img.dataBase64}`
+}
diff --git a/src/lib/llm-client.ts b/src/lib/llm-client.ts
index 9df28699..616c9f7e 100644
--- a/src/lib/llm-client.ts
+++ b/src/lib/llm-client.ts
@@ -4,7 +4,7 @@ import { getProviderConfig, type RequestOverrides } from "./llm-providers"
import { getHttpFetch, isFetchNetworkError } from "./tauri-fetch"
import { countReasoningCharsInLine, extractReasoningTextFromLine } from "./reasoning-detector"
-export type { ChatMessage, RequestOverrides } from "./llm-providers"
+export type { ChatMessage, ContentBlock, RequestOverrides } from "./llm-providers"
export { isFetchNetworkError } from "./tauri-fetch"
export interface StreamCallbacks {
diff --git a/src/stores/chat-messages-to-llm.test.ts b/src/stores/chat-messages-to-llm.test.ts
new file mode 100644
index 00000000..45652eb8
--- /dev/null
+++ b/src/stores/chat-messages-to-llm.test.ts
@@ -0,0 +1,52 @@
+import { describe, it, expect } from "vitest"
+import { chatMessagesToLLM, type DisplayMessage } from "./chat-store"
+
+function msg(partial: Partial<DisplayMessage> & Pick<DisplayMessage, "role" | "content">): DisplayMessage {
+ return {
+ id: "1",
+ timestamp: 0,
+ conversationId: "c1",
+ ...partial,
+ }
+}
+
+describe("chatMessagesToLLM", () => {
+ it("keeps the legacy string shape when a message has no images", () => {
+ const out = chatMessagesToLLM([msg({ role: "user", content: "hello" })])
+ expect(out).toEqual([{ role: "user", content: "hello" }])
+ expect(typeof out[0].content).toBe("string")
+ })
+
+ it("keeps string shape when images is an empty array", () => {
+ const out = chatMessagesToLLM([msg({ role: "user", content: "hi", images: [] })])
+ expect(out[0].content).toBe("hi")
+ })
+
+ it("emits ContentBlock[] (text first, then image blocks) when images are present", () => {
+ const out = chatMessagesToLLM([
+ msg({
+ role: "user",
+ content: "what is this?",
+ images: [
+ { mediaType: "image/png", dataBase64: "AAAA" },
+ { mediaType: "image/jpeg", dataBase64: "BBBB" },
+ ],
+ }),
+ ])
+ expect(out[0].role).toBe("user")
+ expect(out[0].content).toEqual([
+ { type: "text", text: "what is this?" },
+ { type: "image", mediaType: "image/png", dataBase64: "AAAA" },
+ { type: "image", mediaType: "image/jpeg", dataBase64: "BBBB" },
+ ])
+ })
+
+ it("still includes a (possibly empty) text block for image-only messages", () => {
+ const out = chatMessagesToLLM([
+ msg({ role: "user", content: "", images: [{ mediaType: "image/webp", dataBase64: "CCCC" }] }),
+ ])
+ const blocks = out[0].content as Array<{ type: string }>
+ expect(blocks[0]).toEqual({ type: "text", text: "" })
+ expect(blocks[1]).toEqual({ type: "image", mediaType: "image/webp", dataBase64: "CCCC" })
+ })
+})
diff --git a/src/stores/chat-store.ts b/src/stores/chat-store.ts
index b1118c62..4d35efb8 100644
--- a/src/stores/chat-store.ts
+++ b/src/stores/chat-store.ts
@@ -1,7 +1,19 @@
import { create } from "zustand"
-import type { ChatMessage } from "@/lib/llm-client"
+import type { ChatMessage, ContentBlock } from "@/lib/llm-client"
import i18n from "@/i18n"
+/**
+ * An image attached to a user message. Field names mirror the
+ * `image` variant of `ContentBlock` (see llm-providers.ts) so
+ * converting a DisplayMessage into a wire ContentBlock is a no-op
+ * spread — no remapping, no `data:` framing here (the provider
+ * translators own that).
+ */
+export interface MessageImage {
+ mediaType: string
+ dataBase64: string
+}
+
export interface Conversation {
id: string
title: string
@@ -25,6 +37,7 @@ export interface DisplayMessage {
timestamp: number
conversationId: string
references?: MessageReference[] // pages cited in this response, saved at creation time
+ images?: MessageImage[] // images attached to a user message (vision input)
}
interface ChatState {
@@ -44,7 +57,7 @@ interface ChatState {
renameConversation: (id: string, title: string) => void
// Message management
- addMessage: (role: DisplayMessage["role"], content: string) => void
+ addMessage: (role: DisplayMessage["role"], content: string, images?: MessageImage[]) => void
setMessages: (messages: DisplayMessage[]) => void
setConversations: (conversations: Conversation[]) => void
setStreaming: (streaming: boolean) => void
@@ -120,7 +133,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
),
})),
- addMessage: (role, content) =>
+ addMessage: (role, content, images) =>
set((state) => {
const { activeConversationId, conversations } = state
if (!activeConversationId) return state
@@ -131,6 +144,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
content,
timestamp: Date.now(),
conversationId: activeConversationId,
+ ...(images && images.length > 0 ? { images } : {}),
}
// Auto-set title from first user message (first 50 chars)
@@ -141,7 +155,14 @@ export const useChatStore = create<ChatState>((set, get) => ({
role === "user" && convMessages.length === 0
? conversations.map((c) =>
c.id === activeConversationId
- ? { ...c, title: content.slice(0, 50), updatedAt: Date.now() }
+ ? {
+ ...c,
+ // Image-only first message has empty text; fall
+ // back to a generic title so the sidebar entry
+ // isn't blank.
+ title: content.slice(0, 50) || (images && images.length > 0 ? i18n.t("chat.imageMessage") : c.title),
+ updatedAt: Date.now(),
+ }
: c
)
: conversations.map((c) =>
@@ -233,8 +254,26 @@ export const useChatStore = create<ChatState>((set, get) => ({
}))
export function chatMessagesToLLM(messages: DisplayMessage[]): ChatMessage[] {
- return messages.map((m) => ({
- role: m.role,
- content: m.content,
- }))
+ return messages.map((m) => {
+ // No images → keep the legacy string shape. Providers and the
+ // single-string fast paths in the translators stay unchanged,
+ // and existing tests that assert `content: "..."` keep passing.
+ if (!m.images || m.images.length === 0) {
+ return { role: m.role, content: m.content }
+ }
+ // Images present → emit a ContentBlock[]. Text first (so the
+ // model reads the prompt before the images), then one image
+ // block per attachment. An empty text (image-only message)
+ // still gets a text block — harmless, and keeps the shape
+ // uniform.
+ const blocks: ContentBlock[] = [
+ { type: "text", text: m.content },
+ ...m.images.map((img): ContentBlock => ({
+ type: "image",
+ mediaType: img.mediaType,
+ dataBase64: img.dataBase64,
+ })),
+ ]
+ return { role: m.role, content: blocks }
+ })
}