🤖 Open-source, local-first alternative to Grok Bot, OpenAI dots and Claude Duet. Your own AI bots on any model: GPT, Claude, Gemini, Grok, DeepSeek, Kimi, GLM and 70+ more through one Modelflare key. Reach them in the app, WeChat or Feishu; they remember you, work in the background, run commands and use your computer. Pay per use.
Python ai-agentai-assistantchatbotclaudecomputer-usedeepseekdesktop-appdshelectronfeishugeminigptgrokllmlocal-firstmodelflaremulti-modelopenai-compatiblepersonal-ai-agentwechat
feat(weixin): receive images and files from WeChat; vision models see them
Marla0f664f34c239
diff --git a/README.md b/README.mdindex bb6b20d..01308f7 100644--- a/README.md+++ b/README.md@@ -132,6 +132,7 @@ docker exec amb cat /data/ui-token # 打开 http://<主机>:3600/?token=<令- 信息面板「渠道」→ 微信 → 「扫码登录」,用微信扫描二维码并确认;或命令行 `npm run weixin:login`(终端显示二维码,同时保存 PNG 到数据目录)。- 登录态、同步游标、context_token 保存在数据目录,重启后自动续用。- 平台限制:用户每发来一条消息后 24 小时内最多可回复 10 条;回复在生成结束后一次发出,超过 4000 字自动分段;生成期间显示“正在输入”。+- 图片、文件、视频:用户发来(或引用)的会从微信 CDN 下载解密(单个最大 50 MB)。代理模式下存进机器人工作区的 `inbox/`,消息里写明路径,代理可以读取、处理或当生图的参考图;能看图的模型(Claude、GPT-5/6、Gemini、Grok 4、Kimi K3、Muse Spark 等)同时直接收到图片本身,其余模型只拿到路径。Linux 版代理暂不直接收图(sharp 在 Electron 下会崩溃),只给路径。机器人交付的文件(present)按图片或文件发回微信。- 登录过期(错误码 -14)时自动暂停并在界面提示,重新扫码即可。## 开发与测试diff --git a/scripts/verify-dsh-bundle.mjs b/scripts/verify-dsh-bundle.mjsindex 5a3e594..b5e395a 100644--- a/scripts/verify-dsh-bundle.mjs+++ b/scripts/verify-dsh-bundle.mjs@@ -68,7 +68,9 @@ await step("DSH host answers through a mock model", async () => {if (!body.stream) return res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ choices: [{ message: { role: "assistant", content: "title" }, finish_reason: "stop" }] }));res.writeHead(200, { "content-type": "text/event-stream" });const send = (o) => res.write(`data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", model: body.model, ...o })}\n\n`);- send({ choices: [{ index: 0, delta: { role: "assistant", content: "bundle-pong" } }] });+ // a user image reaches the model as an image_url part (after DSH admitted and normalized it with sharp)+ const image = body.messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p?.type === "image_url"));+ send({ choices: [{ index: 0, delta: { role: "assistant", content: image ? "bundle-image-seen" : "bundle-pong" } }] });send({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] });send({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } });res.end("data: [DONE]\n\n");@@ -91,6 +93,7 @@ await step("DSH host answers through a mock model", async () => {" name: mock"," contextWindow: 128000"," maxTokens: 16000",+ " input: [text, image]","",].join("\n"),);@@ -106,7 +109,16 @@ await step("DSH host answers through a mock model", async () => {console.log(`DSH_HOST_INIT_MS=${started}`);const r = await withTimeout(h.session().run("ping"), 60_000, "turn");if (!String(r.finalResponse).includes("bundle-pong")) throw new Error(`unexpected reply ${JSON.stringify(r.finalResponse)}`);- return `(sdk import ${sdkMs} ms, host spawn -> initialized ${started} ms)`;+ // An image the user sent in chat (src/agent/host.ts imageInput). Not on Linux: sharp crashes under Electron+ // there, so the app never passes images to DSH on Linux.+ let image = "";+ if (process.platform !== "linux") {+ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";+ const ri = await withTimeout(h.session().run([{ type: "text", text: "what is this?" }, { type: "image", data: png, mimeType: "image/png" }]), 60_000, "image turn");+ if (!String(ri.finalResponse).includes("bundle-image-seen")) throw new Error(`image did not reach the model: ${JSON.stringify(ri.finalResponse)}`);+ image = ", image turn ok";+ }+ return `(sdk import ${sdkMs} ms, host spawn -> initialized ${started} ms${image})`;} finally {await h.close().catch(() => {});llm.close();diff --git a/src/agent/host.ts b/src/agent/host.tsindex e457c66..4584b5b 100644--- a/src/agent/host.ts+++ b/src/agent/host.ts@@ -11,6 +11,7 @@ import { adaptSkill, mcpPatchLines, resolveComputerUse, type CuResolved } from "import { cuaConfig, cuaMcpEnv, discoverAppsCached, type CuaApp } from "./cua-driver.js";import { pwshPatchLines, pwshProbe } from "./pwsh.js";import { DEFAULT_SETTINGS, permissionMode, type BotSettings } from "./settings.js";+import { acceptsImages } from "../llm/client.js";const log = logger("agent.host");@@ -191,6 +192,21 @@ export function modelLimits(model: string): { window: number; maxTokens: numberreturn { window: lim[1], maxTokens: lim[2] };}+/** 随消息交给模型的图片(DSH SDK 的内联图片块:base64 + 格式,进会话前存入 DSH 的附件库) */+export interface TurnImage {+ data: string;+ mimeType: "image/png" | "image/jpeg" | "image/gif" | "image/webp";+}++export interface TurnInput {+ conversationId: string;+ text: string;+ model: string;+ observer?: TurnObserver;+ /** 只在模型能看图时传(acceptsImages),否则模型只拿到文字里的工作区路径 */+ images?: TurnImage[];+}+/** 回合结束时的上下文用量(来自 DSH 事件)。 */export interface TurnUsage {/** 最后一次模型请求的输入+输出 token,约等于当前上下文大小 */@@ -333,6 +349,8 @@ export class AgentHost {` name: ${yamlStr(model.split("/").pop() || model)}`,` contextWindow: ${lim[1]}`,` maxTokens: ${lim[2]}`,+ // 默认只收文字;能看图的模型声明图片输入,用户发来的图片才会作为图片交给它+ ...(acceptsImages(model) ? [" input: [text, image]"] : []),// Windows:命令用检测到的 PowerShell 7(DSH 自己只看 Program Files\PowerShell\7 和 PATH,找不到就退回 5.1)...pwshPatchLines(pwshPath),"- insert:",@@ -764,7 +782,14 @@ exec "$REAL" "$@"this.o.store.clearAgentSession(conversationId);}- async runTurn(input: { conversationId: string; text: string; model: string; observer?: TurnObserver }): Promise<{ text: string; sessionId: string; files: string[]; usage: TurnUsage }> {+ /**+ * 能否把用户发来的图片交给 DSH。DSH 收图时用 sharp 规范化,而 sharp 在 Linux 上的 Electron 里会让进程崩溃+ * (SharpElectronLinux,见 scripts/verify-dsh-bundle.mjs),那里只给工作区路径。其它平台上如果带图的回合+ * 让 DSH 崩了,本次运行期间也不再给图。+ */+ imageInput = !(process.versions.electron && process.platform === "linux") && process.env.AMB_DSH_IMAGES !== "0";++ async runTurn(input: TurnInput): Promise<{ text: string; sessionId: string; files: string[]; usage: TurnUsage }> {const h = await this.ensure(input.model);const s = await this.sessionFor(h, input.conversationId);this.active++;@@ -773,9 +798,11 @@ exec "$REAL" "$@"const presented: string[] = [];const lim = modelLimits(input.model);const usage: TurnUsage = { tokens: 0, window: lim.window, maxTokens: lim.maxTokens };+ const images = this.imageInput ? (input.images ?? []) : [];try {const timeoutMs = this.o.turnTimeoutMs ?? 30 * 60_000;- const run = s.run(input.text, {+ const prompt = images.length ? [{ type: "text", text: input.text }, ...images.map((i) => ({ type: "image", data: i.data, mimeType: i.mimeType }))] : input.text;+ const run = s.run(prompt, {onNotification: (n) => {if (process.env.AMB_DSH_TRACE) fs.appendFileSync(process.env.AMB_DSH_TRACE, JSON.stringify(n).slice(0, 2000) + "\n");if (n.method !== "session.event" || n.params?.sessionId !== s.id) return;@@ -806,6 +833,10 @@ exec "$REAL" "$@"return { text: r.finalResponse, sessionId: s.id, files: deliverables(this.workspace, presented), usage };} catch (e) {if (/timed out/.test((e as Error).message)) await this.closeHarness();+ else if (images.length && this.state !== "running") {+ this.imageInput = false;+ log.warn("dsh died on a turn with images; sending images by path only until the app restarts", { bot: this.o.botId, message: (e as Error).message.slice(0, 200) });+ }throw e;} finally {clearTimeout(timer);@@ -900,7 +931,7 @@ You have no Modelflare key and must not call the API yourself (no curl, no scrip1. \`media_models\` (kind \`image\` or \`video\`) lists the models this bot may use, the default one, their parameter ranges and reference prices. Use the default unless the user named a model or the request needs something the default cannot do (a reference image, a ratio or length it does not support).2. Plain text-to-image or text-to-video with default settings: call \`generate_image\` / \`generate_video\` with a detailed \`prompt\` only.3. Anything else (ratio, resolution, size, length, quality, audio, reference images, first/last frames, editing an image): call \`media_docs\` for the chosen model, then pass \`body\` in exactly the format its page shows, without \`model\`. Every family is different (for example Seedance uses \`content[]\`, Kling \`contents\` and \`settings\`, Wan \`input\` and \`parameters\`, Gemini \`contents[].parts[]\` with \`generationConfig\`), so never reuse field names from another model's page. For OpenAI Images models whose page sends references to \`/v1/images/edits\`, also pass \`endpoint: "edits"\`. Keep \`prompt\` the same as the prompt inside \`body\`: the user sees \`prompt\` in the confirmation.-4. Reference media: a file in the workspace is written as \`"workspace:media/x.png"\` (sent as a data URL) or \`"workspace-base64:media/x.png"\` (raw Base64, e.g. Gemini \`inline_data.data\`), up to 20 MB each. A public https URL can be used as is. Models marked 必须带参考素材 (image-to-video, reference-to-video) only work with references; many models accept reference videos only as public URLs, see the docs. Images the user sent in chat are not in the workspace: if you have no file or link for a reference, ask the user for one.+4. Reference media: a file in the workspace is written as \`"workspace:media/x.png"\` (sent as a data URL) or \`"workspace-base64:media/x.png"\` (raw Base64, e.g. Gemini \`inline_data.data\`), up to 20 MB each. A public https URL can be used as is. Models marked 必须带参考素材 (image-to-video, reference-to-video) only work with references; many models accept reference videos only as public URLs, see the docs. Images and files the user sends in chat are saved in the workspace under \`inbox/\`, and the message shows where (for example \`[图片 inbox/20261031-143012-ab12cd.jpg]\`), so write such an image as \`"workspace:inbox/20261031-143012-ab12cd.jpg"\`. If you have no file or link for a reference, ask the user for one.5. The tool result says what happened:- Generated: the file is under \`media/\`; deliver it with \`present\`.- The user picked another model while you had passed a \`body\`: read that model's page with \`media_docs\`, rewrite \`body\` for it and call again with \`model\` set to it. That call is not confirmed again.diff --git a/src/agent/manager.ts b/src/agent/manager.tsindex 4bfb2f0..ca13c30 100644--- a/src/agent/manager.ts+++ b/src/agent/manager.ts@@ -3,7 +3,7 @@ import type { Store } from "../store/db.js";import { logger } from "../log.js";import { AgentBridge } from "./bridge.js";import path from "node:path";-import { AgentHost, callDetail, type TurnObserver, type TurnUsage } from "./host.js";+import { AgentHost, callDetail, type TurnInput, type TurnObserver, type TurnUsage } from "./host.js";import type { MediaModel, MediaPlan, MediaService } from "./media.js";import { APPROVAL_TTL_MS, ApprovalCenter, type QuestionItem } from "./approvals.js";import { MemoryError, MemoryStore, type Scope } from "./memory.js";@@ -166,7 +166,7 @@ export class AgentManager {/** 会话 → 正在进行的回合开始时间(/status 与故障排查用:后来的消息排在它后面) */readonly turnsInProgress = new Map<string, { botId: string; since: number }>();- async runTurn(botId: string, input: { conversationId: string; text: string; model: string; observer?: TurnObserver }) {+ async runTurn(botId: string, input: TurnInput) {const conv = input.conversationId;this.turnsInProgress.set(conv, { botId, since: Date.now() });try {@@ -176,7 +176,7 @@ export class AgentManager {}}- private async runTurnInner(botId: string, input: { conversationId: string; text: string; model: string; observer?: TurnObserver }) {+ private async runTurnInner(botId: string, input: TurnInput) {const conv = input.conversationId;const gate = this.gates.get(conv);if (gate) await withTimeout(gate, 240_000, undefined);diff --git a/src/app.ts b/src/app.tsindex 1f357ad..51d47fd 100644--- a/src/app.ts+++ b/src/app.ts@@ -505,6 +505,7 @@ export class App {if (bot.weixinEnabled && !ch.weixin) {ch.weixin = new WeixinChannel(bot.id, this.store, this.runtime, this.queue, {fetchImpl: this.opts.weixinFetch,+ cdnBaseUrl: this.config.weixin.cdnBaseUrl,pollTimeoutMs: this.opts.weixinPollTimeoutMs,});ch.weixin.onExpired = () => this.notice("warn", `机器人「${bot.name}」微信登录已过期,请在信息面板的「渠道」里重新扫码`);diff --git a/src/channels/weixin/api.ts b/src/channels/weixin/api.tsindex 5b595ec..abad87d 100644--- a/src/channels/weixin/api.ts+++ b/src/channels/weixin/api.ts@@ -26,9 +26,60 @@ export function encryptAesEcb(plain: Buffer, key: Buffer): Buffer {const c = crypto.createCipheriv("aes-128-ecb", key, null);return Buffer.concat([c.update(plain), c.final()]);}+export function decryptAesEcb(cipher: Buffer, key: Buffer): Buffer {+ const d = crypto.createDecipheriv("aes-128-ecb", key, null);+ return Buffer.concat([d.update(cipher), d.final()]);+}export function aesEcbPaddedSize(n: number): number {return Math.ceil((n + 1) / 16) * 16;}++/**+ * CDNMedia.aes_key 有两种编码(移植自 openclaw-weixin src/cdn/pic-decrypt.ts):+ * 图片是 16 字节原始密钥的 base64;文件/语音/视频是 32 位十六进制字符串的 base64。+ */+export function parseAesKey(aesKeyBase64: string): Buffer {+ const decoded = Buffer.from(aesKeyBase64, "base64");+ if (decoded.length === 16) return decoded;+ const ascii = decoded.toString("ascii");+ if (decoded.length === 32 && /^[0-9a-fA-F]{32}$/.test(ascii)) return Buffer.from(ascii, "hex");+ throw new Error(`aes_key 解码后为 ${decoded.length} 字节,应为 16 字节或 32 位十六进制`);+}++export type ImageMime = "image/png" | "image/jpeg" | "image/gif" | "image/webp";+const IMAGE_MIME_EXT: Record<ImageMime, string> = { "image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp" };++/** 按文件头识别图片格式(微信图片条目不带文件名和 MIME);不认识的返回 undefined。 */+export function sniffImageMime(buf: Buffer): ImageMime | undefined {+ if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg";+ if (buf.length >= 8 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";+ if (buf.length >= 6 && buf.subarray(0, 4).toString("ascii") === "GIF8") return "image/gif";+ if (buf.length >= 12 && buf.subarray(0, 4).toString("ascii") === "RIFF" && buf.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";+ return undefined;+}+export function imageExt(mime: string | undefined): string {+ return IMAGE_MIME_EXT[mime as ImageMime] ?? ".jpg";+}++/** 单个入站媒体(解密后)的大小上限 */+export const MAX_INBOUND_MEDIA_BYTES = 50 * 1024 * 1024;+const CDN_TIMEOUT_MS = 60_000;++export interface InboundMedia {+ kind: "image" | "file" | "video";+ data: Buffer;+ /** 文件条目的原始文件名 */+ name?: string;+ /** 图片按文件头识别出的格式 */+ mime?: ImageMime;+}++/** 图片/文件/视频条目带有可下载的 CDN 引用。语音不下载:有转写文字,原始音频是 SILK。 */+export function hasDownloadableMedia(item: MessageItem | undefined): boolean {+ const media =+ item?.type === MessageItemType.IMAGE ? item.image_item?.media : item?.type === MessageItemType.FILE ? item.file_item?.media : item?.type === MessageItemType.VIDEO ? item.video_item?.media : undefined;+ return !!(media?.encrypt_query_param || media?.full_url);+}export const SESSION_EXPIRED_ERRCODE = -14;const CHANNEL_VERSION = "0.1.0";const BOT_AGENT = "AnymodelBot/0.1.0";@@ -102,14 +153,17 @@ export interface CdnMedia {encrypt_query_param?: string;aes_key?: string;encrypt_type?: number;+ /** 完整下载地址(服务端给出时优先用它,不再自己拼) */+ full_url?: string;}export interface MessageItem {type?: number;text_item?: { text?: string };- voice_item?: { text?: string };+ voice_item?: { text?: string; media?: CdnMedia };file_item?: { file_name?: string; media?: CdnMedia; len?: string };- image_item?: { media?: CdnMedia; mid_size?: number };+ /** aeskey:入站图片的原始密钥(十六进制),优先于 media.aes_key */+ image_item?: { media?: CdnMedia; aeskey?: string; mid_size?: number };video_item?: { media?: CdnMedia; video_size?: number };ref_msg?: { title?: string; message_item?: MessageItem };}@@ -282,17 +336,20 @@ export class IlinkClient {},15_000,);- const up = parseWeixinApiJson<{ ret?: number; errmsg?: string; upload_full_url?: string; upload_param?: string }>(raw || "{}");- const cdnBase = (this.opts.cdnBaseUrl || DEFAULT_CDN_BASE).replace(/\/+$/, "");- const url = up.upload_full_url?.trim() || (up.upload_param ? `${cdnBase}/upload?encrypted_query_param=${encodeURIComponent(up.upload_param)}&filekey=${encodeURIComponent(filekey)}` : "");+ const up = parseWeixinApiJson<{ ret?: number; errcode?: number; errmsg?: string; upload_full_url?: string; upload_param?: string }>(raw || "{}");+ if ((up.ret && up.ret !== 0) || (up.errcode && up.errcode !== 0)) {+ throw new WeixinApiError(`getuploadurl ret=${up.ret} errcode=${up.errcode} errmsg=${up.errmsg ?? ""}`, up.ret, up.errcode);+ }+ const url = up.upload_full_url?.trim() || (up.upload_param ? `${this.cdnBase}/upload?encrypted_query_param=${encodeURIComponent(up.upload_param)}&filekey=${encodeURIComponent(filekey)}` : "");if (!url) throw new WeixinApiError(`getuploadurl returned no upload url (ret=${up.ret ?? ""})`, up.ret);const cipher = encryptAesEcb(plain, aeskey);let param = "";let lastErr: unknown;for (let attempt = 1; attempt <= 3 && !param; attempt++) {try {- const res = await this.fetchImpl(url, { method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: new Uint8Array(cipher) });- if (res.status >= 400 && res.status < 500) throw Object.assign(new Error(`CDN upload client error ${res.status}`), { fatal: true });+ const res = await this.fetchImpl(url, { method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: new Uint8Array(cipher), signal: AbortSignal.timeout(CDN_TIMEOUT_MS) });+ await res.arrayBuffer().catch(() => {});+ if (res.status >= 400 && res.status < 500) throw Object.assign(new Error(`CDN upload client error ${res.status} ${res.headers.get("x-error-message") ?? ""}`.trim()), { fatal: true });if (res.status !== 200) throw new Error(`CDN upload server error ${res.status}`);param = res.headers.get("x-encrypted-param") ?? "";if (!param) throw new Error("CDN upload response missing x-encrypted-param");@@ -313,6 +370,73 @@ export class IlinkClient {await this.sendItem(toUserId, item, contextToken);}+ private get cdnBase(): string {+ return (this.opts.cdnBaseUrl || DEFAULT_CDN_BASE).replace(/\/+$/, "");+ }++ /**+ * 下载并解密用户发来的图片/文件/视频(含被引用消息里的)。+ * 移植自 openclaw-weixin src/media/media-download.ts、src/cdn/pic-decrypt.ts、src/cdn/cdn-url.ts。+ */+ async downloadMedia(item: MessageItem, maxBytes = MAX_INBOUND_MEDIA_BYTES): Promise<InboundMedia> {+ let kind: InboundMedia["kind"];+ let media: CdnMedia | undefined;+ let key: Buffer | undefined;+ if (item.type === MessageItemType.IMAGE) {+ kind = "image";+ media = item.image_item?.media;+ // image_item.aeskey(十六进制)优先;两者都没有时图片未加密+ const hex = item.image_item?.aeskey;+ if (hex && /^[0-9a-fA-F]{32}$/.test(hex)) key = Buffer.from(hex, "hex");+ else if (media?.aes_key) key = parseAesKey(media.aes_key);+ } else if (item.type === MessageItemType.FILE || item.type === MessageItemType.VIDEO) {+ kind = item.type === MessageItemType.FILE ? "file" : "video";+ media = kind === "file" ? item.file_item?.media : item.video_item?.media;+ if (!media?.aes_key) throw new Error("缺少 aes_key");+ key = parseAesKey(media.aes_key);+ } else {+ throw new Error(`不支持下载的条目类型 ${item.type}`);+ }+ const full = media?.full_url?.trim();+ if (full && !/^https?:\/\//i.test(full)) throw new Error("full_url 不是 http(s) 地址");+ if (!full && !media?.encrypt_query_param) throw new Error("没有 CDN 引用");+ const url = full || `${this.cdnBase}/download?encrypted_query_param=${encodeURIComponent(media!.encrypt_query_param!)}`;++ let raw: Buffer | undefined;+ let lastErr: unknown;+ for (let attempt = 1; attempt <= 3 && !raw; attempt++) {+ try {+ const res = await this.fetchImpl(url, { signal: AbortSignal.timeout(CDN_TIMEOUT_MS) });+ if (!res.ok) {+ await res.arrayBuffer().catch(() => {});+ const err = new Error(`CDN 下载失败 HTTP ${res.status} ${res.headers.get("x-error-message") ?? ""}`.trim());+ throw res.status < 500 ? Object.assign(err, { fatal: true }) : err;+ }+ const declared = Number(res.headers.get("content-length") ?? 0);+ if (declared > maxBytes) {+ await res.body?.cancel().catch(() => {});+ throw Object.assign(new Error(`文件过大(${(declared / 1048576).toFixed(1)} MB,上限 ${maxBytes / 1048576} MB)`), { fatal: true });+ }+ const buf = Buffer.from(await res.arrayBuffer());+ if (buf.length > maxBytes) throw Object.assign(new Error(`文件过大(${(buf.length / 1048576).toFixed(1)} MB,上限 ${maxBytes / 1048576} MB)`), { fatal: true });+ raw = buf;+ } catch (e) {+ lastErr = e;+ if ((e as { fatal?: boolean }).fatal) throw e;+ }+ }+ if (!raw) throw lastErr instanceof Error ? lastErr : new Error("CDN 下载失败");+ let data: Buffer;+ try {+ data = key ? decryptAesEcb(raw, key) : raw;+ } catch {+ throw new Error("解密失败(密钥不对?)");+ }+ if (kind === "image") return { kind, data, mime: sniffImageMime(data) };+ if (kind === "file") return { kind, data, name: item.file_item?.file_name || "file" };+ return { kind, data };+ }+private async sendItem(toUserId: string, item: MessageItem, contextToken: string): Promise<void> {const raw = await this.request("POST",diff --git a/src/channels/weixin/channel.ts b/src/channels/weixin/channel.tsindex e958c76..34100a7 100644--- a/src/channels/weixin/channel.ts+++ b/src/channels/weixin/channel.ts@@ -3,7 +3,7 @@ import { splitText } from "../../core/types.js";import { Dedup, KeyedQueue } from "../../core/queue.js";import type { BotRuntime } from "../../core/runtime.js";import type { Store } from "../../store/db.js";-import { IlinkClient, MessageItemType, MessageType, WeixinApiError, type MessageItem, type WeixinMessage } from "./api.js";+import { IlinkClient, MessageItemType, MessageType, WeixinApiError, hasDownloadableMedia, type InboundMedia, type MessageItem, type WeixinMessage } from "./api.js";import { StreamingMarkdownFilter } from "./markdown-filter.js";import { WeixinQuota } from "./quota.js";import { logger } from "../../log.js";@@ -25,14 +25,30 @@ export function filterMarkdown(text: string): string {return (f.feed(text) + f.flush()).trim();}-export function weixinParts(items: MessageItem[] | undefined): Part[] {+/** 媒体下载结果,按条目对象对应(含 ref_msg 里被引用的条目) */+export type WeixinMediaResults = Map<MessageItem, InboundMedia | Error>;++function mediaPart(it: MessageItem, got: InboundMedia | Error | undefined, quoted: boolean): Part {+ const type = it.type === MessageItemType.IMAGE ? "image" : it.type === MessageItemType.FILE ? "file" : "video";+ const part: Part = { type, ...(type === "file" ? { name: it.file_item?.file_name } : {}), ...(quoted ? { quoted: true } : {}) };+ if (got instanceof Error) return { ...part, error: got.message.slice(0, 120) };+ if (got) return { ...part, data: got.data, ...(got.mime ? { mime: got.mime } : {}) };+ return part;+}++export function weixinParts(items: MessageItem[] | undefined, media?: WeixinMediaResults): Part[] {const parts: Part[] = [];for (const it of items ?? []) {switch (it.type) {case MessageItemType.TEXT: {const t = it.text_item?.text ?? "";- const ref = it.ref_msg?.message_item?.text_item?.text || it.ref_msg?.title;- if (ref) parts.push({ type: "quote", text: ref });+ const quoted = it.ref_msg?.message_item;+ // 引用了一张图片/一个文件:把它本身交给运行时,而不是只给标题+ if (quoted && hasDownloadableMedia(quoted)) parts.push(mediaPart(quoted, media?.get(quoted), true));+ else {+ const ref = quoted?.text_item?.text || it.ref_msg?.title;+ if (ref) parts.push({ type: "quote", text: ref });+ }if (t) parts.push({ type: "text", text: t });break;}@@ -40,19 +56,20 @@ export function weixinParts(items: MessageItem[] | undefined): Part[] {parts.push({ type: "voice", transcript: it.voice_item?.text });break;case MessageItemType.IMAGE:- parts.push({ type: "image" });- break;case MessageItemType.FILE:- parts.push({ type: "file", name: it.file_item?.file_name });- break;case MessageItemType.VIDEO:- parts.push({ type: "video" });+ parts.push(mediaPart(it, media?.get(it), false));break;}}return parts;}+/** 消息里需要下载的条目(含被引用的) */+function mediaItems(items: MessageItem[] | undefined): MessageItem[] {+ return (items ?? []).flatMap((it) => [it, ...(it.ref_msg?.message_item ? [it.ref_msg.message_item] : [])]).filter(hasDownloadableMedia);+}+/*** 微信回复会话:不流式;生成期间只显示“正在输入”(每 5 秒续期),结束时一次性发送最终回复(>4000 字切分),遵守配额。* 不发送工具调用进度;只有最终回复、审批/提问提示和交付文件会成为聊天消息,配额全部留给它们。@@ -186,7 +203,7 @@ export class WeixinChannel {private store: Store,private runtime: BotRuntime,private queue: KeyedQueue,- private opts: { fetchImpl?: typeof fetch; pollTimeoutMs?: number; retryDelayMs?: number; backoffMs?: number } = {},+ private opts: { fetchImpl?: typeof fetch; cdnBaseUrl?: string; pollTimeoutMs?: number; retryDelayMs?: number; backoffMs?: number } = {},) {this.quota = new WeixinQuota(store, botId);}@@ -202,7 +219,7 @@ export class WeixinChannel {this.status = { state: "login_required", detail: "尚未扫码登录", since: Date.now() };return;}- this.client = new IlinkClient({ baseUrl: acct.baseUrl, token: acct.botToken, fetchImpl: this.opts.fetchImpl });+ this.client = new IlinkClient({ baseUrl: acct.baseUrl, token: acct.botToken, fetchImpl: this.opts.fetchImpl, cdnBaseUrl: this.opts.cdnBaseUrl });this.abort = new AbortController();this.loop = this.run(this.abort.signal).finally(() => (this.loop = undefined));}@@ -302,9 +319,11 @@ export class WeixinChannel {}return;}+ const media = mediaItems(m.item_list);const p = this.queue.run(`weixin:${this.botId}:${userId}`, async () => {const reply = new WeixinReply(this.client!, userId, this.store.getWeixinPeer(this.botId, userId)?.contextToken ?? peer.contextToken, this.quota);try {+ if (media.length) ev.parts = weixinParts(m.item_list, await this.download(media));await this.runtime.handle(ev, reply);} catch (e) {this.log.error("handle failed", { message: (e as Error).message });@@ -315,6 +334,20 @@ export class WeixinChannel {void p.finally(() => this.pending.delete(p));}+ /** 逐个下载解密;失败的记下原因(消息照常处理,文字里会说明没下载成功) */+ private async download(items: MessageItem[]): Promise<WeixinMediaResults> {+ const out: WeixinMediaResults = new Map();+ for (const it of items) {+ try {+ out.set(it, await this.client!.downloadMedia(it));+ } catch (e) {+ this.log.warn("weixin media download failed", { type: it.type, message: (e as Error).message.slice(0, 200) });+ out.set(it, e as Error);+ }+ }+ return out;+ }+async idle(): Promise<void> {while (this.pending.size) await Promise.all([...this.pending]);}diff --git a/src/config.ts b/src/config.tsindex 958d3ef..69ce8ac 100644--- a/src/config.ts+++ b/src/config.ts@@ -35,7 +35,8 @@ export interface AppConfig {/** 默认的图片 / 视频模型(生成前确认时默认选中);空 = 按推荐顺序的第一个 */media: { imageModel: string; videoModel: string };feishu: { throttleMs: number };- weixin: { apiBaseUrl: string };+ /** cdnBaseUrl:收发图片/文件的微信 CDN(测试时指向假服务) */+ weixin: { apiBaseUrl: string; cdnBaseUrl: string };/** 代理宿主:dsh = DeepSeek Harness(工具/沙箱/审批);off = 纯对话 */agent: { mode: "dsh" | "off"; turnTimeoutMs: number; approvalTtlMs: number };}@@ -94,7 +95,10 @@ export function loadConfig(overrides: { dataDir?: string; env?: NodeJS.ProcessEnvideoModel: typeof file.media?.videoModel === "string" ? file.media.videoModel : "",},feishu: { throttleMs: num(env.AMB_FEISHU_THROTTLE_MS, file.feishu?.throttleMs ?? 400) },- weixin: { apiBaseUrl: env.AMB_WEIXIN_API_BASE || file.weixin?.apiBaseUrl || "https://ilinkai.weixin.qq.com" },+ weixin: {+ apiBaseUrl: env.AMB_WEIXIN_API_BASE || file.weixin?.apiBaseUrl || "https://ilinkai.weixin.qq.com",+ cdnBaseUrl: env.AMB_WEIXIN_CDN_BASE || file.weixin?.cdnBaseUrl || "https://novac2c.cdn.weixin.qq.com/c2c",+ },agent: {mode: (env.AMB_AGENT || file.agent?.mode || "dsh") === "off" ? "off" : "dsh",turnTimeoutMs: num(env.AMB_AGENT_TURN_TIMEOUT_MS, file.agent?.turnTimeoutMs ?? 30 * 60_000),diff --git a/src/core/runtime.ts b/src/core/runtime.tsindex fbab5d5..6ca9f1a 100644--- a/src/core/runtime.ts+++ b/src/core/runtime.ts@@ -1,7 +1,10 @@+import crypto from "node:crypto";+import fs from "node:fs";import path from "node:path";import type { InboundEvent, ReplySession } from "./types.js";import { eventText } from "./types.js";-import type { ModelClient, ChatMessage } from "../llm/client.js";+import { acceptsImages, type ModelClient, type ChatMessage } from "../llm/client.js";+import type { TurnImage } from "../agent/host.js";import type { Store } from "../store/db.js";import { logger } from "../log.js";import type { AgentManager } from "../agent/manager.js";@@ -13,6 +16,48 @@ import type { ApprovalRow } from "../store/db.js";const log = logger("runtime");+const IMAGE_EXT: Record<TurnImage["mimeType"], string> = { "image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp" };+const MAX_INLINE_IMAGES = 4;+const MAX_INLINE_IMAGE_BYTES = 10 * 1024 * 1024;++/** 用户这条消息里能直接交给模型看的图片(格式已识别、不太大) */+export function inboundImages(ev: InboundEvent): TurnImage[] {+ const out: TurnImage[] = [];+ for (const p of ev.parts) {+ if (p.type !== "image" || !p.data || !(p.mime && p.mime in IMAGE_EXT) || p.data.length > MAX_INLINE_IMAGE_BYTES) continue;+ if (out.length < MAX_INLINE_IMAGES) out.push({ data: p.data.toString("base64"), mimeType: p.mime as TurnImage["mimeType"] });+ }+ return out;+}++/** 发送者给的文件名只留最后一段,去掉控制字符和开头的点 */+function safeFileName(name: string | undefined): string {+ const base = path.basename((name ?? "").replace(/\\/g, "/")).replace(/[\x00-\x1f\x7f]/g, "").replace(/^\.+/, "").trim();+ return (base || "file").slice(-80);+}++/**+ * 把用户发来的图片/文件/视频存进机器人工作区的 inbox/,消息文字里换成相对路径:+ * 代理可以读取、处理,或当生图的参考素材(workspace:inbox/...)。+ */+export function storeInbound(workspace: string, ev: InboundEvent, now = new Date()): void {+ const stamp = now.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);+ for (const p of ev.parts) {+ if ((p.type !== "image" && p.type !== "file" && p.type !== "video") || !p.data || p.path) continue;+ const id = `${stamp}-${crypto.randomBytes(3).toString("hex")}`;+ const name =+ p.type === "image" ? `${id}${IMAGE_EXT[p.mime as TurnImage["mimeType"]] ?? ".jpg"}` : p.type === "video" ? `${id}.mp4` : `${id}-${safeFileName(p.name)}`;+ try {+ fs.mkdirSync(path.join(workspace, "inbox"), { recursive: true });+ fs.writeFileSync(path.join(workspace, "inbox", name), p.data);+ p.path = `inbox/${name}`;+ } catch (e) {+ log.warn("save inbound media failed", { type: p.type, message: (e as Error).message.slice(0, 200) });+ p.error = "保存到工作区失败";+ }+ }+}+export interface RuntimeOptions {store: Store;models: ModelClient;@@ -65,8 +110,12 @@ export class BotRuntime {const bot = store.getBot(ev.botId);if (!bot) throw new Error(`unknown bot ${ev.botId}`);const conv = store.ensureConversation(bot.id, ev.channel, ev.conversation.id, ev.sender.name || ev.sender.id, ev.conversation.kind === "group" ? "group" : "dm");+ const agent = this.opts.agent;+ // 代理模式:收到的图片/文件先进工作区,消息里写路径+ if (agent?.available) storeInbound(agent.host(bot.id).workspace, ev);const text = eventText(ev);if (!text) return;+ const images = inboundImages(ev);if (text.startsWith("/")) {const reply = await this.command(text, conv.id, conv.model, bot.defaultModel, bot.id);@@ -82,11 +131,12 @@ export class BotRuntime {});if (!model) return;const routed = this.opts.route ? this.opts.route(bot.id, model) : model;+ // 只有能看图的模型才收到图片本身;其余模型只看到文字里的“[图片 …]”+ const seen = acceptsImages(routed) ? images : [];- const agent = this.opts.agent;let note = "";if (agent?.available) {- const handled = await this.handleAgent(agent, bot.id, conv.id, text, routed, bot.systemPrompt, out);+ const handled = await this.handleAgent(agent, bot.id, conv.id, text, routed, bot.systemPrompt, out, seen);if (handled === true) return;// 代理进程还在启动(第一次可能要几分钟):这条消息先由模型直接回答,不让用户干等if (handled === "starting") note = "(代理还在启动,这条消息先由模型直接回答;启动完成后会自动用回代理。)";@@ -97,6 +147,10 @@ export class BotRuntime {const messages: ChatMessage[] = [];if (bot.systemPrompt.trim()) messages.push({ role: "system", content: bot.systemPrompt });for (const m of history) if (m.role !== "system") messages.push({ role: m.role, content: m.content });+ const last = messages[messages.length - 1];+ if (seen.length && last?.role === "user" && typeof last.content === "string") {+ last.content = [{ type: "text", text: last.content }, ...seen.map((i) => ({ type: "image_url" as const, image_url: { url: `data:${i.mimeType};base64,${i.data}` } }))];+ }await out.apply({ type: "typing", on: true });await out.apply({ type: "stream.start" });@@ -176,7 +230,7 @@ export class BotRuntime {* 代理回合;返回 false 表示宿主不可用且尚无任何输出(调用方回退到纯对话),* "starting" 表示代理进程在等待时限内没有启动好(同样回退,启动在后台继续)。*/- private async handleAgent(agent: AgentManager, botId: string, convId: string, text: string, model: string, systemPrompt: string, out: ReplySession): Promise<boolean | "starting"> {+ private async handleAgent(agent: AgentManager, botId: string, convId: string, text: string, model: string, systemPrompt: string, out: ReplySession, images: TurnImage[] = []): Promise<boolean | "starting"> {const { store } = this.opts;let visible = false;const unbind = agent.approvals.bindSink(convId, async (row) => {@@ -223,6 +277,7 @@ export class BotRuntime {conversationId: convId,text: input,model,+ ...(images.length ? { images } : {}),observer: {// 工具调用不进入聊天:只用于判断代理已开始工作(不再回退到纯对话),正在输入状态由渠道维持onProgress: () => {diff --git a/src/core/types.ts b/src/core/types.tsindex 5761cc6..280324d 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -6,7 +6,21 @@ export type ChannelId = "feishu" | "weixin" | "web";export type Part =| { type: "text"; text: string }- | { type: "image" | "file" | "video"; ref?: string; name?: string }+ | {+ type: "image" | "file" | "video";+ ref?: string;+ name?: string;+ /** 渠道下载到的内容(运行时存进工作区后填 path) */+ data?: Buffer;+ /** 图片格式(按文件头识别) */+ mime?: string;+ /** 存进机器人工作区后的相对路径,如 inbox/20261031-143012-ab12cd.jpg */+ path?: string;+ /** 下载失败的原因 */+ error?: string;+ /** 来自被引用的消息 */+ quoted?: boolean;+ }| { type: "voice"; ref?: string; transcript?: string }| { type: "quote"; text: string };@@ -69,6 +83,8 @@ export type ChannelStatus = {since: number;};+const MEDIA_LABEL = { image: "图片", file: "文件", video: "视频" } as const;+export function eventText(ev: InboundEvent): string {return ev.parts.map((p) => {@@ -80,11 +96,13 @@ export function eventText(ev: InboundEvent): string {case "voice":return p.transcript ? p.transcript : "[语音]";case "image":- return "[图片]";case "file":- return `[文件${p.name ? " " + p.name : ""}]`;- case "video":- return "[视频]";+ case "video": {+ // [图片 inbox/x.jpg]、[引用的文件 报告.pdf → inbox/x-报告.pdf]、[视频(未能下载:…)]+ const label = `${p.quoted ? "引用的" : ""}${MEDIA_LABEL[p.type]}${p.type === "file" && p.name ? " " + p.name : ""}`;+ if (p.path) return `[${label}${p.type === "file" && p.name ? " →" : ""} ${p.path}]`;+ return p.error ? `[${label}(未能下载:${p.error})]` : `[${label}]`;+ }}}).join("\n")diff --git a/src/llm/client.ts b/src/llm/client.tsindex a3075e3..08dc0ff 100644--- a/src/llm/client.ts+++ b/src/llm/client.ts@@ -2,9 +2,21 @@* 模型访问接口。默认实现指向受管 magpie 的 OpenAI 兼容 /v1(流式 SSE)。* 运行时只依赖 ModelClient,可替换为任意 OpenAI 兼容端点。*/+/** OpenAI 兼容的多模态内容;只有用户刚发来的图片会用到(data URL) */+export type ChatContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };+export interface ChatMessage {role: "system" | "user" | "assistant";- content: string;+ content: string | ChatContentPart[];+}++/**+ * 能看图的模型。Modelflare 的模型目录不带输入模态,只能按名字判断;不在其中的模型+ * (DeepSeek、GLM 文本版等)收到图片时只拿到它在工作区里的路径。+ */+const IMAGE_INPUT_MODELS = /claude|gpt-4o|gpt-4\.1|gpt-[5-9]|gemini|grok-4|kimi-k3|muse-spark|qvq|-vl\b|-omni\b|glm-[\d.]+v\b/i;+export function acceptsImages(model: string): boolean {+ return IMAGE_INPUT_MODELS.test(model);}export interface ChatRequest {diff --git a/test/agent.integration.test.ts b/test/agent.integration.test.tsindex 892d8d4..8d25f26 100644--- a/test/agent.integration.test.ts+++ b/test/agent.integration.test.ts@@ -49,10 +49,16 @@ describe.skipIf(!runtime || !hasBwrap || process.platform !== "linux")("agent hofunction plan(messages: any[]): { tool?: { name: string; args: any }; text?: string } {let lastUser = -1;for (let i = messages.length - 1; i >= 0; i--) {- if (messages[i].role === "user" && /QUICKSORT|ESCALATE|SECRET|READFILE|LOOPBACK|PRESENT|ASKQ|NATIVES|\[用户消息\]/.test(textOf(messages[i].content))) { lastUser = i; break; }+ if (messages[i].role === "user" && /QUICKSORT|ESCALATE|SECRET|READFILE|LOOPBACK|PRESENT|ASKQ|NATIVES|LOOKIMG|\[用户消息\]/.test(textOf(messages[i].content))) { lastUser = i; break; }}const user = textOf(messages[lastUser]?.content);const tools = messages.slice(lastUser + 1).filter((m) => m.role === "tool");+ // 看图:回报模型收到的图片(data URL 的头)和文字里有没有工作区路径+ if (/LOOKIMG/.test(user)) {+ const content = messages[lastUser]?.content;+ const urls = (Array.isArray(content) ? content : []).filter((p: any) => p?.type === "image_url").map((p: any) => String(p.image_url?.url ?? "").split(",")[0]);+ return { text: `SEEN<<${urls.join(" ")}|${/\[图片 inbox\/[^\]]+\.png\]/.test(user) ? "path" : "nopath"}>>` };+ }const scripts: Array<[RegExp, Array<{ name: string; args: any }>]> = [[/QUICKSORT/, [{ name: "bash", args: { description: "run", command: "cat > qs.py <<'PY'\ndef qs(a):\n return a if len(a)<2 else qs([x for x in a[1:] if x<a[0]])+[a[0]]+qs([x for x in a[1:] if x>=a[0]])\nprint(qs([5,3,9,1,7,2]))\nPY\npython3 qs.py" } }]],[/ESCALATE/, [{ name: "bash", args: { description: "run", command: "echo escalated-$((40+2))", sandbox_permissions: "danger-full-access", justification: "测试提权" } }]],@@ -244,4 +250,24 @@ describe.skipIf(!runtime || !hasBwrap || process.platform !== "linux")("agent hoexpect(again.reply).not.toContain("⚠️");expect(app.agent.host("default").state).toBe("running");}, 180_000);++ it("images from chat: saved under the workspace inbox/; a vision model gets the image itself, a text model only the path", async () => {+ // 1x1 PNG(DSH 收图时会校验并规范化,必须是真图)+ const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", "base64");+ const send = async (user: string, model: string) => {+ const conv = app.store.ensureConversation("default", "weixin", user, user, "dm");+ app.store.setConversationModel(conv.id, model);+ const ops: any[] = [];+ const ev = { id: `img-${user}`, channel: "weixin" as const, botId: "default", conversation: { kind: "dm" as const, id: user }, sender: { id: user }, parts: [{ type: "text" as const, text: "LOOKIMG" }, { type: "image" as const, data: png, mime: "image/png" }], replyHandle: {}, receivedAt: Date.now() };+ await app.runtime.handle(ev, { apply: async (op) => void ops.push(op) });+ return ops.find((o) => o.type === "stream.finish")?.text as string;+ };+ // Electron on Linux (scripts/test-dsh-bundle.mjs): sharp crashes there, so the host keeps images to paths+ if (app.agent.host("default").imageInput) expect(await send("img-vision", "gpt-5-mock")).toMatch(/^SEEN<<data:image\/(png|jpeg|webp);base64\|path>>$/);+ else expect(await send("img-vision", "gpt-5-mock")).toBe("SEEN<<|path>>");+ expect(await send("img-text", "mock-agent")).toBe("SEEN<<|path>>");+ const inbox = fs.readdirSync(path.join(dataDir, "workspaces", "default", "inbox"));+ expect(inbox.filter((f) => /^\d{8}-\d{6}-[0-9a-f]{6}\.png$/.test(f))).toHaveLength(2);+ expect(fs.readFileSync(path.join(dataDir, "workspaces", "default", "inbox", inbox[0])).equals(png)).toBe(true);+ }, 180_000);});diff --git a/test/e2e.test.ts b/test/e2e.test.tsindex 9f98144..b991a78 100644--- a/test/e2e.test.ts+++ b/test/e2e.test.ts@@ -2,6 +2,7 @@* 离线端到端:mock OpenAI(流式)+ mock iLink + 假飞书传输层 + 真实 App/Web 服务(外部模型端点模式)。*/import { describe, it, expect, beforeAll, afterAll } from "vitest";+import crypto from "node:crypto";import fs from "node:fs";import os from "node:os";import path from "node:path";@@ -31,7 +32,7 @@ beforeAll(async () => {dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "amb-e2e-"));const config = loadConfig({dataDir,- env: { AMB_MAGPIE_BASE_URL: llm.url + "/v1", AMB_MAGPIE_API_KEY: "gw-key", AMB_UI_PORT: "0", AMB_WEIXIN_API_BASE: ilink.url, AMB_FEISHU_THROTTLE_MS: "30", AMB_AGENT: "off" },+ env: { AMB_MAGPIE_BASE_URL: llm.url + "/v1", AMB_MAGPIE_API_KEY: "gw-key", AMB_UI_PORT: "0", AMB_WEIXIN_API_BASE: ilink.url, AMB_WEIXIN_CDN_BASE: ilink.cdnBase, AMB_FEISHU_THROTTLE_MS: "30", AMB_AGENT: "off" },});process.env.FEISHU_APP_ID = "cli_test";process.env.FEISHU_APP_SECRET = "secret_test";@@ -120,6 +121,47 @@ describe("微信 E2E", () => {expect(st.json.weixin.quota[0]).toMatchObject({ userId: "[email protected]", remaining: 9 });expect(app.store.getWeixinAccount("default")!.cursor).toMatch(/^cur-/);});+ const png = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), crypto.randomBytes(64)]);+ it("收图片/文件:CDN 下载解密;能看图的模型收到图片本身,其余模型只收到标注;下载失败照常回复", async () => {+ const conv = app.store.ensureConversation("default", "weixin", "[email protected]", "[email protected]", "dm");+ app.store.setConversationModel(conv.id, "gpt-5.4");+ let prev = llm.calls.length;+ const img = ilink.imageItem(png);+ ilink.pushItems("[email protected]", [{ type: 1, text_item: { text: "这是什么?" } }, img]);+ await until(() => llm.calls.length > prev, 8000);+ expect(ilink.state.cdnDownloads).toContain(img.image_item.media.encrypt_query_param);+ let call = llm.calls.at(-1)!;+ expect(call.model).toBe("gpt-5.4");+ expect(call.messages.at(-1).content).toEqual([+ { type: "text", text: "这是什么?\n[图片]" },+ { type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } },+ ]);+ await until(() => ilink.state.sent.some((s) => s.to === "[email protected]"), 8000);++ // 纯文字模型(mock-a):只有标注,不发图片+ prev = llm.calls.length;+ ilink.pushItems("[email protected]", [ilink.imageItem(png), ilink.fileItem(Buffer.from("hi"), "notes.txt")]);+ await until(() => llm.calls.length > prev, 8000);+ expect(llm.calls.at(-1)!.messages.at(-1).content).toBe("[图片]\n[文件 notes.txt]");++ prev = llm.calls.length;+ ilink.pushItems("[email protected]", [{ type: 2, image_item: { media: { encrypt_query_param: "gone" } } }]);+ await until(() => llm.calls.length > prev, 8000);+ expect(llm.calls.at(-1)!.messages.at(-1).content).toBe("[图片(未能下载:CDN 下载失败 HTTP 400 invalid encrypted_param)]");+ await app.channels.get("default")!.weixin!.idle();+ });+ it("发图片:getuploadurl → 加密上传 CDN → 图片消息;收件方按消息里的引用和密钥能取回原图", async () => {+ const f = path.join(dataDir, "out.png");+ fs.writeFileSync(f, png);+ const client = (app.channels.get("default")!.weixin as any).client;+ const before = ilink.state.sent.length;+ await client.sendMedia("[email protected]", f, "ctx-1");+ const s = ilink.state.sent.at(-1)!;+ expect(ilink.state.sent.length).toBe(before + 1);+ expect(s).toMatchObject({ to: "[email protected]", contextToken: "ctx-1", item: { type: 2, image_item: { mid_size: 80, media: { encrypt_type: 1 } } } });+ const got = await client.downloadMedia(s.item);+ expect(got.data.equals(png)).toBe(true);+ });it("超长回复按 4000 切分并受配额约束", async () => {const big = "段落".repeat(2600) + "\n\n" + "尾".repeat(3000); // ~8200 字const prev = ilink.state.sent.length;diff --git a/test/helpers.ts b/test/helpers.tsindex 6300baf..40b5607 100644--- a/test/helpers.ts+++ b/test/helpers.ts@@ -110,7 +110,10 @@ export async function mockIlink() {const state = {statusSeq: ["wait", "scaned", "confirmed"] as string[],inbox: [] as any[],- sent: [] as Array<{ to: string; text: string; contextToken?: string }>,+ sent: [] as Array<{ to: string; text: string; contextToken?: string; item?: any }>,+ /** 假 CDN:encrypt_query_param → 密文(下载);上传的密文也存这里 */+ cdn: new Map<string, Buffer>(),+ cdnDownloads: [] as string[],typing: [] as number[],expireNext: false,authSeen: [] as string[],@@ -121,6 +124,19 @@ export async function mockIlink() {const u = new URL(req.url!, "http://x");const json = (o: unknown) => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(o));if (req.headers.authorization) state.authSeen.push(String(req.headers.authorization));+ if (u.pathname === "/c2c/download") {+ const q = u.searchParams.get("encrypted_query_param") ?? "";+ state.cdnDownloads.push(q);+ const b = state.cdn.get(q);+ return b ? res.writeHead(200, { "content-type": "application/octet-stream" }).end(b) : res.writeHead(400, { "x-error-message": "invalid encrypted_param" }).end();+ }+ if (u.pathname === "/c2c/upload") {+ const chunks: Buffer[] = [];+ for await (const c of req) chunks.push(c as Buffer);+ const id = `down-${state.cdn.size + 1}`;+ state.cdn.set(id, Buffer.concat(chunks));+ return res.writeHead(200, { "x-encrypted-param": id }).end();+ }switch (u.pathname) {case "/ilink/bot/get_bot_qrcode":return json({ qrcode: "qr-1", qrcode_img_content: "https://liteapp.weixin.qq.com/q/mock?qrcode=qr-1" });@@ -142,9 +158,14 @@ export async function mockIlink() {}case "/ilink/bot/sendmessage": {const b = await body(req);- state.sent.push({ to: b.msg.to_user_id, text: b.msg.item_list[0].text_item.text, contextToken: b.msg.context_token });+ const item = b.msg.item_list[0];+ state.sent.push({ to: b.msg.to_user_id, text: item.text_item?.text ?? "", contextToken: b.msg.context_token, ...(item.text_item ? {} : { item }) });return json({ ret: 0 });}+ case "/ilink/bot/getuploadurl": {+ const b = await body(req);+ return json({ ret: 0, upload_param: `up-${b.filekey}` });+ }case "/ilink/bot/getconfig":await body(req);return json({ ret: 0, typing_ticket: "ticket-1" });@@ -158,9 +179,26 @@ export async function mockIlink() {});base = srv.url;let seq = 0;- const push = (from: string, text: string, ctx = "ctx-1") =>- state.inbox.push({ seq: ++seq, message_id: `m${seq}`, from_user_id: from, to_user_id: "[email protected]", create_time_ms: Date.now(), message_type: 1, message_state: 2, context_token: ctx, item_list: [{ type: 1, text_item: { text } }] });- return { ...srv, state, push };+ const pushItems = (from: string, items: any[], ctx = "ctx-1") =>+ state.inbox.push({ seq: ++seq, message_id: `m${seq}`, from_user_id: from, to_user_id: "[email protected]", create_time_ms: Date.now(), message_type: 1, message_state: 2, context_token: ctx, item_list: items });+ const push = (from: string, text: string, ctx = "ctx-1") => pushItems(from, [{ type: 1, text_item: { text } }], ctx);+ /** 像微信那样把明文用随机密钥加密放到假 CDN,返回图片条目(密钥放在 image_item.aeskey) */+ const imageItem = (plain: Buffer) => {+ const key = crypto.randomBytes(16);+ const c = crypto.createCipheriv("aes-128-ecb", key, null);+ const q = `img+${crypto.randomBytes(4).toString("hex")}/=`;+ state.cdn.set(q, Buffer.concat([c.update(plain), c.final()]));+ return { type: 2, image_item: { aeskey: key.toString("hex"), media: { encrypt_query_param: q, aes_key: key.toString("base64"), encrypt_type: 1 } } };+ };+ /** 文件条目:aes_key 是十六进制密钥字符串的 base64(与微信一致) */+ const fileItem = (plain: Buffer, name: string) => {+ const key = crypto.randomBytes(16);+ const c = crypto.createCipheriv("aes-128-ecb", key, null);+ const q = `file+${crypto.randomBytes(4).toString("hex")}/=`;+ state.cdn.set(q, Buffer.concat([c.update(plain), c.final()]));+ return { type: 4, file_item: { file_name: name, len: String(plain.length), media: { encrypt_query_param: q, aes_key: Buffer.from(key.toString("hex")).toString("base64"), encrypt_type: 1 } } };+ };+ return { ...srv, state, push, pushItems, imageItem, fileItem, cdnBase: `${srv.url}/c2c` };}/** 假飞书传输层:记录全部调用 */diff --git a/test/weixin-media.unit.test.ts b/test/weixin-media.unit.test.tsnew file mode 100644index 0000000..cd59aa6--- /dev/null+++ b/test/weixin-media.unit.test.ts@@ -0,0 +1,163 @@+/** 微信入站媒体:iLink CDN 下载解密、条目 → Part、存进工作区、交给模型的图片。 */+import { describe, it, expect } from "vitest";+import crypto from "node:crypto";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import { IlinkClient, hasDownloadableMedia, parseAesKey, sniffImageMime, type MessageItem } from "../src/channels/weixin/api.js";+import { weixinParts } from "../src/channels/weixin/channel.js";+import { eventText, type InboundEvent, type Part } from "../src/core/types.js";+import { inboundImages, storeInbound } from "../src/core/runtime.js";+import { acceptsImages } from "../src/llm/client.js";++const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), crypto.randomBytes(200)]);+const JPG = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), crypto.randomBytes(300)]);+const enc = (b: Buffer, k: Buffer) => {+ const c = crypto.createCipheriv("aes-128-ecb", k, null);+ return Buffer.concat([c.update(b), c.final()]);+};++/** 假 CDN:/c2c/download?encrypted_query_param=… 与 full_url 两种取法 */+function cdn(blobs: Record<string, Buffer | number>) {+ const urls: string[] = [];+ const fetchImpl = (async (url: string) => {+ urls.push(url);+ const u = new URL(url);+ const id = u.pathname === "/c2c/download" ? u.searchParams.get("encrypted_query_param")! : u.pathname.slice(1);+ const b = blobs[id];+ if (typeof b === "number") return new Response("", { status: b, headers: { "x-error-message": "boom" } });+ return b ? new Response(new Uint8Array(b)) : new Response("", { status: 400, headers: { "x-error-message": "invalid encrypted_param" } });+ }) as any;+ return { client: new IlinkClient({ baseUrl: "http://ilink", token: "t", fetchImpl, cdnBaseUrl: "http://cdn/c2c" }), urls };+}++describe("IlinkClient.downloadMedia", () => {+ const key = crypto.randomBytes(16);+ const hexKeyB64 = Buffer.from(key.toString("hex")).toString("base64");++ it("image: image_item.aeskey (hex) wins; query param is URL-encoded; format sniffed", async () => {+ const { client, urls } = cdn({ "a+b/=": enc(PNG, key) });+ const got = await client.downloadMedia({ type: 2, image_item: { aeskey: key.toString("hex"), media: { encrypt_query_param: "a+b/=", aes_key: "ignored" } } });+ expect(got).toMatchObject({ kind: "image", mime: "image/png" });+ expect(got.data.equals(PNG)).toBe(true);+ expect(urls[0]).toBe("http://cdn/c2c/download?encrypted_query_param=a%2Bb%2F%3D");+ });++ it("image: media.aes_key as base64 of the raw key, or no key at all (plain bytes)", async () => {+ const { client } = cdn({ e: enc(JPG, key), p: JPG });+ expect((await client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "e", aes_key: key.toString("base64") } } })).mime).toBe("image/jpeg");+ expect((await client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "p" } } })).data.equals(JPG)).toBe(true);+ });++ it("file: aes_key is base64 of the hex string; keeps the file name. video: same key form", async () => {+ const { client } = cdn({ f: enc(Buffer.from("季度数据\n"), key), v: enc(Buffer.from("mp4"), key) });+ const f = await client.downloadMedia({ type: 4, file_item: { file_name: "报告.txt", media: { encrypt_query_param: "f", aes_key: hexKeyB64 } } });+ expect(f).toMatchObject({ kind: "file", name: "报告.txt" });+ expect(f.data.toString()).toBe("季度数据\n");+ expect((await client.downloadMedia({ type: 5, video_item: { media: { encrypt_query_param: "v", aes_key: hexKeyB64 } } })).kind).toBe("video");+ });++ it("prefers full_url when the server gives one", async () => {+ const { client, urls } = cdn({ "dl/1": enc(PNG, key) });+ const got = await client.downloadMedia({ type: 2, image_item: { aeskey: key.toString("hex"), media: { full_url: "http://cdn2/dl/1", encrypt_query_param: "x" } } });+ expect(got.data.equals(PNG)).toBe(true);+ expect(urls).toEqual(["http://cdn2/dl/1"]);+ });++ it("errors: 4xx is final, 5xx is retried 3 times, wrong key, missing key, oversize, odd scheme", async () => {+ const { client, urls } = cdn({ bad: 503, img: enc(PNG, key) });+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "nope" } } })).rejects.toThrow(/HTTP 400 invalid encrypted_param/);+ expect(urls).toHaveLength(1);+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "bad" } } })).rejects.toThrow(/HTTP 503/);+ expect(urls.filter((u) => u.endsWith("=bad"))).toHaveLength(3);+ await expect(client.downloadMedia({ type: 2, image_item: { aeskey: crypto.randomBytes(16).toString("hex"), media: { encrypt_query_param: "img" } } })).rejects.toThrow(/解密失败/);+ await expect(client.downloadMedia({ type: 4, file_item: { media: { encrypt_query_param: "img" } } })).rejects.toThrow(/aes_key/);+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "img" } } }, 100)).rejects.toThrow(/过大/);+ await expect(client.downloadMedia({ type: 2, image_item: { media: { full_url: "file:///etc/passwd" } } })).rejects.toThrow(/http/);+ });++ it("helpers: key encodings, format sniffing, which items are downloadable", () => {+ expect(parseAesKey(key.toString("base64")).equals(key)).toBe(true);+ expect(parseAesKey(hexKeyB64).equals(key)).toBe(true);+ expect(() => parseAesKey(Buffer.from("short").toString("base64"))).toThrow();+ expect(sniffImageMime(Buffer.from("GIF89a....."))).toBe("image/gif");+ expect(sniffImageMime(Buffer.from("RIFF0000WEBPVP8 "))).toBe("image/webp");+ expect(sniffImageMime(Buffer.from("BM.........."))).toBeUndefined();+ expect(hasDownloadableMedia({ type: 2, image_item: { media: { encrypt_query_param: "q" } } })).toBe(true);+ expect(hasDownloadableMedia({ type: 2, image_item: {} })).toBe(false);+ expect(hasDownloadableMedia({ type: 3, voice_item: { media: { encrypt_query_param: "q" } } })).toBe(false); // 语音用转写文字+ });+});++describe("weixinParts with downloads", () => {+ const img: MessageItem = { type: 2, image_item: { media: { encrypt_query_param: "i" } } };+ const quotedImg: MessageItem = { type: 2, image_item: { media: { encrypt_query_param: "q" } } };+ const items: MessageItem[] = [+ { type: 1, text_item: { text: "这两张有什么不同?" }, ref_msg: { title: "[图片]", message_item: quotedImg } },+ img,+ { type: 4, file_item: { file_name: "a.pdf", media: { encrypt_query_param: "f" } } },+ ];++ it("before download: placeholders only (approval interception sees plain labels)", () => {+ expect(eventText(ev(weixinParts(items)))).toBe("[引用的图片]\n这两张有什么不同?\n[图片]\n[文件 a.pdf]");+ });++ it("after download: data on the parts, errors reported, quoted media instead of its title", () => {+ const media = new Map<MessageItem, any>([+ [img, { kind: "image", data: PNG, mime: "image/png" }],+ [quotedImg, { kind: "image", data: JPG, mime: "image/jpeg" }],+ [items[2], new Error("CDN 下载失败 HTTP 400")],+ ]);+ const parts = weixinParts(items, media);+ expect(parts.map((p) => p.type)).toEqual(["image", "text", "image", "file"]);+ expect(parts[0]).toMatchObject({ quoted: true, mime: "image/jpeg" });+ expect((parts[2] as any).data.equals(PNG)).toBe(true);+ expect(eventText(ev(parts))).toBe("[引用的图片]\n这两张有什么不同?\n[图片]\n[文件 a.pdf(未能下载:CDN 下载失败 HTTP 400)]");+ });+});++function ev(parts: Part[]): InboundEvent {+ return { id: "m1", channel: "weixin", botId: "b", conversation: { kind: "dm", id: "u" }, sender: { id: "u" }, parts, replyHandle: {}, receivedAt: 0 };+}++describe("storeInbound / inboundImages", () => {+ it("saves media under inbox/ with safe names and rewrites the message text to the paths", () => {+ const ws = fs.mkdtempSync(path.join(os.tmpdir(), "amb-inbox-"));+ const e = ev([+ { type: "text", text: "看看" },+ { type: "image", data: PNG, mime: "image/png" },+ { type: "image", data: Buffer.from("BM...."), mime: undefined },+ { type: "file", name: "../../etc/evil\u0007.txt", data: Buffer.from("x") },+ { type: "video", data: Buffer.from("v") },+ { type: "image", error: "CDN 下载失败 HTTP 400" },+ ]);+ storeInbound(ws, e, new Date("2026-10-31T06:43:12Z"));+ const paths = e.parts.map((p: any) => p.path).filter(Boolean);+ expect(paths).toHaveLength(4);+ expect(paths[0]).toMatch(/^inbox\/20261031-064312-[0-9a-f]{6}\.png$/);+ expect(paths[1]).toMatch(/\.jpg$/); // 认不出的格式按 jpg 存+ expect(paths[2]).toMatch(/^inbox\/20261031-064312-[0-9a-f]{6}-evil\.txt$/);+ expect(paths[3]).toMatch(/\.mp4$/);+ expect(fs.readFileSync(path.join(ws, paths[0])).equals(PNG)).toBe(true);+ expect(fs.readdirSync(ws)).toEqual(["inbox"]);+ expect(eventText(e)).toBe(`看看\n[图片 ${paths[0]}]\n[图片 ${paths[1]}]\n[文件 ../../etc/evil\u0007.txt → ${paths[2]}]\n[视频 ${paths[3]}]\n[图片(未能下载:CDN 下载失败 HTTP 400)]`);+ });++ it("only recognised, reasonably small images go to the model, at most 4", () => {+ const parts: Part[] = [+ { type: "image", data: PNG, mime: "image/png" },+ { type: "image", data: Buffer.from("BM"), mime: undefined },+ { type: "image", data: Buffer.alloc(11 * 1024 * 1024, 1), mime: "image/jpeg" },+ ...Array.from({ length: 5 }, () => ({ type: "image" as const, data: JPG, mime: "image/jpeg" })),+ ];+ const imgs = inboundImages(ev(parts));+ expect(imgs).toHaveLength(4);+ expect(imgs[0]).toEqual({ data: PNG.toString("base64"), mimeType: "image/png" });+ expect(imgs.slice(1).every((i) => i.mimeType === "image/jpeg")).toBe(true);+ });++ it("acceptsImages: vision families by name; DeepSeek / GLM text models / unknown are text-only", () => {+ for (const m of ["claude-opus-5-5", "gpt-5.4", "gpt-6.1-sol", "gemini-3.8-flash", "grok-4.7", "kimi-k3", "muse-spark-1.3", "qwen3-vl-plus", "glm-4.5v", "key1/claude-sonnet-5"]) expect(acceptsImages(m), m).toBe(true);+ for (const m of ["deepseek-v4-flash", "glm-5.3", "codex-auto-review", "qwen3.8-max", "mock-a", "jev-1.13"]) expect(acceptsImages(m), m).toBe(false);+ });+});