Skip to content

🤖 Open-source, local-first alternative to Grok Bot, OpenAI dots and Claude Duet. Your own AI bots on any model: GPT, Claude, Gemini, Grok, DeepSeek, Kimi, GLM and 70+ more through one Modelflare key. Reach them in the app, WeChat or Feishu; they remember you, work in the background, run commands and use your computer. Pay per use.

feat(weixin): receive images and files from WeChat; vision models see them

Marla0f664f34c239
diff --git a/README.md b/README.md
index bb6b20d..01308f7 100644
--- a/README.md
+++ b/README.md
@@ -132,6 +132,7 @@ docker exec amb cat /data/ui-token # 打开 http://<主机>:3600/?token=<令
- 信息面板「渠道」→ 微信 → 「扫码登录」,用微信扫描二维码并确认;或命令行 `npm run weixin:login`(终端显示二维码,同时保存 PNG 到数据目录)。
- 登录态、同步游标、context_token 保存在数据目录,重启后自动续用。
- 平台限制:用户每发来一条消息后 24 小时内最多可回复 10 条;回复在生成结束后一次发出,超过 4000 字自动分段;生成期间显示“正在输入”。
+- 图片、文件、视频:用户发来(或引用)的会从微信 CDN 下载解密(单个最大 50 MB)。代理模式下存进机器人工作区的 `inbox/`,消息里写明路径,代理可以读取、处理或当生图的参考图;能看图的模型(Claude、GPT-5/6、Gemini、Grok 4、Kimi K3、Muse Spark 等)同时直接收到图片本身,其余模型只拿到路径。Linux 版代理暂不直接收图(sharp 在 Electron 下会崩溃),只给路径。机器人交付的文件(present)按图片或文件发回微信。
- 登录过期(错误码 -14)时自动暂停并在界面提示,重新扫码即可。
## 开发与测试
diff --git a/scripts/verify-dsh-bundle.mjs b/scripts/verify-dsh-bundle.mjs
index 5a3e594..b5e395a 100644
--- a/scripts/verify-dsh-bundle.mjs
+++ b/scripts/verify-dsh-bundle.mjs
@@ -68,7 +68,9 @@ await step("DSH host answers through a mock model", async () => {
if (!body.stream) return res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ choices: [{ message: { role: "assistant", content: "title" }, finish_reason: "stop" }] }));
res.writeHead(200, { "content-type": "text/event-stream" });
const send = (o) => res.write(`data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", model: body.model, ...o })}\n\n`);
- send({ choices: [{ index: 0, delta: { role: "assistant", content: "bundle-pong" } }] });
+ // a user image reaches the model as an image_url part (after DSH admitted and normalized it with sharp)
+ const image = body.messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p?.type === "image_url"));
+ send({ choices: [{ index: 0, delta: { role: "assistant", content: image ? "bundle-image-seen" : "bundle-pong" } }] });
send({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] });
send({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } });
res.end("data: [DONE]\n\n");
@@ -91,6 +93,7 @@ await step("DSH host answers through a mock model", async () => {
" name: mock",
" contextWindow: 128000",
" maxTokens: 16000",
+ " input: [text, image]",
"",
].join("\n"),
);
@@ -106,7 +109,16 @@ await step("DSH host answers through a mock model", async () => {
console.log(`DSH_HOST_INIT_MS=${started}`);
const r = await withTimeout(h.session().run("ping"), 60_000, "turn");
if (!String(r.finalResponse).includes("bundle-pong")) throw new Error(`unexpected reply ${JSON.stringify(r.finalResponse)}`);
- return `(sdk import ${sdkMs} ms, host spawn -> initialized ${started} ms)`;
+ // An image the user sent in chat (src/agent/host.ts imageInput). Not on Linux: sharp crashes under Electron
+ // there, so the app never passes images to DSH on Linux.
+ let image = "";
+ if (process.platform !== "linux") {
+ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
+ const ri = await withTimeout(h.session().run([{ type: "text", text: "what is this?" }, { type: "image", data: png, mimeType: "image/png" }]), 60_000, "image turn");
+ if (!String(ri.finalResponse).includes("bundle-image-seen")) throw new Error(`image did not reach the model: ${JSON.stringify(ri.finalResponse)}`);
+ image = ", image turn ok";
+ }
+ return `(sdk import ${sdkMs} ms, host spawn -> initialized ${started} ms${image})`;
} finally {
await h.close().catch(() => {});
llm.close();
diff --git a/src/agent/host.ts b/src/agent/host.ts
index e457c66..4584b5b 100644
--- a/src/agent/host.ts
+++ b/src/agent/host.ts
@@ -11,6 +11,7 @@ import { adaptSkill, mcpPatchLines, resolveComputerUse, type CuResolved } from "
import { cuaConfig, cuaMcpEnv, discoverAppsCached, type CuaApp } from "./cua-driver.js";
import { pwshPatchLines, pwshProbe } from "./pwsh.js";
import { DEFAULT_SETTINGS, permissionMode, type BotSettings } from "./settings.js";
+import { acceptsImages } from "../llm/client.js";
const log = logger("agent.host");
@@ -191,6 +192,21 @@ export function modelLimits(model: string): { window: number; maxTokens: number
return { window: lim[1], maxTokens: lim[2] };
}
+/** 随消息交给模型的图片(DSH SDK 的内联图片块:base64 + 格式,进会话前存入 DSH 的附件库) */
+export interface TurnImage {
+ data: string;
+ mimeType: "image/png" | "image/jpeg" | "image/gif" | "image/webp";
+}
+
+export interface TurnInput {
+ conversationId: string;
+ text: string;
+ model: string;
+ observer?: TurnObserver;
+ /** 只在模型能看图时传(acceptsImages),否则模型只拿到文字里的工作区路径 */
+ images?: TurnImage[];
+}
+
/** 回合结束时的上下文用量(来自 DSH 事件)。 */
export interface TurnUsage {
/** 最后一次模型请求的输入+输出 token,约等于当前上下文大小 */
@@ -333,6 +349,8 @@ export class AgentHost {
` name: ${yamlStr(model.split("/").pop() || model)}`,
` contextWindow: ${lim[1]}`,
` maxTokens: ${lim[2]}`,
+ // 默认只收文字;能看图的模型声明图片输入,用户发来的图片才会作为图片交给它
+ ...(acceptsImages(model) ? [" input: [text, image]"] : []),
// Windows:命令用检测到的 PowerShell 7(DSH 自己只看 Program Files\PowerShell\7 和 PATH,找不到就退回 5.1)
...pwshPatchLines(pwshPath),
"- insert:",
@@ -764,7 +782,14 @@ exec "$REAL" "$@"
this.o.store.clearAgentSession(conversationId);
}
- async runTurn(input: { conversationId: string; text: string; model: string; observer?: TurnObserver }): Promise<{ text: string; sessionId: string; files: string[]; usage: TurnUsage }> {
+ /**
+ * 能否把用户发来的图片交给 DSH。DSH 收图时用 sharp 规范化,而 sharp 在 Linux 上的 Electron 里会让进程崩溃
+ * (SharpElectronLinux,见 scripts/verify-dsh-bundle.mjs),那里只给工作区路径。其它平台上如果带图的回合
+ * 让 DSH 崩了,本次运行期间也不再给图。
+ */
+ imageInput = !(process.versions.electron && process.platform === "linux") && process.env.AMB_DSH_IMAGES !== "0";
+
+ async runTurn(input: TurnInput): Promise<{ text: string; sessionId: string; files: string[]; usage: TurnUsage }> {
const h = await this.ensure(input.model);
const s = await this.sessionFor(h, input.conversationId);
this.active++;
@@ -773,9 +798,11 @@ exec "$REAL" "$@"
const presented: string[] = [];
const lim = modelLimits(input.model);
const usage: TurnUsage = { tokens: 0, window: lim.window, maxTokens: lim.maxTokens };
+ const images = this.imageInput ? (input.images ?? []) : [];
try {
const timeoutMs = this.o.turnTimeoutMs ?? 30 * 60_000;
- const run = s.run(input.text, {
+ const prompt = images.length ? [{ type: "text", text: input.text }, ...images.map((i) => ({ type: "image", data: i.data, mimeType: i.mimeType }))] : input.text;
+ const run = s.run(prompt, {
onNotification: (n) => {
if (process.env.AMB_DSH_TRACE) fs.appendFileSync(process.env.AMB_DSH_TRACE, JSON.stringify(n).slice(0, 2000) + "\n");
if (n.method !== "session.event" || n.params?.sessionId !== s.id) return;
@@ -806,6 +833,10 @@ exec "$REAL" "$@"
return { text: r.finalResponse, sessionId: s.id, files: deliverables(this.workspace, presented), usage };
} catch (e) {
if (/timed out/.test((e as Error).message)) await this.closeHarness();
+ else if (images.length && this.state !== "running") {
+ this.imageInput = false;
+ log.warn("dsh died on a turn with images; sending images by path only until the app restarts", { bot: this.o.botId, message: (e as Error).message.slice(0, 200) });
+ }
throw e;
} finally {
clearTimeout(timer);
@@ -900,7 +931,7 @@ You have no Modelflare key and must not call the API yourself (no curl, no scrip
1. \`media_models\` (kind \`image\` or \`video\`) lists the models this bot may use, the default one, their parameter ranges and reference prices. Use the default unless the user named a model or the request needs something the default cannot do (a reference image, a ratio or length it does not support).
2. Plain text-to-image or text-to-video with default settings: call \`generate_image\` / \`generate_video\` with a detailed \`prompt\` only.
3. Anything else (ratio, resolution, size, length, quality, audio, reference images, first/last frames, editing an image): call \`media_docs\` for the chosen model, then pass \`body\` in exactly the format its page shows, without \`model\`. Every family is different (for example Seedance uses \`content[]\`, Kling \`contents\` and \`settings\`, Wan \`input\` and \`parameters\`, Gemini \`contents[].parts[]\` with \`generationConfig\`), so never reuse field names from another model's page. For OpenAI Images models whose page sends references to \`/v1/images/edits\`, also pass \`endpoint: "edits"\`. Keep \`prompt\` the same as the prompt inside \`body\`: the user sees \`prompt\` in the confirmation.
-4. Reference media: a file in the workspace is written as \`"workspace:media/x.png"\` (sent as a data URL) or \`"workspace-base64:media/x.png"\` (raw Base64, e.g. Gemini \`inline_data.data\`), up to 20 MB each. A public https URL can be used as is. Models marked 必须带参考素材 (image-to-video, reference-to-video) only work with references; many models accept reference videos only as public URLs, see the docs. Images the user sent in chat are not in the workspace: if you have no file or link for a reference, ask the user for one.
+4. Reference media: a file in the workspace is written as \`"workspace:media/x.png"\` (sent as a data URL) or \`"workspace-base64:media/x.png"\` (raw Base64, e.g. Gemini \`inline_data.data\`), up to 20 MB each. A public https URL can be used as is. Models marked 必须带参考素材 (image-to-video, reference-to-video) only work with references; many models accept reference videos only as public URLs, see the docs. Images and files the user sends in chat are saved in the workspace under \`inbox/\`, and the message shows where (for example \`[图片 inbox/20261031-143012-ab12cd.jpg]\`), so write such an image as \`"workspace:inbox/20261031-143012-ab12cd.jpg"\`. If you have no file or link for a reference, ask the user for one.
5. The tool result says what happened:
- Generated: the file is under \`media/\`; deliver it with \`present\`.
- The user picked another model while you had passed a \`body\`: read that model's page with \`media_docs\`, rewrite \`body\` for it and call again with \`model\` set to it. That call is not confirmed again.
diff --git a/src/agent/manager.ts b/src/agent/manager.ts
index 4bfb2f0..ca13c30 100644
--- a/src/agent/manager.ts
+++ b/src/agent/manager.ts
@@ -3,7 +3,7 @@ import type { Store } from "../store/db.js";
import { logger } from "../log.js";
import { AgentBridge } from "./bridge.js";
import path from "node:path";
-import { AgentHost, callDetail, type TurnObserver, type TurnUsage } from "./host.js";
+import { AgentHost, callDetail, type TurnInput, type TurnObserver, type TurnUsage } from "./host.js";
import type { MediaModel, MediaPlan, MediaService } from "./media.js";
import { APPROVAL_TTL_MS, ApprovalCenter, type QuestionItem } from "./approvals.js";
import { MemoryError, MemoryStore, type Scope } from "./memory.js";
@@ -166,7 +166,7 @@ export class AgentManager {
/** 会话 → 正在进行的回合开始时间(/status 与故障排查用:后来的消息排在它后面) */
readonly turnsInProgress = new Map<string, { botId: string; since: number }>();
- async runTurn(botId: string, input: { conversationId: string; text: string; model: string; observer?: TurnObserver }) {
+ async runTurn(botId: string, input: TurnInput) {
const conv = input.conversationId;
this.turnsInProgress.set(conv, { botId, since: Date.now() });
try {
@@ -176,7 +176,7 @@ export class AgentManager {
}
}
- private async runTurnInner(botId: string, input: { conversationId: string; text: string; model: string; observer?: TurnObserver }) {
+ private async runTurnInner(botId: string, input: TurnInput) {
const conv = input.conversationId;
const gate = this.gates.get(conv);
if (gate) await withTimeout(gate, 240_000, undefined);
diff --git a/src/app.ts b/src/app.ts
index 1f357ad..51d47fd 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -505,6 +505,7 @@ export class App {
if (bot.weixinEnabled && !ch.weixin) {
ch.weixin = new WeixinChannel(bot.id, this.store, this.runtime, this.queue, {
fetchImpl: this.opts.weixinFetch,
+ cdnBaseUrl: this.config.weixin.cdnBaseUrl,
pollTimeoutMs: this.opts.weixinPollTimeoutMs,
});
ch.weixin.onExpired = () => this.notice("warn", `机器人「${bot.name}」微信登录已过期,请在信息面板的「渠道」里重新扫码`);
diff --git a/src/channels/weixin/api.ts b/src/channels/weixin/api.ts
index 5b595ec..abad87d 100644
--- a/src/channels/weixin/api.ts
+++ b/src/channels/weixin/api.ts
@@ -26,9 +26,60 @@ export function encryptAesEcb(plain: Buffer, key: Buffer): Buffer {
const c = crypto.createCipheriv("aes-128-ecb", key, null);
return Buffer.concat([c.update(plain), c.final()]);
}
+export function decryptAesEcb(cipher: Buffer, key: Buffer): Buffer {
+ const d = crypto.createDecipheriv("aes-128-ecb", key, null);
+ return Buffer.concat([d.update(cipher), d.final()]);
+}
export function aesEcbPaddedSize(n: number): number {
return Math.ceil((n + 1) / 16) * 16;
}
+
+/**
+ * CDNMedia.aes_key 有两种编码(移植自 openclaw-weixin src/cdn/pic-decrypt.ts):
+ * 图片是 16 字节原始密钥的 base64;文件/语音/视频是 32 位十六进制字符串的 base64。
+ */
+export function parseAesKey(aesKeyBase64: string): Buffer {
+ const decoded = Buffer.from(aesKeyBase64, "base64");
+ if (decoded.length === 16) return decoded;
+ const ascii = decoded.toString("ascii");
+ if (decoded.length === 32 && /^[0-9a-fA-F]{32}$/.test(ascii)) return Buffer.from(ascii, "hex");
+ throw new Error(`aes_key 解码后为 ${decoded.length} 字节,应为 16 字节或 32 位十六进制`);
+}
+
+export type ImageMime = "image/png" | "image/jpeg" | "image/gif" | "image/webp";
+const IMAGE_MIME_EXT: Record<ImageMime, string> = { "image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp" };
+
+/** 按文件头识别图片格式(微信图片条目不带文件名和 MIME);不认识的返回 undefined。 */
+export function sniffImageMime(buf: Buffer): ImageMime | undefined {
+ if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg";
+ if (buf.length >= 8 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";
+ if (buf.length >= 6 && buf.subarray(0, 4).toString("ascii") === "GIF8") return "image/gif";
+ if (buf.length >= 12 && buf.subarray(0, 4).toString("ascii") === "RIFF" && buf.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";
+ return undefined;
+}
+export function imageExt(mime: string | undefined): string {
+ return IMAGE_MIME_EXT[mime as ImageMime] ?? ".jpg";
+}
+
+/** 单个入站媒体(解密后)的大小上限 */
+export const MAX_INBOUND_MEDIA_BYTES = 50 * 1024 * 1024;
+const CDN_TIMEOUT_MS = 60_000;
+
+export interface InboundMedia {
+ kind: "image" | "file" | "video";
+ data: Buffer;
+ /** 文件条目的原始文件名 */
+ name?: string;
+ /** 图片按文件头识别出的格式 */
+ mime?: ImageMime;
+}
+
+/** 图片/文件/视频条目带有可下载的 CDN 引用。语音不下载:有转写文字,原始音频是 SILK。 */
+export function hasDownloadableMedia(item: MessageItem | undefined): boolean {
+ const media =
+ item?.type === MessageItemType.IMAGE ? item.image_item?.media : item?.type === MessageItemType.FILE ? item.file_item?.media : item?.type === MessageItemType.VIDEO ? item.video_item?.media : undefined;
+ return !!(media?.encrypt_query_param || media?.full_url);
+}
export const SESSION_EXPIRED_ERRCODE = -14;
const CHANNEL_VERSION = "0.1.0";
const BOT_AGENT = "AnymodelBot/0.1.0";
@@ -102,14 +153,17 @@ export interface CdnMedia {
encrypt_query_param?: string;
aes_key?: string;
encrypt_type?: number;
+ /** 完整下载地址(服务端给出时优先用它,不再自己拼) */
+ full_url?: string;
}
export interface MessageItem {
type?: number;
text_item?: { text?: string };
- voice_item?: { text?: string };
+ voice_item?: { text?: string; media?: CdnMedia };
file_item?: { file_name?: string; media?: CdnMedia; len?: string };
- image_item?: { media?: CdnMedia; mid_size?: number };
+ /** aeskey:入站图片的原始密钥(十六进制),优先于 media.aes_key */
+ image_item?: { media?: CdnMedia; aeskey?: string; mid_size?: number };
video_item?: { media?: CdnMedia; video_size?: number };
ref_msg?: { title?: string; message_item?: MessageItem };
}
@@ -282,17 +336,20 @@ export class IlinkClient {
},
15_000,
);
- const up = parseWeixinApiJson<{ ret?: number; errmsg?: string; upload_full_url?: string; upload_param?: string }>(raw || "{}");
- const cdnBase = (this.opts.cdnBaseUrl || DEFAULT_CDN_BASE).replace(/\/+$/, "");
- const url = up.upload_full_url?.trim() || (up.upload_param ? `${cdnBase}/upload?encrypted_query_param=${encodeURIComponent(up.upload_param)}&filekey=${encodeURIComponent(filekey)}` : "");
+ const up = parseWeixinApiJson<{ ret?: number; errcode?: number; errmsg?: string; upload_full_url?: string; upload_param?: string }>(raw || "{}");
+ if ((up.ret && up.ret !== 0) || (up.errcode && up.errcode !== 0)) {
+ throw new WeixinApiError(`getuploadurl ret=${up.ret} errcode=${up.errcode} errmsg=${up.errmsg ?? ""}`, up.ret, up.errcode);
+ }
+ const url = up.upload_full_url?.trim() || (up.upload_param ? `${this.cdnBase}/upload?encrypted_query_param=${encodeURIComponent(up.upload_param)}&filekey=${encodeURIComponent(filekey)}` : "");
if (!url) throw new WeixinApiError(`getuploadurl returned no upload url (ret=${up.ret ?? ""})`, up.ret);
const cipher = encryptAesEcb(plain, aeskey);
let param = "";
let lastErr: unknown;
for (let attempt = 1; attempt <= 3 && !param; attempt++) {
try {
- const res = await this.fetchImpl(url, { method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: new Uint8Array(cipher) });
- if (res.status >= 400 && res.status < 500) throw Object.assign(new Error(`CDN upload client error ${res.status}`), { fatal: true });
+ const res = await this.fetchImpl(url, { method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: new Uint8Array(cipher), signal: AbortSignal.timeout(CDN_TIMEOUT_MS) });
+ await res.arrayBuffer().catch(() => {});
+ if (res.status >= 400 && res.status < 500) throw Object.assign(new Error(`CDN upload client error ${res.status} ${res.headers.get("x-error-message") ?? ""}`.trim()), { fatal: true });
if (res.status !== 200) throw new Error(`CDN upload server error ${res.status}`);
param = res.headers.get("x-encrypted-param") ?? "";
if (!param) throw new Error("CDN upload response missing x-encrypted-param");
@@ -313,6 +370,73 @@ export class IlinkClient {
await this.sendItem(toUserId, item, contextToken);
}
+ private get cdnBase(): string {
+ return (this.opts.cdnBaseUrl || DEFAULT_CDN_BASE).replace(/\/+$/, "");
+ }
+
+ /**
+ * 下载并解密用户发来的图片/文件/视频(含被引用消息里的)。
+ * 移植自 openclaw-weixin src/media/media-download.ts、src/cdn/pic-decrypt.ts、src/cdn/cdn-url.ts。
+ */
+ async downloadMedia(item: MessageItem, maxBytes = MAX_INBOUND_MEDIA_BYTES): Promise<InboundMedia> {
+ let kind: InboundMedia["kind"];
+ let media: CdnMedia | undefined;
+ let key: Buffer | undefined;
+ if (item.type === MessageItemType.IMAGE) {
+ kind = "image";
+ media = item.image_item?.media;
+ // image_item.aeskey(十六进制)优先;两者都没有时图片未加密
+ const hex = item.image_item?.aeskey;
+ if (hex && /^[0-9a-fA-F]{32}$/.test(hex)) key = Buffer.from(hex, "hex");
+ else if (media?.aes_key) key = parseAesKey(media.aes_key);
+ } else if (item.type === MessageItemType.FILE || item.type === MessageItemType.VIDEO) {
+ kind = item.type === MessageItemType.FILE ? "file" : "video";
+ media = kind === "file" ? item.file_item?.media : item.video_item?.media;
+ if (!media?.aes_key) throw new Error("缺少 aes_key");
+ key = parseAesKey(media.aes_key);
+ } else {
+ throw new Error(`不支持下载的条目类型 ${item.type}`);
+ }
+ const full = media?.full_url?.trim();
+ if (full && !/^https?:\/\//i.test(full)) throw new Error("full_url 不是 http(s) 地址");
+ if (!full && !media?.encrypt_query_param) throw new Error("没有 CDN 引用");
+ const url = full || `${this.cdnBase}/download?encrypted_query_param=${encodeURIComponent(media!.encrypt_query_param!)}`;
+
+ let raw: Buffer | undefined;
+ let lastErr: unknown;
+ for (let attempt = 1; attempt <= 3 && !raw; attempt++) {
+ try {
+ const res = await this.fetchImpl(url, { signal: AbortSignal.timeout(CDN_TIMEOUT_MS) });
+ if (!res.ok) {
+ await res.arrayBuffer().catch(() => {});
+ const err = new Error(`CDN 下载失败 HTTP ${res.status} ${res.headers.get("x-error-message") ?? ""}`.trim());
+ throw res.status < 500 ? Object.assign(err, { fatal: true }) : err;
+ }
+ const declared = Number(res.headers.get("content-length") ?? 0);
+ if (declared > maxBytes) {
+ await res.body?.cancel().catch(() => {});
+ throw Object.assign(new Error(`文件过大(${(declared / 1048576).toFixed(1)} MB,上限 ${maxBytes / 1048576} MB)`), { fatal: true });
+ }
+ const buf = Buffer.from(await res.arrayBuffer());
+ if (buf.length > maxBytes) throw Object.assign(new Error(`文件过大(${(buf.length / 1048576).toFixed(1)} MB,上限 ${maxBytes / 1048576} MB)`), { fatal: true });
+ raw = buf;
+ } catch (e) {
+ lastErr = e;
+ if ((e as { fatal?: boolean }).fatal) throw e;
+ }
+ }
+ if (!raw) throw lastErr instanceof Error ? lastErr : new Error("CDN 下载失败");
+ let data: Buffer;
+ try {
+ data = key ? decryptAesEcb(raw, key) : raw;
+ } catch {
+ throw new Error("解密失败(密钥不对?)");
+ }
+ if (kind === "image") return { kind, data, mime: sniffImageMime(data) };
+ if (kind === "file") return { kind, data, name: item.file_item?.file_name || "file" };
+ return { kind, data };
+ }
+
private async sendItem(toUserId: string, item: MessageItem, contextToken: string): Promise<void> {
const raw = await this.request(
"POST",
diff --git a/src/channels/weixin/channel.ts b/src/channels/weixin/channel.ts
index e958c76..34100a7 100644
--- a/src/channels/weixin/channel.ts
+++ b/src/channels/weixin/channel.ts
@@ -3,7 +3,7 @@ import { splitText } from "../../core/types.js";
import { Dedup, KeyedQueue } from "../../core/queue.js";
import type { BotRuntime } from "../../core/runtime.js";
import type { Store } from "../../store/db.js";
-import { IlinkClient, MessageItemType, MessageType, WeixinApiError, type MessageItem, type WeixinMessage } from "./api.js";
+import { IlinkClient, MessageItemType, MessageType, WeixinApiError, hasDownloadableMedia, type InboundMedia, type MessageItem, type WeixinMessage } from "./api.js";
import { StreamingMarkdownFilter } from "./markdown-filter.js";
import { WeixinQuota } from "./quota.js";
import { logger } from "../../log.js";
@@ -25,14 +25,30 @@ export function filterMarkdown(text: string): string {
return (f.feed(text) + f.flush()).trim();
}
-export function weixinParts(items: MessageItem[] | undefined): Part[] {
+/** 媒体下载结果,按条目对象对应(含 ref_msg 里被引用的条目) */
+export type WeixinMediaResults = Map<MessageItem, InboundMedia | Error>;
+
+function mediaPart(it: MessageItem, got: InboundMedia | Error | undefined, quoted: boolean): Part {
+ const type = it.type === MessageItemType.IMAGE ? "image" : it.type === MessageItemType.FILE ? "file" : "video";
+ const part: Part = { type, ...(type === "file" ? { name: it.file_item?.file_name } : {}), ...(quoted ? { quoted: true } : {}) };
+ if (got instanceof Error) return { ...part, error: got.message.slice(0, 120) };
+ if (got) return { ...part, data: got.data, ...(got.mime ? { mime: got.mime } : {}) };
+ return part;
+}
+
+export function weixinParts(items: MessageItem[] | undefined, media?: WeixinMediaResults): Part[] {
const parts: Part[] = [];
for (const it of items ?? []) {
switch (it.type) {
case MessageItemType.TEXT: {
const t = it.text_item?.text ?? "";
- const ref = it.ref_msg?.message_item?.text_item?.text || it.ref_msg?.title;
- if (ref) parts.push({ type: "quote", text: ref });
+ const quoted = it.ref_msg?.message_item;
+ // 引用了一张图片/一个文件:把它本身交给运行时,而不是只给标题
+ if (quoted && hasDownloadableMedia(quoted)) parts.push(mediaPart(quoted, media?.get(quoted), true));
+ else {
+ const ref = quoted?.text_item?.text || it.ref_msg?.title;
+ if (ref) parts.push({ type: "quote", text: ref });
+ }
if (t) parts.push({ type: "text", text: t });
break;
}
@@ -40,19 +56,20 @@ export function weixinParts(items: MessageItem[] | undefined): Part[] {
parts.push({ type: "voice", transcript: it.voice_item?.text });
break;
case MessageItemType.IMAGE:
- parts.push({ type: "image" });
- break;
case MessageItemType.FILE:
- parts.push({ type: "file", name: it.file_item?.file_name });
- break;
case MessageItemType.VIDEO:
- parts.push({ type: "video" });
+ parts.push(mediaPart(it, media?.get(it), false));
break;
}
}
return parts;
}
+/** 消息里需要下载的条目(含被引用的) */
+function mediaItems(items: MessageItem[] | undefined): MessageItem[] {
+ return (items ?? []).flatMap((it) => [it, ...(it.ref_msg?.message_item ? [it.ref_msg.message_item] : [])]).filter(hasDownloadableMedia);
+}
+
/**
* 微信回复会话:不流式;生成期间只显示“正在输入”(每 5 秒续期),结束时一次性发送最终回复(>4000 字切分),遵守配额。
* 不发送工具调用进度;只有最终回复、审批/提问提示和交付文件会成为聊天消息,配额全部留给它们。
@@ -186,7 +203,7 @@ export class WeixinChannel {
private store: Store,
private runtime: BotRuntime,
private queue: KeyedQueue,
- private opts: { fetchImpl?: typeof fetch; pollTimeoutMs?: number; retryDelayMs?: number; backoffMs?: number } = {},
+ private opts: { fetchImpl?: typeof fetch; cdnBaseUrl?: string; pollTimeoutMs?: number; retryDelayMs?: number; backoffMs?: number } = {},
) {
this.quota = new WeixinQuota(store, botId);
}
@@ -202,7 +219,7 @@ export class WeixinChannel {
this.status = { state: "login_required", detail: "尚未扫码登录", since: Date.now() };
return;
}
- this.client = new IlinkClient({ baseUrl: acct.baseUrl, token: acct.botToken, fetchImpl: this.opts.fetchImpl });
+ this.client = new IlinkClient({ baseUrl: acct.baseUrl, token: acct.botToken, fetchImpl: this.opts.fetchImpl, cdnBaseUrl: this.opts.cdnBaseUrl });
this.abort = new AbortController();
this.loop = this.run(this.abort.signal).finally(() => (this.loop = undefined));
}
@@ -302,9 +319,11 @@ export class WeixinChannel {
}
return;
}
+ const media = mediaItems(m.item_list);
const p = this.queue.run(`weixin:${this.botId}:${userId}`, async () => {
const reply = new WeixinReply(this.client!, userId, this.store.getWeixinPeer(this.botId, userId)?.contextToken ?? peer.contextToken, this.quota);
try {
+ if (media.length) ev.parts = weixinParts(m.item_list, await this.download(media));
await this.runtime.handle(ev, reply);
} catch (e) {
this.log.error("handle failed", { message: (e as Error).message });
@@ -315,6 +334,20 @@ export class WeixinChannel {
void p.finally(() => this.pending.delete(p));
}
+ /** 逐个下载解密;失败的记下原因(消息照常处理,文字里会说明没下载成功) */
+ private async download(items: MessageItem[]): Promise<WeixinMediaResults> {
+ const out: WeixinMediaResults = new Map();
+ for (const it of items) {
+ try {
+ out.set(it, await this.client!.downloadMedia(it));
+ } catch (e) {
+ this.log.warn("weixin media download failed", { type: it.type, message: (e as Error).message.slice(0, 200) });
+ out.set(it, e as Error);
+ }
+ }
+ return out;
+ }
+
async idle(): Promise<void> {
while (this.pending.size) await Promise.all([...this.pending]);
}
diff --git a/src/config.ts b/src/config.ts
index 958d3ef..69ce8ac 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -35,7 +35,8 @@ export interface AppConfig {
/** 默认的图片 / 视频模型(生成前确认时默认选中);空 = 按推荐顺序的第一个 */
media: { imageModel: string; videoModel: string };
feishu: { throttleMs: number };
- weixin: { apiBaseUrl: string };
+ /** cdnBaseUrl:收发图片/文件的微信 CDN(测试时指向假服务) */
+ weixin: { apiBaseUrl: string; cdnBaseUrl: string };
/** 代理宿主:dsh = DeepSeek Harness(工具/沙箱/审批);off = 纯对话 */
agent: { mode: "dsh" | "off"; turnTimeoutMs: number; approvalTtlMs: number };
}
@@ -94,7 +95,10 @@ export function loadConfig(overrides: { dataDir?: string; env?: NodeJS.ProcessEn
videoModel: typeof file.media?.videoModel === "string" ? file.media.videoModel : "",
},
feishu: { throttleMs: num(env.AMB_FEISHU_THROTTLE_MS, file.feishu?.throttleMs ?? 400) },
- weixin: { apiBaseUrl: env.AMB_WEIXIN_API_BASE || file.weixin?.apiBaseUrl || "https://ilinkai.weixin.qq.com" },
+ weixin: {
+ apiBaseUrl: env.AMB_WEIXIN_API_BASE || file.weixin?.apiBaseUrl || "https://ilinkai.weixin.qq.com",
+ cdnBaseUrl: env.AMB_WEIXIN_CDN_BASE || file.weixin?.cdnBaseUrl || "https://novac2c.cdn.weixin.qq.com/c2c",
+ },
agent: {
mode: (env.AMB_AGENT || file.agent?.mode || "dsh") === "off" ? "off" : "dsh",
turnTimeoutMs: num(env.AMB_AGENT_TURN_TIMEOUT_MS, file.agent?.turnTimeoutMs ?? 30 * 60_000),
diff --git a/src/core/runtime.ts b/src/core/runtime.ts
index fbab5d5..6ca9f1a 100644
--- a/src/core/runtime.ts
+++ b/src/core/runtime.ts
@@ -1,7 +1,10 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
import path from "node:path";
import type { InboundEvent, ReplySession } from "./types.js";
import { eventText } from "./types.js";
-import type { ModelClient, ChatMessage } from "../llm/client.js";
+import { acceptsImages, type ModelClient, type ChatMessage } from "../llm/client.js";
+import type { TurnImage } from "../agent/host.js";
import type { Store } from "../store/db.js";
import { logger } from "../log.js";
import type { AgentManager } from "../agent/manager.js";
@@ -13,6 +16,48 @@ import type { ApprovalRow } from "../store/db.js";
const log = logger("runtime");
+const IMAGE_EXT: Record<TurnImage["mimeType"], string> = { "image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif", "image/webp": ".webp" };
+const MAX_INLINE_IMAGES = 4;
+const MAX_INLINE_IMAGE_BYTES = 10 * 1024 * 1024;
+
+/** 用户这条消息里能直接交给模型看的图片(格式已识别、不太大) */
+export function inboundImages(ev: InboundEvent): TurnImage[] {
+ const out: TurnImage[] = [];
+ for (const p of ev.parts) {
+ if (p.type !== "image" || !p.data || !(p.mime && p.mime in IMAGE_EXT) || p.data.length > MAX_INLINE_IMAGE_BYTES) continue;
+ if (out.length < MAX_INLINE_IMAGES) out.push({ data: p.data.toString("base64"), mimeType: p.mime as TurnImage["mimeType"] });
+ }
+ return out;
+}
+
+/** 发送者给的文件名只留最后一段,去掉控制字符和开头的点 */
+function safeFileName(name: string | undefined): string {
+ const base = path.basename((name ?? "").replace(/\\/g, "/")).replace(/[\x00-\x1f\x7f]/g, "").replace(/^\.+/, "").trim();
+ return (base || "file").slice(-80);
+}
+
+/**
+ * 把用户发来的图片/文件/视频存进机器人工作区的 inbox/,消息文字里换成相对路径:
+ * 代理可以读取、处理,或当生图的参考素材(workspace:inbox/...)。
+ */
+export function storeInbound(workspace: string, ev: InboundEvent, now = new Date()): void {
+ const stamp = now.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
+ for (const p of ev.parts) {
+ if ((p.type !== "image" && p.type !== "file" && p.type !== "video") || !p.data || p.path) continue;
+ const id = `${stamp}-${crypto.randomBytes(3).toString("hex")}`;
+ const name =
+ p.type === "image" ? `${id}${IMAGE_EXT[p.mime as TurnImage["mimeType"]] ?? ".jpg"}` : p.type === "video" ? `${id}.mp4` : `${id}-${safeFileName(p.name)}`;
+ try {
+ fs.mkdirSync(path.join(workspace, "inbox"), { recursive: true });
+ fs.writeFileSync(path.join(workspace, "inbox", name), p.data);
+ p.path = `inbox/${name}`;
+ } catch (e) {
+ log.warn("save inbound media failed", { type: p.type, message: (e as Error).message.slice(0, 200) });
+ p.error = "保存到工作区失败";
+ }
+ }
+}
+
export interface RuntimeOptions {
store: Store;
models: ModelClient;
@@ -65,8 +110,12 @@ export class BotRuntime {
const bot = store.getBot(ev.botId);
if (!bot) throw new Error(`unknown bot ${ev.botId}`);
const conv = store.ensureConversation(bot.id, ev.channel, ev.conversation.id, ev.sender.name || ev.sender.id, ev.conversation.kind === "group" ? "group" : "dm");
+ const agent = this.opts.agent;
+ // 代理模式:收到的图片/文件先进工作区,消息里写路径
+ if (agent?.available) storeInbound(agent.host(bot.id).workspace, ev);
const text = eventText(ev);
if (!text) return;
+ const images = inboundImages(ev);
if (text.startsWith("/")) {
const reply = await this.command(text, conv.id, conv.model, bot.defaultModel, bot.id);
@@ -82,11 +131,12 @@ export class BotRuntime {
});
if (!model) return;
const routed = this.opts.route ? this.opts.route(bot.id, model) : model;
+ // 只有能看图的模型才收到图片本身;其余模型只看到文字里的“[图片 …]”
+ const seen = acceptsImages(routed) ? images : [];
- const agent = this.opts.agent;
let note = "";
if (agent?.available) {
- const handled = await this.handleAgent(agent, bot.id, conv.id, text, routed, bot.systemPrompt, out);
+ const handled = await this.handleAgent(agent, bot.id, conv.id, text, routed, bot.systemPrompt, out, seen);
if (handled === true) return;
// 代理进程还在启动(第一次可能要几分钟):这条消息先由模型直接回答,不让用户干等
if (handled === "starting") note = "(代理还在启动,这条消息先由模型直接回答;启动完成后会自动用回代理。)";
@@ -97,6 +147,10 @@ export class BotRuntime {
const messages: ChatMessage[] = [];
if (bot.systemPrompt.trim()) messages.push({ role: "system", content: bot.systemPrompt });
for (const m of history) if (m.role !== "system") messages.push({ role: m.role, content: m.content });
+ const last = messages[messages.length - 1];
+ if (seen.length && last?.role === "user" && typeof last.content === "string") {
+ last.content = [{ type: "text", text: last.content }, ...seen.map((i) => ({ type: "image_url" as const, image_url: { url: `data:${i.mimeType};base64,${i.data}` } }))];
+ }
await out.apply({ type: "typing", on: true });
await out.apply({ type: "stream.start" });
@@ -176,7 +230,7 @@ export class BotRuntime {
* 代理回合;返回 false 表示宿主不可用且尚无任何输出(调用方回退到纯对话),
* "starting" 表示代理进程在等待时限内没有启动好(同样回退,启动在后台继续)。
*/
- private async handleAgent(agent: AgentManager, botId: string, convId: string, text: string, model: string, systemPrompt: string, out: ReplySession): Promise<boolean | "starting"> {
+ private async handleAgent(agent: AgentManager, botId: string, convId: string, text: string, model: string, systemPrompt: string, out: ReplySession, images: TurnImage[] = []): Promise<boolean | "starting"> {
const { store } = this.opts;
let visible = false;
const unbind = agent.approvals.bindSink(convId, async (row) => {
@@ -223,6 +277,7 @@ export class BotRuntime {
conversationId: convId,
text: input,
model,
+ ...(images.length ? { images } : {}),
observer: {
// 工具调用不进入聊天:只用于判断代理已开始工作(不再回退到纯对话),正在输入状态由渠道维持
onProgress: () => {
diff --git a/src/core/types.ts b/src/core/types.ts
index 5761cc6..280324d 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -6,7 +6,21 @@ export type ChannelId = "feishu" | "weixin" | "web";
export type Part =
| { type: "text"; text: string }
- | { type: "image" | "file" | "video"; ref?: string; name?: string }
+ | {
+ type: "image" | "file" | "video";
+ ref?: string;
+ name?: string;
+ /** 渠道下载到的内容(运行时存进工作区后填 path) */
+ data?: Buffer;
+ /** 图片格式(按文件头识别) */
+ mime?: string;
+ /** 存进机器人工作区后的相对路径,如 inbox/20261031-143012-ab12cd.jpg */
+ path?: string;
+ /** 下载失败的原因 */
+ error?: string;
+ /** 来自被引用的消息 */
+ quoted?: boolean;
+ }
| { type: "voice"; ref?: string; transcript?: string }
| { type: "quote"; text: string };
@@ -69,6 +83,8 @@ export type ChannelStatus = {
since: number;
};
+const MEDIA_LABEL = { image: "图片", file: "文件", video: "视频" } as const;
+
export function eventText(ev: InboundEvent): string {
return ev.parts
.map((p) => {
@@ -80,11 +96,13 @@ export function eventText(ev: InboundEvent): string {
case "voice":
return p.transcript ? p.transcript : "[语音]";
case "image":
- return "[图片]";
case "file":
- return `[文件${p.name ? " " + p.name : ""}]`;
- case "video":
- return "[视频]";
+ case "video": {
+ // [图片 inbox/x.jpg]、[引用的文件 报告.pdf → inbox/x-报告.pdf]、[视频(未能下载:…)]
+ const label = `${p.quoted ? "引用的" : ""}${MEDIA_LABEL[p.type]}${p.type === "file" && p.name ? " " + p.name : ""}`;
+ if (p.path) return `[${label}${p.type === "file" && p.name ? " →" : ""} ${p.path}]`;
+ return p.error ? `[${label}(未能下载:${p.error})]` : `[${label}]`;
+ }
}
})
.join("\n")
diff --git a/src/llm/client.ts b/src/llm/client.ts
index a3075e3..08dc0ff 100644
--- a/src/llm/client.ts
+++ b/src/llm/client.ts
@@ -2,9 +2,21 @@
* 模型访问接口。默认实现指向受管 magpie 的 OpenAI 兼容 /v1(流式 SSE)。
* 运行时只依赖 ModelClient,可替换为任意 OpenAI 兼容端点。
*/
+/** OpenAI 兼容的多模态内容;只有用户刚发来的图片会用到(data URL) */
+export type ChatContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
+
export interface ChatMessage {
role: "system" | "user" | "assistant";
- content: string;
+ content: string | ChatContentPart[];
+}
+
+/**
+ * 能看图的模型。Modelflare 的模型目录不带输入模态,只能按名字判断;不在其中的模型
+ * (DeepSeek、GLM 文本版等)收到图片时只拿到它在工作区里的路径。
+ */
+const IMAGE_INPUT_MODELS = /claude|gpt-4o|gpt-4\.1|gpt-[5-9]|gemini|grok-4|kimi-k3|muse-spark|qvq|-vl\b|-omni\b|glm-[\d.]+v\b/i;
+export function acceptsImages(model: string): boolean {
+ return IMAGE_INPUT_MODELS.test(model);
}
export interface ChatRequest {
diff --git a/test/agent.integration.test.ts b/test/agent.integration.test.ts
index 892d8d4..8d25f26 100644
--- a/test/agent.integration.test.ts
+++ b/test/agent.integration.test.ts
@@ -49,10 +49,16 @@ describe.skipIf(!runtime || !hasBwrap || process.platform !== "linux")("agent ho
function plan(messages: any[]): { tool?: { name: string; args: any }; text?: string } {
let lastUser = -1;
for (let i = messages.length - 1; i >= 0; i--) {
- if (messages[i].role === "user" && /QUICKSORT|ESCALATE|SECRET|READFILE|LOOPBACK|PRESENT|ASKQ|NATIVES|\[用户消息\]/.test(textOf(messages[i].content))) { lastUser = i; break; }
+ if (messages[i].role === "user" && /QUICKSORT|ESCALATE|SECRET|READFILE|LOOPBACK|PRESENT|ASKQ|NATIVES|LOOKIMG|\[用户消息\]/.test(textOf(messages[i].content))) { lastUser = i; break; }
}
const user = textOf(messages[lastUser]?.content);
const tools = messages.slice(lastUser + 1).filter((m) => m.role === "tool");
+ // 看图:回报模型收到的图片(data URL 的头)和文字里有没有工作区路径
+ if (/LOOKIMG/.test(user)) {
+ const content = messages[lastUser]?.content;
+ const urls = (Array.isArray(content) ? content : []).filter((p: any) => p?.type === "image_url").map((p: any) => String(p.image_url?.url ?? "").split(",")[0]);
+ return { text: `SEEN<<${urls.join(" ")}|${/\[图片 inbox\/[^\]]+\.png\]/.test(user) ? "path" : "nopath"}>>` };
+ }
const scripts: Array<[RegExp, Array<{ name: string; args: any }>]> = [
[/QUICKSORT/, [{ name: "bash", args: { description: "run", command: "cat > qs.py <<'PY'\ndef qs(a):\n return a if len(a)<2 else qs([x for x in a[1:] if x<a[0]])+[a[0]]+qs([x for x in a[1:] if x>=a[0]])\nprint(qs([5,3,9,1,7,2]))\nPY\npython3 qs.py" } }]],
[/ESCALATE/, [{ name: "bash", args: { description: "run", command: "echo escalated-$((40+2))", sandbox_permissions: "danger-full-access", justification: "测试提权" } }]],
@@ -244,4 +250,24 @@ describe.skipIf(!runtime || !hasBwrap || process.platform !== "linux")("agent ho
expect(again.reply).not.toContain("⚠️");
expect(app.agent.host("default").state).toBe("running");
}, 180_000);
+
+ it("images from chat: saved under the workspace inbox/; a vision model gets the image itself, a text model only the path", async () => {
+ // 1x1 PNG(DSH 收图时会校验并规范化,必须是真图)
+ const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", "base64");
+ const send = async (user: string, model: string) => {
+ const conv = app.store.ensureConversation("default", "weixin", user, user, "dm");
+ app.store.setConversationModel(conv.id, model);
+ const ops: any[] = [];
+ const ev = { id: `img-${user}`, channel: "weixin" as const, botId: "default", conversation: { kind: "dm" as const, id: user }, sender: { id: user }, parts: [{ type: "text" as const, text: "LOOKIMG" }, { type: "image" as const, data: png, mime: "image/png" }], replyHandle: {}, receivedAt: Date.now() };
+ await app.runtime.handle(ev, { apply: async (op) => void ops.push(op) });
+ return ops.find((o) => o.type === "stream.finish")?.text as string;
+ };
+ // Electron on Linux (scripts/test-dsh-bundle.mjs): sharp crashes there, so the host keeps images to paths
+ if (app.agent.host("default").imageInput) expect(await send("img-vision", "gpt-5-mock")).toMatch(/^SEEN<<data:image\/(png|jpeg|webp);base64\|path>>$/);
+ else expect(await send("img-vision", "gpt-5-mock")).toBe("SEEN<<|path>>");
+ expect(await send("img-text", "mock-agent")).toBe("SEEN<<|path>>");
+ const inbox = fs.readdirSync(path.join(dataDir, "workspaces", "default", "inbox"));
+ expect(inbox.filter((f) => /^\d{8}-\d{6}-[0-9a-f]{6}\.png$/.test(f))).toHaveLength(2);
+ expect(fs.readFileSync(path.join(dataDir, "workspaces", "default", "inbox", inbox[0])).equals(png)).toBe(true);
+ }, 180_000);
});
diff --git a/test/e2e.test.ts b/test/e2e.test.ts
index 9f98144..b991a78 100644
--- a/test/e2e.test.ts
+++ b/test/e2e.test.ts
@@ -2,6 +2,7 @@
* 离线端到端:mock OpenAI(流式)+ mock iLink + 假飞书传输层 + 真实 App/Web 服务(外部模型端点模式)。
*/
import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import crypto from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -31,7 +32,7 @@ beforeAll(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "amb-e2e-"));
const config = loadConfig({
dataDir,
- env: { AMB_MAGPIE_BASE_URL: llm.url + "/v1", AMB_MAGPIE_API_KEY: "gw-key", AMB_UI_PORT: "0", AMB_WEIXIN_API_BASE: ilink.url, AMB_FEISHU_THROTTLE_MS: "30", AMB_AGENT: "off" },
+ env: { AMB_MAGPIE_BASE_URL: llm.url + "/v1", AMB_MAGPIE_API_KEY: "gw-key", AMB_UI_PORT: "0", AMB_WEIXIN_API_BASE: ilink.url, AMB_WEIXIN_CDN_BASE: ilink.cdnBase, AMB_FEISHU_THROTTLE_MS: "30", AMB_AGENT: "off" },
});
process.env.FEISHU_APP_ID = "cli_test";
process.env.FEISHU_APP_SECRET = "secret_test";
@@ -120,6 +121,47 @@ describe("微信 E2E", () => {
expect(st.json.weixin.quota[0]).toMatchObject({ userId: "[email protected]", remaining: 9 });
expect(app.store.getWeixinAccount("default")!.cursor).toMatch(/^cur-/);
});
+ const png = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), crypto.randomBytes(64)]);
+ it("收图片/文件:CDN 下载解密;能看图的模型收到图片本身,其余模型只收到标注;下载失败照常回复", async () => {
+ const conv = app.store.ensureConversation("default", "weixin", "[email protected]", "[email protected]", "dm");
+ app.store.setConversationModel(conv.id, "gpt-5.4");
+ let prev = llm.calls.length;
+ const img = ilink.imageItem(png);
+ ilink.pushItems("[email protected]", [{ type: 1, text_item: { text: "这是什么?" } }, img]);
+ await until(() => llm.calls.length > prev, 8000);
+ expect(ilink.state.cdnDownloads).toContain(img.image_item.media.encrypt_query_param);
+ let call = llm.calls.at(-1)!;
+ expect(call.model).toBe("gpt-5.4");
+ expect(call.messages.at(-1).content).toEqual([
+ { type: "text", text: "这是什么?\n[图片]" },
+ { type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } },
+ ]);
+ await until(() => ilink.state.sent.some((s) => s.to === "[email protected]"), 8000);
+
+ // 纯文字模型(mock-a):只有标注,不发图片
+ prev = llm.calls.length;
+ ilink.pushItems("[email protected]", [ilink.imageItem(png), ilink.fileItem(Buffer.from("hi"), "notes.txt")]);
+ await until(() => llm.calls.length > prev, 8000);
+ expect(llm.calls.at(-1)!.messages.at(-1).content).toBe("[图片]\n[文件 notes.txt]");
+
+ prev = llm.calls.length;
+ ilink.pushItems("[email protected]", [{ type: 2, image_item: { media: { encrypt_query_param: "gone" } } }]);
+ await until(() => llm.calls.length > prev, 8000);
+ expect(llm.calls.at(-1)!.messages.at(-1).content).toBe("[图片(未能下载:CDN 下载失败 HTTP 400 invalid encrypted_param)]");
+ await app.channels.get("default")!.weixin!.idle();
+ });
+ it("发图片:getuploadurl → 加密上传 CDN → 图片消息;收件方按消息里的引用和密钥能取回原图", async () => {
+ const f = path.join(dataDir, "out.png");
+ fs.writeFileSync(f, png);
+ const client = (app.channels.get("default")!.weixin as any).client;
+ const before = ilink.state.sent.length;
+ await client.sendMedia("[email protected]", f, "ctx-1");
+ const s = ilink.state.sent.at(-1)!;
+ expect(ilink.state.sent.length).toBe(before + 1);
+ expect(s).toMatchObject({ to: "[email protected]", contextToken: "ctx-1", item: { type: 2, image_item: { mid_size: 80, media: { encrypt_type: 1 } } } });
+ const got = await client.downloadMedia(s.item);
+ expect(got.data.equals(png)).toBe(true);
+ });
it("超长回复按 4000 切分并受配额约束", async () => {
const big = "段落".repeat(2600) + "\n\n" + "尾".repeat(3000); // ~8200 字
const prev = ilink.state.sent.length;
diff --git a/test/helpers.ts b/test/helpers.ts
index 6300baf..40b5607 100644
--- a/test/helpers.ts
+++ b/test/helpers.ts
@@ -110,7 +110,10 @@ export async function mockIlink() {
const state = {
statusSeq: ["wait", "scaned", "confirmed"] as string[],
inbox: [] as any[],
- sent: [] as Array<{ to: string; text: string; contextToken?: string }>,
+ sent: [] as Array<{ to: string; text: string; contextToken?: string; item?: any }>,
+ /** 假 CDN:encrypt_query_param → 密文(下载);上传的密文也存这里 */
+ cdn: new Map<string, Buffer>(),
+ cdnDownloads: [] as string[],
typing: [] as number[],
expireNext: false,
authSeen: [] as string[],
@@ -121,6 +124,19 @@ export async function mockIlink() {
const u = new URL(req.url!, "http://x");
const json = (o: unknown) => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(o));
if (req.headers.authorization) state.authSeen.push(String(req.headers.authorization));
+ if (u.pathname === "/c2c/download") {
+ const q = u.searchParams.get("encrypted_query_param") ?? "";
+ state.cdnDownloads.push(q);
+ const b = state.cdn.get(q);
+ return b ? res.writeHead(200, { "content-type": "application/octet-stream" }).end(b) : res.writeHead(400, { "x-error-message": "invalid encrypted_param" }).end();
+ }
+ if (u.pathname === "/c2c/upload") {
+ const chunks: Buffer[] = [];
+ for await (const c of req) chunks.push(c as Buffer);
+ const id = `down-${state.cdn.size + 1}`;
+ state.cdn.set(id, Buffer.concat(chunks));
+ return res.writeHead(200, { "x-encrypted-param": id }).end();
+ }
switch (u.pathname) {
case "/ilink/bot/get_bot_qrcode":
return json({ qrcode: "qr-1", qrcode_img_content: "https://liteapp.weixin.qq.com/q/mock?qrcode=qr-1" });
@@ -142,9 +158,14 @@ export async function mockIlink() {
}
case "/ilink/bot/sendmessage": {
const b = await body(req);
- state.sent.push({ to: b.msg.to_user_id, text: b.msg.item_list[0].text_item.text, contextToken: b.msg.context_token });
+ const item = b.msg.item_list[0];
+ state.sent.push({ to: b.msg.to_user_id, text: item.text_item?.text ?? "", contextToken: b.msg.context_token, ...(item.text_item ? {} : { item }) });
return json({ ret: 0 });
}
+ case "/ilink/bot/getuploadurl": {
+ const b = await body(req);
+ return json({ ret: 0, upload_param: `up-${b.filekey}` });
+ }
case "/ilink/bot/getconfig":
await body(req);
return json({ ret: 0, typing_ticket: "ticket-1" });
@@ -158,9 +179,26 @@ export async function mockIlink() {
});
base = srv.url;
let seq = 0;
- const push = (from: string, text: string, ctx = "ctx-1") =>
- state.inbox.push({ seq: ++seq, message_id: `m${seq}`, from_user_id: from, to_user_id: "[email protected]", create_time_ms: Date.now(), message_type: 1, message_state: 2, context_token: ctx, item_list: [{ type: 1, text_item: { text } }] });
- return { ...srv, state, push };
+ const pushItems = (from: string, items: any[], ctx = "ctx-1") =>
+ state.inbox.push({ seq: ++seq, message_id: `m${seq}`, from_user_id: from, to_user_id: "[email protected]", create_time_ms: Date.now(), message_type: 1, message_state: 2, context_token: ctx, item_list: items });
+ const push = (from: string, text: string, ctx = "ctx-1") => pushItems(from, [{ type: 1, text_item: { text } }], ctx);
+ /** 像微信那样把明文用随机密钥加密放到假 CDN,返回图片条目(密钥放在 image_item.aeskey) */
+ const imageItem = (plain: Buffer) => {
+ const key = crypto.randomBytes(16);
+ const c = crypto.createCipheriv("aes-128-ecb", key, null);
+ const q = `img+${crypto.randomBytes(4).toString("hex")}/=`;
+ state.cdn.set(q, Buffer.concat([c.update(plain), c.final()]));
+ return { type: 2, image_item: { aeskey: key.toString("hex"), media: { encrypt_query_param: q, aes_key: key.toString("base64"), encrypt_type: 1 } } };
+ };
+ /** 文件条目:aes_key 是十六进制密钥字符串的 base64(与微信一致) */
+ const fileItem = (plain: Buffer, name: string) => {
+ const key = crypto.randomBytes(16);
+ const c = crypto.createCipheriv("aes-128-ecb", key, null);
+ const q = `file+${crypto.randomBytes(4).toString("hex")}/=`;
+ state.cdn.set(q, Buffer.concat([c.update(plain), c.final()]));
+ return { type: 4, file_item: { file_name: name, len: String(plain.length), media: { encrypt_query_param: q, aes_key: Buffer.from(key.toString("hex")).toString("base64"), encrypt_type: 1 } } };
+ };
+ return { ...srv, state, push, pushItems, imageItem, fileItem, cdnBase: `${srv.url}/c2c` };
}
/** 假飞书传输层:记录全部调用 */
diff --git a/test/weixin-media.unit.test.ts b/test/weixin-media.unit.test.ts
new file mode 100644
index 0000000..cd59aa6
--- /dev/null
+++ b/test/weixin-media.unit.test.ts
@@ -0,0 +1,163 @@
+/** 微信入站媒体:iLink CDN 下载解密、条目 → Part、存进工作区、交给模型的图片。 */
+import { describe, it, expect } from "vitest";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { IlinkClient, hasDownloadableMedia, parseAesKey, sniffImageMime, type MessageItem } from "../src/channels/weixin/api.js";
+import { weixinParts } from "../src/channels/weixin/channel.js";
+import { eventText, type InboundEvent, type Part } from "../src/core/types.js";
+import { inboundImages, storeInbound } from "../src/core/runtime.js";
+import { acceptsImages } from "../src/llm/client.js";
+
+const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), crypto.randomBytes(200)]);
+const JPG = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), crypto.randomBytes(300)]);
+const enc = (b: Buffer, k: Buffer) => {
+ const c = crypto.createCipheriv("aes-128-ecb", k, null);
+ return Buffer.concat([c.update(b), c.final()]);
+};
+
+/** 假 CDN:/c2c/download?encrypted_query_param=… 与 full_url 两种取法 */
+function cdn(blobs: Record<string, Buffer | number>) {
+ const urls: string[] = [];
+ const fetchImpl = (async (url: string) => {
+ urls.push(url);
+ const u = new URL(url);
+ const id = u.pathname === "/c2c/download" ? u.searchParams.get("encrypted_query_param")! : u.pathname.slice(1);
+ const b = blobs[id];
+ if (typeof b === "number") return new Response("", { status: b, headers: { "x-error-message": "boom" } });
+ return b ? new Response(new Uint8Array(b)) : new Response("", { status: 400, headers: { "x-error-message": "invalid encrypted_param" } });
+ }) as any;
+ return { client: new IlinkClient({ baseUrl: "http://ilink", token: "t", fetchImpl, cdnBaseUrl: "http://cdn/c2c" }), urls };
+}
+
+describe("IlinkClient.downloadMedia", () => {
+ const key = crypto.randomBytes(16);
+ const hexKeyB64 = Buffer.from(key.toString("hex")).toString("base64");
+
+ it("image: image_item.aeskey (hex) wins; query param is URL-encoded; format sniffed", async () => {
+ const { client, urls } = cdn({ "a+b/=": enc(PNG, key) });
+ const got = await client.downloadMedia({ type: 2, image_item: { aeskey: key.toString("hex"), media: { encrypt_query_param: "a+b/=", aes_key: "ignored" } } });
+ expect(got).toMatchObject({ kind: "image", mime: "image/png" });
+ expect(got.data.equals(PNG)).toBe(true);
+ expect(urls[0]).toBe("http://cdn/c2c/download?encrypted_query_param=a%2Bb%2F%3D");
+ });
+
+ it("image: media.aes_key as base64 of the raw key, or no key at all (plain bytes)", async () => {
+ const { client } = cdn({ e: enc(JPG, key), p: JPG });
+ expect((await client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "e", aes_key: key.toString("base64") } } })).mime).toBe("image/jpeg");
+ expect((await client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "p" } } })).data.equals(JPG)).toBe(true);
+ });
+
+ it("file: aes_key is base64 of the hex string; keeps the file name. video: same key form", async () => {
+ const { client } = cdn({ f: enc(Buffer.from("季度数据\n"), key), v: enc(Buffer.from("mp4"), key) });
+ const f = await client.downloadMedia({ type: 4, file_item: { file_name: "报告.txt", media: { encrypt_query_param: "f", aes_key: hexKeyB64 } } });
+ expect(f).toMatchObject({ kind: "file", name: "报告.txt" });
+ expect(f.data.toString()).toBe("季度数据\n");
+ expect((await client.downloadMedia({ type: 5, video_item: { media: { encrypt_query_param: "v", aes_key: hexKeyB64 } } })).kind).toBe("video");
+ });
+
+ it("prefers full_url when the server gives one", async () => {
+ const { client, urls } = cdn({ "dl/1": enc(PNG, key) });
+ const got = await client.downloadMedia({ type: 2, image_item: { aeskey: key.toString("hex"), media: { full_url: "http://cdn2/dl/1", encrypt_query_param: "x" } } });
+ expect(got.data.equals(PNG)).toBe(true);
+ expect(urls).toEqual(["http://cdn2/dl/1"]);
+ });
+
+ it("errors: 4xx is final, 5xx is retried 3 times, wrong key, missing key, oversize, odd scheme", async () => {
+ const { client, urls } = cdn({ bad: 503, img: enc(PNG, key) });
+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "nope" } } })).rejects.toThrow(/HTTP 400 invalid encrypted_param/);
+ expect(urls).toHaveLength(1);
+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "bad" } } })).rejects.toThrow(/HTTP 503/);
+ expect(urls.filter((u) => u.endsWith("=bad"))).toHaveLength(3);
+ await expect(client.downloadMedia({ type: 2, image_item: { aeskey: crypto.randomBytes(16).toString("hex"), media: { encrypt_query_param: "img" } } })).rejects.toThrow(/解密失败/);
+ await expect(client.downloadMedia({ type: 4, file_item: { media: { encrypt_query_param: "img" } } })).rejects.toThrow(/aes_key/);
+ await expect(client.downloadMedia({ type: 2, image_item: { media: { encrypt_query_param: "img" } } }, 100)).rejects.toThrow(/过大/);
+ await expect(client.downloadMedia({ type: 2, image_item: { media: { full_url: "file:///etc/passwd" } } })).rejects.toThrow(/http/);
+ });
+
+ it("helpers: key encodings, format sniffing, which items are downloadable", () => {
+ expect(parseAesKey(key.toString("base64")).equals(key)).toBe(true);
+ expect(parseAesKey(hexKeyB64).equals(key)).toBe(true);
+ expect(() => parseAesKey(Buffer.from("short").toString("base64"))).toThrow();
+ expect(sniffImageMime(Buffer.from("GIF89a....."))).toBe("image/gif");
+ expect(sniffImageMime(Buffer.from("RIFF0000WEBPVP8 "))).toBe("image/webp");
+ expect(sniffImageMime(Buffer.from("BM.........."))).toBeUndefined();
+ expect(hasDownloadableMedia({ type: 2, image_item: { media: { encrypt_query_param: "q" } } })).toBe(true);
+ expect(hasDownloadableMedia({ type: 2, image_item: {} })).toBe(false);
+ expect(hasDownloadableMedia({ type: 3, voice_item: { media: { encrypt_query_param: "q" } } })).toBe(false); // 语音用转写文字
+ });
+});
+
+describe("weixinParts with downloads", () => {
+ const img: MessageItem = { type: 2, image_item: { media: { encrypt_query_param: "i" } } };
+ const quotedImg: MessageItem = { type: 2, image_item: { media: { encrypt_query_param: "q" } } };
+ const items: MessageItem[] = [
+ { type: 1, text_item: { text: "这两张有什么不同?" }, ref_msg: { title: "[图片]", message_item: quotedImg } },
+ img,
+ { type: 4, file_item: { file_name: "a.pdf", media: { encrypt_query_param: "f" } } },
+ ];
+
+ it("before download: placeholders only (approval interception sees plain labels)", () => {
+ expect(eventText(ev(weixinParts(items)))).toBe("[引用的图片]\n这两张有什么不同?\n[图片]\n[文件 a.pdf]");
+ });
+
+ it("after download: data on the parts, errors reported, quoted media instead of its title", () => {
+ const media = new Map<MessageItem, any>([
+ [img, { kind: "image", data: PNG, mime: "image/png" }],
+ [quotedImg, { kind: "image", data: JPG, mime: "image/jpeg" }],
+ [items[2], new Error("CDN 下载失败 HTTP 400")],
+ ]);
+ const parts = weixinParts(items, media);
+ expect(parts.map((p) => p.type)).toEqual(["image", "text", "image", "file"]);
+ expect(parts[0]).toMatchObject({ quoted: true, mime: "image/jpeg" });
+ expect((parts[2] as any).data.equals(PNG)).toBe(true);
+ expect(eventText(ev(parts))).toBe("[引用的图片]\n这两张有什么不同?\n[图片]\n[文件 a.pdf(未能下载:CDN 下载失败 HTTP 400)]");
+ });
+});
+
+function ev(parts: Part[]): InboundEvent {
+ return { id: "m1", channel: "weixin", botId: "b", conversation: { kind: "dm", id: "u" }, sender: { id: "u" }, parts, replyHandle: {}, receivedAt: 0 };
+}
+
+describe("storeInbound / inboundImages", () => {
+ it("saves media under inbox/ with safe names and rewrites the message text to the paths", () => {
+ const ws = fs.mkdtempSync(path.join(os.tmpdir(), "amb-inbox-"));
+ const e = ev([
+ { type: "text", text: "看看" },
+ { type: "image", data: PNG, mime: "image/png" },
+ { type: "image", data: Buffer.from("BM...."), mime: undefined },
+ { type: "file", name: "../../etc/evil\u0007.txt", data: Buffer.from("x") },
+ { type: "video", data: Buffer.from("v") },
+ { type: "image", error: "CDN 下载失败 HTTP 400" },
+ ]);
+ storeInbound(ws, e, new Date("2026-10-31T06:43:12Z"));
+ const paths = e.parts.map((p: any) => p.path).filter(Boolean);
+ expect(paths).toHaveLength(4);
+ expect(paths[0]).toMatch(/^inbox\/20261031-064312-[0-9a-f]{6}\.png$/);
+ expect(paths[1]).toMatch(/\.jpg$/); // 认不出的格式按 jpg 存
+ expect(paths[2]).toMatch(/^inbox\/20261031-064312-[0-9a-f]{6}-evil\.txt$/);
+ expect(paths[3]).toMatch(/\.mp4$/);
+ expect(fs.readFileSync(path.join(ws, paths[0])).equals(PNG)).toBe(true);
+ expect(fs.readdirSync(ws)).toEqual(["inbox"]);
+ expect(eventText(e)).toBe(`看看\n[图片 ${paths[0]}]\n[图片 ${paths[1]}]\n[文件 ../../etc/evil\u0007.txt → ${paths[2]}]\n[视频 ${paths[3]}]\n[图片(未能下载:CDN 下载失败 HTTP 400)]`);
+ });
+
+ it("only recognised, reasonably small images go to the model, at most 4", () => {
+ const parts: Part[] = [
+ { type: "image", data: PNG, mime: "image/png" },
+ { type: "image", data: Buffer.from("BM"), mime: undefined },
+ { type: "image", data: Buffer.alloc(11 * 1024 * 1024, 1), mime: "image/jpeg" },
+ ...Array.from({ length: 5 }, () => ({ type: "image" as const, data: JPG, mime: "image/jpeg" })),
+ ];
+ const imgs = inboundImages(ev(parts));
+ expect(imgs).toHaveLength(4);
+ expect(imgs[0]).toEqual({ data: PNG.toString("base64"), mimeType: "image/png" });
+ expect(imgs.slice(1).every((i) => i.mimeType === "image/jpeg")).toBe(true);
+ });
+
+ it("acceptsImages: vision families by name; DeepSeek / GLM text models / unknown are text-only", () => {
+ for (const m of ["claude-opus-5-5", "gpt-5.4", "gpt-6.1-sol", "gemini-3.8-flash", "grok-4.7", "kimi-k3", "muse-spark-1.3", "qwen3-vl-plus", "glm-4.5v", "key1/claude-sonnet-5"]) expect(acceptsImages(m), m).toBe(true);
+ for (const m of ["deepseek-v4-flash", "glm-5.3", "codex-auto-review", "qwen3.8-max", "mock-a", "jev-1.13"]) expect(acceptsImages(m), m).toBe(false);
+ });
+});