The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
import { readdirSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
const USER_TEXT = "Describe the attached image briefly.";
const REPLY = "The image is attached.";
const EDITED_TEXT = "Describe the same attached image with more detail.";
const EDITED_REPLY = "The same image remains attached after editing.";
const FILE_NAME = "typed-roundtrip.png";
const IMAGE_BYTES = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
"base64",
);
function unwrap(value) {
for (let depth = 0; depth < 6 && value && typeof value === "object"; depth += 1) {
if (value.error) throw new Error(`core RPC failed: ${JSON.stringify(value.error)}`);
const wrapper = ["result", "data", "value"].find((key) => key in value);
if (!wrapper) break;
value = value[wrapper];
}
return value;
}
function findStoredBytes(root, expected) {
for (const name of readdirSync(root)) {
const path = join(root, name);
const stat = statSync(path);
if (stat.isDirectory()) {
const found = findStoredBytes(path, expected);
if (found) return found;
} else if (stat.isFile() && stat.size === expected.length) {
const bytes = readFileSync(path);
if (bytes.equals(expected)) return bytes;
}
}
return null;
}
function hasSourceAuthority(value) {
if (Array.isArray(value)) return value.some(hasSourceAuthority);
if (!value || typeof value !== "object") return false;
return Object.entries(value).some(([key, child]) =>
(["path", "url", "localCachePath"].includes(key) &&
typeof child === "string" &&
child.length > 0) ||
hasSourceAuthority(child),
);
}
export default async function typedImageAttachmentRoundtrip({
page,
mock,
log,
logDir,
}) {
const actionDir = process.env.OPENHUMAN_ACTION_DIR;
if (!actionDir) throw new Error("OPENHUMAN_ACTION_DIR must isolate staged uploads");
const rpcCalls = [];
let rpcUrl;
let authorization;
page.on("request", async (request) => {
if (!request.url().endsWith("/rpc")) return;
rpcUrl = request.url();
authorization = (await request.allHeaders()).authorization;
try {
const body = JSON.parse(request.postData() ?? "{}");
rpcCalls.push({ method: body.method, params: body.params });
} catch {}
});
mock.set("llmStreamScript", [{ text: REPLY }, { finish: "stop" }]);
await page.goto(new URL("/#/chat", page.url()).toString());
const composer = page.getByRole("textbox", { name: "Message input" });
await composer.waitFor({ state: "visible", timeout: 30_000 });
let threadId = await page.evaluate(
() =>
window.__OPENHUMAN_STORE__?.getState?.().thread?.selectedThreadId ?? null,
);
if (!threadId) {
await page.getByRole("button", { name: /new conversation/i }).click();
threadId = await page
.waitForFunction(
() => window.__OPENHUMAN_STORE__?.getState?.().thread?.selectedThreadId,
null,
{ timeout: 15_000 },
)
.then((handle) => handle.jsonValue());
}
if (!threadId) throw new Error("new chat thread was not selected");
await page.locator('input[type="file"]').setInputFiles({
name: FILE_NAME,
mimeType: "image/png",
buffer: IMAGE_BYTES,
});
await page
.getByText(FILE_NAME, { exact: true })
.waitFor({ state: "visible" });
await composer.fill(USER_TEXT);
await page.getByTestId("send-message-button").click();
await page.getByText(REPLY, { exact: false }).waitFor({ timeout: 90_000 });
const append = rpcCalls.find(
(call) => call.method === "openhuman.threads_message_append",
);
if (!append)
throw new Error(
"composer did not persist the user message through the core",
);
const part = append.params?.attachments?.[0];
const dataUri = part?.media?.url;
if (
part?.kind !== "image" ||
part?.media?.contentType !== "image/png" ||
part?.media?.fileName !== FILE_NAME ||
!dataUri?.startsWith("data:image/png;name=")
) {
throw new Error(
`core append lacked the typed PNG payload: ${JSON.stringify(part)}`,
);
}
const stagedMessage = unwrap(
await page.evaluate(
async ({ rpcUrl: url, token, threadId: id }) => {
const response = await fetch(url, {
method: "POST",
headers: { authorization: token, "content-type": "application/json" },
body: JSON.stringify({
jsonrpc: "2.0",
id: "typed-image-attachment-roundtrip-read",
method: "openhuman.threads_messages_list",
params: { thread_id: id },
}),
});
return response.json();
},
{ rpcUrl, token: authorization, threadId },
),
);
const persisted = stagedMessage?.messages?.find(
(message) =>
message.sender === "user" && message.id === append.params?.message?.id,
);
if (!persisted)
throw new Error("typed image was not persisted in the thread message list");
if (/\[(?:ATTACHMENT|IMAGE|FILE):/i.test(persisted.content))
throw new Error(
"modern typed upload leaked a staging marker into new persisted content",
);
const reference = persisted.attachments?.[0];
if (
reference?.kind !== "image" ||
reference.media?.contentType !== "image/png" ||
reference.media?.fileName !== FILE_NAME ||
reference.media?.sizeBytes !== IMAGE_BYTES.length ||
!reference.media?.opaqueId ||
reference.media?.path ||
reference.media?.url
) {
throw new Error(
`persisted image reference is not opaque and typed: ${JSON.stringify(reference)}`,
);
}
if (persisted.content !== USER_TEXT)
throw new Error("persisted user text differs from clean composer text");
const stagedBytes = findStoredBytes(actionDir, IMAGE_BYTES);
if (!stagedBytes)
throw new Error("staged image bytes differ from the browser upload");
log(
`core staged ${reference.media.contentType} ${reference.media.fileName} (${stagedBytes.length} bytes) behind opaque ref ${reference.media.opaqueId}`,
);
const chatRequest = rpcCalls.find(
(call) => call.method === "openhuman.channel_web_chat",
);
if (
!chatRequest ||
chatRequest.params?.message !== persisted.content ||
chatRequest.params?.persisted_message_id !== persisted.id
)
throw new Error(
"chat turn did not reference the marker-free persisted user row",
);
const clientId = chatRequest.params?.client_id;
if (typeof clientId !== "string" || clientId.length === 0)
throw new Error("initial chat request did not expose its live client ID");
if (/\[(?:ATTACHMENT|IMAGE|FILE):/i.test(chatRequest.params.message))
throw new Error(
"chat turn sent a staging marker in its modern user text",
);
if (
JSON.stringify(chatRequest.params?.attachments) !==
JSON.stringify(persisted.attachments)
)
throw new Error("chat turn did not carry the persisted opaque media ref");
if (
/data:[^\s\"]+|base64,/i.test(JSON.stringify(chatRequest.params.attachments)) ||
hasSourceAuthority(chatRequest.params.attachments)
)
throw new Error(
`chat turn sent source bytes or a filesystem/network path: ${JSON.stringify(chatRequest.params.attachments)}`,
);
const inferenceRequest = mock
.requests()
.find((request) => /(?:^|\/)chat\/completions(?:\?|$)/.test(request.url));
if (!inferenceRequest)
throw new Error(
"mock backend did not receive the model request for the image turn",
);
const inferenceBody = JSON.parse(inferenceRequest.body);
const userMessages =
inferenceBody.messages?.filter((message) => message.role === "user") ?? [];
const userText = JSON.stringify(userMessages);
if (/\[(?:ATTACHMENT|IMAGE|FILE):/i.test(userText))
throw new Error(
"legacy or durable staging markers reached the model's user text",
);
if (!userText.includes(IMAGE_BYTES.toString("base64")))
throw new Error(
"model request did not receive the uploaded image bytes as typed media",
);
log("mock model request contained typed image bytes and clean user text");
// Exercise the real edit RPC with the persisted row's opaque refs. The core
// must resolve them against that exact user row and restage trusted bytes;
// it must not turn the refs into user-controlled paths or markers.
mock.set("llmStreamScript", [
{ text: EDITED_REPLY },
{ finish: "stop" },
]);
const editResponse = unwrap(
await page.evaluate(
async ({ rpcUrl: url, token, threadId: id, messageId, attachments, clientId: socketId }) => {
const response = await fetch(url, {
method: "POST",
headers: { authorization: token, "content-type": "application/json" },
body: JSON.stringify({
jsonrpc: "2.0",
id: "typed-image-edit-roundtrip",
method: "openhuman.threads_edit_message",
params: {
thread_id: id,
message_id: messageId,
content: "Describe the same attached image with more detail.",
attachments,
client_id: socketId,
},
}),
});
return response.json();
},
{
rpcUrl,
token: authorization,
threadId,
messageId: persisted.id,
attachments: persisted.attachments,
clientId,
},
),
);
if (typeof editResponse?.request_id !== "string")
throw new Error(
`edit RPC did not admit the replacement turn: ${JSON.stringify(editResponse)}`,
);
await page.getByText(EDITED_REPLY, { exact: false }).waitFor({ timeout: 90_000 });
const editedMessages = unwrap(
await page.evaluate(
async ({ rpcUrl: url, token, threadId: id }) => {
const response = await fetch(url, {
method: "POST",
headers: { authorization: token, "content-type": "application/json" },
body: JSON.stringify({
jsonrpc: "2.0",
id: "typed-image-edit-roundtrip-read",
method: "openhuman.threads_messages_list",
params: { thread_id: id },
}),
});
return response.json();
},
{ rpcUrl, token: authorization, threadId },
),
);
const editedUserRows =
editedMessages?.messages?.filter(
(message) => message.sender === "user" && message.content === EDITED_TEXT,
) ?? [];
const editedUserRow = editedUserRows[0];
const editedRef = editedUserRow?.attachments?.[0];
if (
editedUserRows.length !== 1 ||
editedMessages.messages.some(
(message) =>
(message.sender === "user" && message.content === USER_TEXT) ||
(message.sender !== "user" && message.content === REPLY),
) ||
editedUserRow.content.includes("[ATTACHMENT:") ||
editedRef?.kind !== "image" ||
editedRef.media?.opaqueId === reference.media.opaqueId ||
editedRef.media?.path ||
editedRef.media?.url
)
throw new Error(
`edited row did not own fresh opaque image refs: ${JSON.stringify(editedUserRow)}`,
);
const modelRequests = mock
.requests()
.filter((request) => /(?:^|\/)chat\/completions(?:\?|$)/.test(request.url));
const editedModelRequest = [...modelRequests].reverse().find((request) => {
const body = JSON.parse(request.body);
return body.messages?.some(
(message) =>
message.role === "user" &&
JSON.stringify(message.content).includes(EDITED_TEXT) &&
JSON.stringify(message.content).includes(IMAGE_BYTES.toString("base64")),
);
});
const editedModelBody = editedModelRequest && JSON.parse(editedModelRequest.body);
const editedUserMessages =
editedModelBody?.messages?.filter((message) => message.role === "user") ?? [];
const editedPrompt = JSON.stringify(editedUserMessages);
if (
!editedModelRequest ||
!editedPrompt.includes(IMAGE_BYTES.toString("base64")) ||
/\[(?:ATTACHMENT|IMAGE|FILE):/i.test(editedPrompt)
)
throw new Error("edited model request did not receive clean text and original image bytes");
log("editing preserved image bytes and assigned fresh opaque ownership");
await page.reload();
await composer.waitFor({ state: "visible", timeout: 30_000 });
await page.evaluate(
(id) =>
window.__OPENHUMAN_STORE__?.dispatch({
type: "thread/setSelectedThread",
payload: id,
}),
threadId,
);
await page.getByText(EDITED_REPLY, { exact: false }).waitFor({ timeout: 30_000 });
const restoredEdit = await page.evaluate(
({ id, content }) => {
const messages =
window.__OPENHUMAN_STORE__?.getState?.().thread?.messagesByThreadId?.[
id
] ?? [];
return {
row: messages.find(
(message) => message.sender === "user" && message.content === content,
),
staleOriginal: messages.some(
(message) =>
(message.sender === "user" && message.content === "Describe the attached image briefly.") ||
(message.sender !== "user" && message.content === "The image is attached."),
),
};
},
{ id: threadId, content: EDITED_TEXT },
);
if (
!restoredEdit.row ||
restoredEdit.staleOriginal ||
restoredEdit.row.content !== EDITED_TEXT ||
/\[(?:ATTACHMENT|IMAGE|FILE):/i.test(restoredEdit.row.content ?? "") ||
restoredEdit.row.attachments?.[0]?.media?.opaqueId !== editedRef.media.opaqueId
)
throw new Error("edited image row did not reload with clean text and fresh opaque reference");
log(
"typed image attachment survived core staging, chat submission and reload",
);
}