Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { createHash } from "node:crypto";
import { HttpAgent, type Message, type RunAgentInput } from "@ag-ui/client";
import { EventSchemas } from "@ag-ui/core/schemas";
import type { BuiltInAgentClassicConfig } from "@copilotkit/runtime/v2";
import { PLAN_RUN_COMPLETED } from "@openbot/toolkit/copilotkit/agents";
import type { Subscription } from "rxjs";
import {
clearDesktopConnectionFailure,
recordDesktopConnectionFailure,
} from "../desktop-connection-failure";
// Use the model contract of the runtime we actually ship, without a second AI SDK version.
export type PlanLanguageModel = Extract<
BuiltInAgentClassicConfig["model"],
{ specificationVersion: "v3" }
>;
type Options = Parameters<PlanLanguageModel["doStream"]>[0];
type StreamResult = Awaited<ReturnType<PlanLanguageModel["doStream"]>>;
type Part = StreamResult["stream"] extends ReadableStream<infer T> ? T : never;
type GenerateResult = Awaited<ReturnType<PlanLanguageModel["doGenerate"]>>;
export type PlanModelConfig = {
provider: "chatgpt" | "claude";
/** Resolved from the deployment's installed harness, never an agent registration. */
endpoint: URL;
token: string;
};
export { PLAN_RUN_COMPLETED };
/** A plan is an explicit desktop credential choice, not an API-key fallback. */
export function planModelForEnvironment(
environment: Record<string, string | undefined>,
): PlanModelConfig | undefined {
const claude = !!environment.CLAUDE_CODE_OAUTH_TOKEN?.trim();
const chatgpt = !!environment.CHATGPT_AUTH_FILE?.trim();
if (!claude && !chatgpt) return undefined;
if (claude && chatgpt)
throw new Error("Choose one model plan for built-in Bots.");
const image = environment.PICKED_HARNESS_IMAGE?.trim() ?? "";
const harness = claude ? "agent-claude-sdk" : "agent-langgraph-agui";
if (
environment.PICKED_HARNESS_SOURCE?.trim() === "byo" ||
!new RegExp(`(?:^|/)(?:openbot-)?${harness}(?::|@|$)`).test(image)
) {
throw new Error(
`The selected plan requires the installed ${harness} harness for built-in Bots.`,
);
}
const address = environment.PICKED_HARNESS_URL?.trim();
const token = environment.MANAGED_AGENT_TOKEN?.trim();
if (!address || !token)
throw new Error(
"The selected plan's owned harness endpoint and token are missing.",
);
const endpoint = new URL(address);
if (
!["http:", "https:"].includes(endpoint.protocol) ||
endpoint.username ||
endpoint.password
) {
throw new Error(
"The selected plan's harness must have an HTTP endpoint without URL credentials.",
);
}
if (claude)
endpoint.pathname = `${endpoint.pathname.replace(/\/$/, "")}/model`;
return { provider: claude ? "claude" : "chatgpt", endpoint, token };
}
const UNKNOWN_USAGE: GenerateResult["usage"] = {
inputTokens: {
total: undefined,
noCache: undefined,
cacheRead: undefined,
cacheWrite: undefined,
},
outputTokens: { total: undefined, text: undefined, reasoning: undefined },
};
export function planMessages(prompt: Options["prompt"]): Message[] {
const messages: Message[] = [];
for (const [index, message] of prompt.entries()) {
const id = `prompt-${index}`;
if (message.role === "system") {
messages.push({ id, role: "system", content: message.content });
} else if (message.role === "user") {
messages.push({
id,
role: "user",
content: message.content.map((part) => {
if (part.type === "text") return { type: "text", text: part.text };
if (!part.mediaType.startsWith("image/"))
throw new Error(
`Plan models cannot receive ${part.mediaType} attachments.`,
);
const data = part.data;
return {
type: "image",
source:
data instanceof URL
? { type: "url", value: data.href, mimeType: part.mediaType }
: {
type: "data",
value:
typeof data === "string"
? data
: Buffer.from(data).toString("base64"),
mimeType: part.mediaType,
},
};
}),
});
} else if (message.role === "assistant") {
const toolCalls = message.content.flatMap((part) =>
part.type === "tool-call"
? [
{
id: part.toolCallId,
type: "function" as const,
function: {
name: part.toolName,
arguments: JSON.stringify(part.input),
},
},
]
: [],
);
messages.push({
id,
role: "assistant",
content: message.content
.flatMap((part) => (part.type === "text" ? [part.text] : []))
.join(""),
...(toolCalls.length ? { toolCalls } : {}),
});
} else {
for (const part of message.content) {
if (part.type !== "tool-result")
throw new Error(
"Plan models do not execute provider approval requests.",
);
const output = part.output;
const content =
output.type === "execution-denied"
? `Tool execution denied: ${output.reason ?? "not approved"}`
: typeof output.value === "string"
? output.value
: JSON.stringify(output.value);
messages.push({
id: `result-${part.toolCallId}`,
role: "tool",
toolCallId: part.toolCallId,
content,
});
}
}
}
return messages;
}
/** Model transport only. BuiltInAgent still validates and executes every server tool. */
export class PlanModel implements PlanLanguageModel {
readonly specificationVersion = "v3";
readonly provider: string;
readonly modelId: string;
readonly supportedUrls = {};
private readonly threadId: string;
private pendingTools: string[] = [];
private streaming = false;
constructor(
private readonly config: PlanModelConfig,
private readonly botId: string,
input: Pick<RunAgentInput, "threadId">,
frontendTools: ReadonlySet<string> = new Set(),
signal?: AbortSignal,
) {
this.provider = `openbot-${config.provider}-plan`;
this.modelId = config.provider;
// Separate SDK/checkpoint state for two Bots in the same channel, stable across tool continuations.
this.threadId = createHash("sha256")
.update(JSON.stringify([botId, input.threadId]))
.digest("hex");
signal?.addEventListener(
"abort",
async () => {
if (
config.provider !== "claude" ||
(!this.streaming && this.pendingTools.length === 0)
)
return;
// A normal frontend-tool boundary must keep the SDK query for the next HTTP
// continuation. Stop, a cancelled server tool, and the step limit must not.
if (
signal.reason === PLAN_RUN_COMPLETED &&
this.pendingTools.some((name) => frontendTools.has(name))
)
return;
try {
const response = await fetch(
`${config.endpoint.href.replace(/\/$/, "")}/cancel`,
{
method: "POST",
redirect: "error",
signal: AbortSignal.timeout(5000),
headers: {
"content-type": "application/json",
"x-openbot-agent-token": config.token,
},
body: JSON.stringify({ threadId: this.threadId }),
},
);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
} catch {
// No request details: the URL and token belong to the deployment. The
// harness's finite query timeout remains its final cleanup boundary.
console.error("The owned Claude model query could not be cancelled.");
}
},
{ once: true },
);
}
async doGenerate(options: Options): Promise<GenerateResult> {
const { stream } = await this.doStream(options);
const content: GenerateResult["content"] = [];
const texts = new Map<string, { type: "text"; text: string }>();
let finish: Extract<Part, { type: "finish" }> | undefined;
for await (const part of stream) {
if (part.type === "text-start") {
const text = { type: "text" as const, text: "" };
texts.set(part.id, text);
content.push(text);
}
if (part.type === "text-delta") {
const text = texts.get(part.id);
if (text) text.text += part.delta;
}
if (part.type === "tool-call") content.push(part);
if (part.type === "finish") finish = part;
if (part.type === "error") throw part.error;
}
if (!finish) throw new Error("The plan model ended without a result.");
return {
content,
finishReason: finish.finishReason,
usage: finish.usage,
warnings: [],
};
}
async doStream(options: Options): Promise<StreamResult> {
this.streaming = true;
this.pendingTools = [];
const messages = planMessages(options.prompt);
const tools = (options.tools ?? []).map((tool) => {
if (tool.type !== "function")
throw new Error(
"Built-in plan Bots only support caller-owned function tools.",
);
return {
name: tool.name,
description: tool.description ?? "",
parameters: tool.inputSchema,
};
});
const names = new Set(tools.map((tool) => tool.name));
const agent = new HttpAgent({
url: this.config.endpoint.href,
headers: { "x-openbot-agent-token": this.config.token },
fetch: (url, init) =>
fetch(url, {
...init,
redirect: "error",
signal: AbortSignal.any([
...(init?.signal ? [init.signal] : []),
...(options.abortSignal ? [options.abortSignal] : []),
]),
}),
});
let subscription: Subscription | undefined;
const stream = new ReadableStream<Part>({
start: (controller) => {
controller.enqueue({ type: "stream-start", warnings: [] });
const calls = new Map<
string,
{ name: string; args: string; ended: boolean }
>();
let terminal = false;
const fail = (error: unknown) => {
if (!terminal) {
terminal = true;
controller.error(error);
subscription?.unsubscribe();
}
};
subscription = agent
.run({
threadId:
this.config.provider === "claude"
? this.threadId
: `${this.threadId}-${crypto.randomUUID()}`,
runId: crypto.randomUUID(),
messages: messages.filter((message) => message.role !== "system"),
tools,
context: messages
.filter((message) => message.role === "system")
.map((message) => ({
description: "Built-in Bot instructions",
value: message.content ?? "",
})),
state: null,
forwardedProps: {
openbotModelOnly: true,
openbotBotId: this.botId,
},
})
.subscribe({
next: (rawEvent) => {
if (terminal) return;
try {
// Python snapshots can carry newer media parts than this JS client.
// The model consumes text/tool deltas, never remote conversation state.
if (
![
"TEXT_MESSAGE_START",
"TEXT_MESSAGE_CONTENT",
"TEXT_MESSAGE_END",
"TOOL_CALL_START",
"TOOL_CALL_ARGS",
"TOOL_CALL_END",
"RUN_ERROR",
"RUN_FINISHED",
].includes(rawEvent.type)
)
return;
const event = EventSchemas.parse(rawEvent);
switch (event.type) {
case "TEXT_MESSAGE_START":
controller.enqueue({
type: "text-start",
id: event.messageId,
});
break;
case "TEXT_MESSAGE_CONTENT":
controller.enqueue({
type: "text-delta",
id: event.messageId,
delta: event.delta,
});
break;
case "TEXT_MESSAGE_END":
controller.enqueue({
type: "text-end",
id: event.messageId,
});
break;
case "TOOL_CALL_START":
if (!names.has(event.toolCallName))
throw new Error(
`The plan harness attempted an unoffered tool: ${event.toolCallName}`,
);
calls.set(event.toolCallId, {
name: event.toolCallName,
args: "",
ended: false,
});
controller.enqueue({
type: "tool-input-start",
id: event.toolCallId,
toolName: event.toolCallName,
});
break;
case "TOOL_CALL_ARGS": {
const call = calls.get(event.toolCallId);
if (!call || call.ended)
throw new Error("Unexpected plan tool arguments.");
call.args += event.delta;
controller.enqueue({
type: "tool-input-delta",
id: event.toolCallId,
delta: event.delta,
});
break;
}
case "TOOL_CALL_END": {
const call = calls.get(event.toolCallId);
if (!call || call.ended)
throw new Error("Unexpected plan tool completion.");
call.ended = true;
controller.enqueue({
type: "tool-input-end",
id: event.toolCallId,
});
controller.enqueue({
type: "tool-call",
toolCallId: event.toolCallId,
toolName: call.name,
input: call.args,
});
break;
}
case "RUN_ERROR": {
const error = new Error(event.message);
if (event.code === "OPENBOT_MODEL_AUTH_REQUIRED") {
error.name = "OpenBotModelAuthenticationError";
recordDesktopConnectionFailure({
connection: "model",
code: "provider_authentication_failed",
});
}
throw error;
}
case "RUN_FINISHED":
if ([...calls.values()].some((call) => !call.ended))
throw new Error(
"The plan harness ended during a tool call.",
);
terminal = true;
this.streaming = false;
this.pendingTools = [...calls.values()].map(
(call) => call.name,
);
clearDesktopConnectionFailure("model");
controller.enqueue({
type: "finish",
finishReason: {
unified: calls.size ? "tool-calls" : "stop",
raw: undefined,
},
usage: UNKNOWN_USAGE,
});
controller.close();
break;
}
} catch (error) {
fail(error);
}
},
error: fail,
complete: () => {
if (!terminal)
fail(
new Error(
"The plan harness disconnected before completing its response.",
),
);
},
});
},
cancel: () => {
subscription?.unsubscribe();
agent.abortRun();
},
});
return { stream };
}
}