Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import {
AbstractAgent,
type BaseEvent,
EventType,
type Message,
type MessagesSnapshotEvent,
Middleware,
type RunAgentInput,
type RunFinishedEvent,
type ToolCall,
} from "@ag-ui/client";
import { map, Observable, type Subscription } from "rxjs";
import type { AcquireLearnedSkills, LearnedSkillInvocation } from "./runtime";
const CATALOG_ID = "openbot:learned-skills";
const CATALOG_CONTEXT = "OpenBot learned skills";
const RESERVED_TOOLS = new Set([
"copilotkit_load_skill",
"copilotkit_read_skill_file",
]);
const MAX_SKILL_STEPS = 10;
/**
* Remote frameworks use their normal AG-UI client-tool boundary for skill reads.
* The snapshot and Intelligence credential remain on this server. A continuation
* calls the same transport with the captured catalog, tool results and state;
* only a new outer invocation acquires a new snapshot.
*/
export class RemoteLearnedSkillsMiddleware extends Middleware {
constructor(
private readonly agentId: string,
private readonly acquire: AcquireLearnedSkills,
) {
super();
}
run(
originalInput: RunAgentInput,
next: AbstractAgent,
): Observable<BaseEvent> {
return new Observable((subscriber) => {
// Snapshot-emitting frameworks can replay our prior system message.
// Remove it even when delivery was disabled after that invocation.
const input = {
...originalInput,
messages: originalInput.messages.filter(
(message) => message.id !== CATALOG_ID,
),
context: originalInput.context.filter(
(entry) => entry.description !== CATALOG_CONTEXT,
),
};
const controller = new AbortController();
let subscription: Subscription | undefined;
let started = false;
const work = async () => {
const invocation = await this.acquire(this.agentId, controller.signal);
if (subscriber.closed) return;
const tools = new Set(invocation?.tools.map((tool) => tool.name) ?? []);
const checkAvailability = skillAvailabilityCheck(tools, input.messages);
if (!invocation) {
await new Promise<void>((resolve, reject) => {
subscription = this.runNext(input, next)
.pipe(
map((event) => {
checkAvailability(event);
return event;
}),
)
.subscribe({
next: (event) => subscriber.next(event),
error: reject,
complete: resolve,
});
});
return;
}
if (input.tools.some((tool) => RESERVED_TOOLS.has(tool.name))) {
throw new Error("Automatic Learning tool names are reserved.");
}
let request: RunAgentInput = {
...input,
messages: [
...(invocation.catalog
? [
{
id: CATALOG_ID,
role: "system" as const,
content: invocation.catalog,
},
]
: []),
...input.messages.filter((message) => message.id !== CATALOG_ID),
],
context: [
...input.context.filter(
(entry) => entry.description !== CATALOG_CONTEXT,
),
...(invocation.catalog
? [{ description: CATALOG_CONTEXT, value: invocation.catalog }]
: []),
],
tools: [...input.tools, ...invocation.tools],
};
for (let step = 0; step <= MAX_SKILL_STEPS; step += 1) {
if (subscriber.closed) return;
const emitted = new Set<string>();
const previousCalls = new Set(
request.messages.flatMap((message) =>
message.role === "assistant"
? (message.toolCalls ?? []).map((call) => call.id)
: [],
),
);
let messages: Message[] = request.messages;
let state: unknown = request.state;
let terminal: RunFinishedEvent | undefined;
let failed = false;
await new Promise<void>((resolve, reject) => {
subscription = this.runNextWithState(
request,
new InvocationTransport(next, request),
)
.pipe(
map((current) => {
checkAvailability(current.event);
return current;
}),
)
.subscribe({
next: ({
event,
messages: currentMessages,
state: currentState,
}) => {
messages = currentMessages;
state = currentState;
// CrewAI publishes complete calls in message snapshots instead
// of individual tool events. Only newly produced calls belong
// to this segment; never replay a historical unanswered call.
if (event.type === EventType.MESSAGES_SNAPSHOT) {
for (const message of currentMessages) {
if (message.role !== "assistant") continue;
for (const call of message.toolCalls ?? []) {
if (!previousCalls.has(call.id)) emitted.add(call.id);
}
}
}
if (event.type === EventType.TOOL_CALL_START) {
emitted.add(String(event.toolCallId));
}
if (event.type === EventType.RUN_FINISHED) {
terminal = event as RunFinishedEvent;
return;
}
if (event.type === EventType.RUN_STARTED) {
if (started) return;
started = true;
}
if (event.type === EventType.RUN_ERROR) failed = true;
subscriber.next(event);
},
error: reject,
complete: resolve,
});
});
if (subscriber.closed || failed) return;
if (!terminal)
throw new Error("Remote agent ended without a terminal event.");
const answered = new Set(
messages.flatMap((message) =>
message.role === "tool" ? [message.toolCallId] : [],
),
);
const pending: ToolCall[] = messages.flatMap((message) =>
message.role === "assistant"
? (message.toolCalls ?? []).filter(
(call) => emitted.has(call.id) && !answered.has(call.id),
)
: [],
);
const reads = pending.filter((call) => tools.has(call.function.name));
if (reads.length === 0) {
subscriber.next(terminal);
return;
}
if (step === MAX_SKILL_STEPS)
throw new Error("Automatic Learning tool step limit reached.");
const results = new Map<string, string>();
for (const call of reads) {
if (subscriber.closed) return;
const content = await answerRead(invocation, call);
if (subscriber.closed) return;
const message: Message = {
id: crypto.randomUUID(),
role: "tool",
toolCallId: call.id,
content,
};
messages = [...messages, message];
results.set(call.id, content);
subscriber.next({
type: EventType.TOOL_CALL_RESULT,
messageId: message.id,
toolCallId: call.id,
role: "tool",
content,
});
}
const interrupts =
terminal.outcome?.type === "interrupt"
? terminal.outcome.interrupts
: [];
const unansweredInterrupts = interrupts.filter(
(interrupt) => !results.has(interrupt.toolCallId ?? interrupt.id),
);
// A UI or governed tool still belongs to its existing executor. Return
// the boundary with our reads answered, without inventing its result.
if (
reads.length !== pending.length ||
unansweredInterrupts.length > 0
) {
subscriber.next(
unansweredInterrupts.length > 0
? {
...terminal,
outcome: {
type: "interrupt",
interrupts: unansweredInterrupts,
},
}
: terminal,
);
return;
}
request = {
...request,
messages,
state,
resume:
interrupts.length > 0
? interrupts.map((interrupt) => ({
interruptId: interrupt.id,
status: "resolved" as const,
payload: results.get(interrupt.toolCallId ?? interrupt.id),
}))
: undefined,
};
}
};
void work().then(
() => {
if (!subscriber.closed) subscriber.complete();
},
(error: unknown) => {
if (subscriber.closed) return;
subscriber.next({
type: EventType.RUN_ERROR,
code: "OPENBOT_LEARNED_SKILLS_ERROR",
message:
error instanceof Error
? error.message
: "Automatic Learning delivery failed.",
});
subscriber.complete();
},
);
return () => {
controller.abort();
subscription?.unsubscribe();
};
});
}
}
/**
* One skill read's result, including a read the snapshot cannot answer.
*
* A model asks for a skill by a name the catalog does not hold, or for a file the skill does not
* list, or sends arguments that are not JSON. A built-in Bot's loop (the AI SDK) hands the thrown
* message back to the model as that call's result and runs the next step, so the model reads
* "Skill is unavailable." and carries on. The reason goes back the same way here, rather than
* ending a remote Bot's whole turn. The snapshot is already in memory, so nothing but the call's
* own input can make a read fail.
*/
async function answerRead(
invocation: LearnedSkillInvocation,
call: ToolCall,
): Promise<string> {
try {
return await invocation.execute(
call.function.name,
JSON.parse(call.function.arguments || "{}"),
);
} catch (error) {
return error instanceof Error && error.message
? error.message
: "That learned Skill could not be read.";
}
}
/** AG-UI accumulates from the agent's initial history, not input.messages. */
class InvocationTransport extends AbstractAgent {
constructor(
private readonly transport: AbstractAgent,
input: RunAgentInput,
) {
super({ initialMessages: input.messages, initialState: input.state });
}
run(input: RunAgentInput) {
return this.transport.run(input);
}
}
/** Retained native queries can still call a tool removed between UI segments. */
function skillAvailabilityCheck(offered: Set<string>, history: Message[]) {
const answered = new Set(
history.flatMap((message) =>
message.role === "tool" ? [message.toolCallId] : [],
),
);
const check = (name: string, id: string) => {
if (RESERVED_TOOLS.has(name) && !offered.has(name) && !answered.has(id)) {
throw new Error(
"Automatic Learning is unavailable for this run. Start a new message to continue without learned skills.",
);
}
};
return (event: BaseEvent) => {
if (event.type === EventType.TOOL_CALL_RESULT)
answered.add(String(event.toolCallId));
if (event.type === EventType.TOOL_CALL_START) {
check(String(event.toolCallName), String(event.toolCallId));
}
if (event.type === EventType.MESSAGES_SNAPSHOT) {
const messages = (event as MessagesSnapshotEvent).messages;
for (const message of messages) {
if (message.role === "tool") answered.add(message.toolCallId);
}
for (const message of messages) {
if (message.role !== "assistant") continue;
for (const call of message.toolCalls ?? [])
check(call.function.name, call.id);
}
}
};
}