Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { NOT_SHOWN, NOT_SHOWN_TAIL } from "@openbot/shared/component-markers";
import type { HeadlessTool } from "@openbot/toolkit/headless-tools";
import type { ComponentCatalog } from "@openbot/toolkit/ports";
import { z } from "zod";
import type { AuditInitiator, AuditSink } from "../audit";
import type { ComponentStore } from "./store";
/**
* Gallery components a headless turn can answer with, when no browser is attached to draw them.
*
* In a person's own turn a component is a frontend tool: the browser registers it, asks the server
* whether this Bot may use it, and draws it. A routine or responsibility turn has no browser, so the
* same call would go unanswered. Here the server answers it instead, under the SAME governance: the
* published description, the per-Bot withholding and the data-function grants in `store.ts`, decided
* again at call time. The call and its result reach the Intelligence thread as ordinary AG-UI tool
* events, so opening the conversation later draws the real component with the renderer the app
* registers under the same name. Nothing here draws anything.
*
* THE PARAMETER SCHEMAS ARE A MIRROR of the ones beside each React chart and card in
* `app/src/components/gallery/`. The server image does not ship the app's sources, and a shared
* module cannot import zod under the isolated install, so the schema is declared twice and
* `server/tests/headless-components.test.ts` fails the moment the two stop producing the same JSON
* Schema. Decisions (`askApproval`, `askChoice`, `askForm`) are deliberately absent: they suspend
* the run for a person's answer, which a headless turn reaches through `ask_person`, not by drawing a
* card nobody is looking at.
*/
const point = z.object({
label: z
.string()
.describe("What this point is called, e.g. a month or a team"),
value: z.number().describe("Its value"),
});
/** Where the figures came from. The same fields, and the same reason, as the app's charts. */
const provenance = {
basis: z
.enum(["read", "given", "own-knowledge"])
.describe(
"Where the figures come from. read: you read them with a tool in this conversation, and sources lists what you read. given: the person supplied them, in the conversation or a file. own-knowledge: you recalled them without looking them up; the chart then says they are unchecked. For figures that change over time, look them up before charting if you can",
),
sources: z
.array(
z.object({
title: z
.string()
.describe(
"The page, report or dataset, named the way a person would",
),
url: z.string().optional().describe("Its address, when it has one"),
}),
)
.optional()
.describe("What you read the figures from. Required when basis is read"),
asOf: z
.string()
.optional()
.describe(
"When the figures are from, e.g. 'Q3 2026' or '24 Oct 2026'. Leave out for figures that do not date",
),
};
const common = {
title: z.string().describe("A short title for the chart"),
caption: z
.string()
.optional()
.describe(
"One line under the title saying what the reader should take from it",
),
...provenance,
};
const BarChartProps = z.object({
...common,
points: z
.array(point)
.describe("One bar per point, in the order they should appear"),
});
const PieChartProps = z.object({
...common,
points: z
.array(point)
.describe("One slice per point. Values are summed to make the whole"),
});
const seriesSchema = z.object({
name: z.string().describe("What this line is called"),
values: z
.array(z.number())
.describe("One value per label, in the same order"),
});
const LineChartProps = z.object({
...common,
labels: z.array(z.string()).describe("The x axis, e.g. months"),
series: z.array(seriesSchema).describe("One line per series"),
});
const ProgressChartProps = z.object({
...common,
points: z
.array(
point.extend({
target: z.number().describe("What the value is measured against"),
}),
)
.describe("One row per thing being tracked"),
});
const tone = z
.enum(["neutral", "positive", "caution", "negative"])
.describe(
"How this reads at a glance. Use negative and caution sparingly, for a refusal, a breach or a failure, not for anything merely notable",
);
const RecordCardProps = z.object({
title: z.string().describe("What this record is, e.g. a person or an order"),
subtitle: z
.string()
.optional()
.describe("One line of context under the title"),
status: z.string().optional().describe("A short status word, e.g. Approved"),
statusTone: tone.optional(),
fields: z
.array(
z.object({
label: z.string(),
value: z.string().describe("Already formatted for a person to read"),
}),
)
.describe("The fields, in the order they should be read"),
});
const MetricsCardProps = z.object({
title: z.string().describe("What these figures are about"),
caption: z.string().optional(),
metrics: z
.array(
z.object({
label: z.string(),
value: z
.string()
.describe("Already formatted, including any unit or currency"),
change: z
.string()
.optional()
.describe("The movement, e.g. '+12% on last month'"),
changeTone: tone.optional(),
}),
)
.max(6)
.describe("Up to six figures. More than that wanted a table"),
});
const ChecklistCardProps = z.object({
title: z.string().describe("What this list is"),
caption: z.string().optional(),
items: z
.array(
z.object({
text: z.string(),
done: z.boolean().describe("Whether this one is already finished"),
note: z
.string()
.optional()
.describe("A short aside, e.g. who it is waiting on"),
}),
)
.describe("The items, in the order they should be done"),
});
const NoticeCardProps = z.object({
title: z.string().describe("The headline, in a few words"),
body: z.string().describe("The explanation, in one or two sentences"),
tone: tone.optional(),
points: z
.array(z.string())
.optional()
.describe("Supporting points, if there are any"),
});
const DataTableProps = z
.object({
title: z.string().trim().min(1).max(120),
caption: z.string().max(500).optional(),
columns: z
.array(z.string().trim().min(1).max(80))
.min(1)
.max(8)
.describe("Unique column labels, in display order"),
rows: z
.array(
z.object({
cells: z
.array(z.union([z.string().max(2000), z.number()]))
.min(1)
.max(8),
}),
)
.max(100)
.describe(
"One cell per column. Use numbers for numeric sorting; strings for formatted values.",
),
})
.refine(
(table) =>
new Set(table.columns).size === table.columns.length &&
table.rows.every((row) => row.cells.length === table.columns.length),
"Column labels must be unique and every row must have one cell per column",
);
const QuoteCardProps = z.object({
quote: z
.string()
.describe("The quotation itself, without surrounding quote marks"),
attribution: z
.string()
.describe(
"Who said or wrote it, e.g. 'Grace Hopper' or 'the 2026 annual report'",
),
context: z
.string()
.optional()
.describe(
"One short line of context: where it is from, or why it matters here",
),
});
const ActivityReportProps = z.object({
report: z
.enum(["activity", "refusals"])
.describe(
"Which report to show: 'activity' for how much each Bot has done, 'refusals' for what this deployment recently refused",
),
title: z
.string()
.optional()
.describe("A heading for the report, in a few words"),
days: z
.number()
.optional()
.describe(
"For the activity report: how many days back to count. Defaults to 7",
),
});
/**
* Which server-side function each activity report reads. The same map `activity.tsx` holds, and a
* Map there for the same reason: the key is a string somebody else chose, and an object lookup
* answers a borrowed `Object.prototype` member with something truthy that is no function name.
*/
const ACTIVITY_FUNCTION_FOR: ReadonlyMap<string, string> = new Map([
["activity", "botActivity"],
["refusals", "recentRefusals"],
]);
export type HeadlessComponentSpec = {
/** The tool name, the catalogue key, and the renderer's key in the app. */
name: string;
title: string;
parameters: z.ZodType<Record<string, unknown>>;
/** What the model is told once the call is recorded. Nobody is watching, so it says so. */
confirmation: string;
/** The data functions the component reads when drawn, decided before the call is recorded. */
reads?: (args: Record<string, unknown>) => readonly string[];
};
const saved = (what: string) =>
`The ${what} is saved in this conversation and is drawn when the person opens it.`;
/** Every display component a headless turn can record. Keyed by the same names the app registers. */
export const HEADLESS_COMPONENTS: readonly HeadlessComponentSpec[] = [
{
name: "showAreaChart",
title: "Area chart",
parameters: LineChartProps,
confirmation: saved("area chart"),
},
{
name: "showBarChart",
title: "Bar chart",
parameters: BarChartProps,
confirmation: saved("bar chart"),
},
{
name: "showLineChart",
title: "Line chart",
parameters: LineChartProps,
confirmation: saved("line chart"),
},
{
name: "showPieChart",
title: "Donut chart",
parameters: PieChartProps,
confirmation: saved("donut chart"),
},
{
name: "showProgress",
title: "Progress against target",
parameters: ProgressChartProps,
confirmation: saved("progress chart"),
},
{
name: "showRecord",
title: "Record",
parameters: RecordCardProps,
confirmation: saved("record"),
},
{
name: "showMetrics",
title: "Headline figures",
parameters: MetricsCardProps,
confirmation:
"The figures are saved in this conversation and are drawn when the person opens it.",
},
{
name: "showChecklist",
title: "Checklist",
parameters: ChecklistCardProps,
confirmation: saved("checklist"),
},
{
name: "showNotice",
title: "Notice",
parameters: NoticeCardProps,
confirmation: saved("notice"),
},
{
name: "showTable",
title: "Data table",
parameters: DataTableProps as z.ZodType<Record<string, unknown>>,
confirmation: saved("sortable table"),
},
{
name: "showQuote",
title: "Quotation",
parameters: QuoteCardProps,
confirmation: saved("quotation"),
},
{
name: "showActivityReport",
title: "Activity report",
parameters: ActivityReportProps,
confirmation:
"The report is saved in this conversation. Its figures are read from this deployment when the person opens it; you were not given them.",
reads: (args) => {
const functionName =
typeof args.report === "string"
? ACTIVITY_FUNCTION_FOR.get(args.report)
: undefined;
return functionName ? [functionName] : [];
},
},
];
/** The same sentence the browser answers a refused call with, so transcript and card agree. */
export function notShown(title: string, reason: string): string {
return `${NOT_SHOWN}${title}. ${reason}${NOT_SHOWN_TAIL}`;
}
/**
* The governed gallery as a headless turn sees it: every component's name, and the tools for one
* Bot, decided by the component store and refused into the audit trail.
*/
export function headlessComponentCatalog(options: {
store: ComponentStore;
audit?: AuditSink;
}): ComponentCatalog {
const { store, audit } = options;
return {
names: new Set(HEADLESS_COMPONENTS.map(({ name }) => name)),
toolsForTurn: ({ agentId, ownerUserId, initiator }) =>
createHeadlessComponentTools({
store,
botId: agentId,
ownerUserId,
initiator,
...(audit ? { audit } : {}),
}),
};
}
/**
* The component tools for one headless turn, bound to the Bot and its owner outside the arguments.
*
* Offered exactly what the browser would offer this Bot: published, not withheld, with the
* published description the model reads. A component this build mirrors but the Bot does not hold is
* still registered, hidden, so a call the model makes anyway (a stale tool list, a remote agent with
* its own) is answered with a governed refusal instead of being left unanswered.
*/
export async function createHeadlessComponentTools(options: {
store: ComponentStore;
botId: string;
ownerUserId: string;
initiator: AuditInitiator;
audit?: AuditSink;
specs?: readonly HeadlessComponentSpec[];
}): Promise<HeadlessTool[]> {
const {
store,
botId,
ownerUserId,
initiator,
audit,
specs = HEADLESS_COMPONENTS,
} = options;
const held = new Map(
(await store.listForAgent(botId)).map((entry) => [
entry.name,
entry.description,
]),
);
const refuse = async (
spec: HeadlessComponentSpec,
reason: string,
extra: Record<string, unknown> = {},
) => {
if (audit)
await audit.record({
eventType: extra.function
? "component.function_refused"
: "component.refused",
targetType: "component",
targetId: spec.name,
actorUserId: ownerUserId,
initiator,
payload: { bot: botId, reason, headless: true, ...extra },
});
return notShown(spec.title, reason);
};
return specs.map((spec) => {
const description = held.get(spec.name);
return {
definition: {
name: spec.name,
description: description ?? spec.title,
parameters: z.toJSONSchema(spec.parameters) as Record<string, unknown>,
},
...(description === undefined ? { hidden: true } : {}),
async execute(args) {
const parsed = spec.parameters.safeParse(args);
if (!parsed.success)
return refuse(
spec,
`The ${spec.title.toLowerCase()} was called with data that does not match its declared shape: ${z.prettifyError(parsed.error)}.`,
{ invalid: true },
);
// Asked again at call time: the offered list is a snapshot from when the turn started.
const decision = await store.decide(spec.name, botId);
if (!decision.allowed) return refuse(spec, decision.reason);
for (const functionName of spec.reads?.(parsed.data) ?? []) {
if (await store.mayCall(spec.name, functionName)) continue;
return refuse(
spec,
`${spec.name} can't use the function ${functionName}. An admin has to give it to this component.`,
{ function: functionName },
);
}
return spec.confirmation;
},
};
});
}