Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
import { GlobalRegistrator } from "@happy-dom/global-registrator";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import { startEventCapture } from "../src/content/collector";
import {
type Box,
chooseSpot,
coveredIn,
INDICATOR_TAG,
type IndicatorView,
type MarkSpot,
showIndicator,
spotBox,
} from "../src/content/indicator";
import { createSensitivityTracker } from "../src/content/masking-context";
import { NOTICE_TOKENS } from "../src/content/notice";
/**
* The recording mark a followed tab shows from its first step (content/indicator.ts): what it says, the one click that
* leaves the site out of every recording or records it again, and that a page cannot press it for the person.
*/
beforeAll(() =>
GlobalRegistrator.register({ url: "https://mail.example.test/inbox" }),
);
afterAll(() => GlobalRegistrator.unregister());
let shown: { remove(): void } | undefined;
afterEach(() => {
shown?.remove();
shown = undefined;
document.body.innerHTML = "";
});
const RECORDING: IndicatorView = {
site: "example.test",
excluded: false,
narrating: false,
};
function show(
view: IndicatorView,
choose: (recorded: boolean) => void = () => {},
trusted = true,
) {
const mark = showIndicator({
view,
choose,
...(trusted ? { isTrusted: () => true } : {}),
});
shown = mark;
return mark;
}
/** Inside a shown mark: its closed shadow root, which only the content script that showed it holds. */
type Shown = { root: ShadowRoot };
/** What the mark shows, part by part, as a person reads it. */
const text = (mark: Shown) =>
[...mark.root.querySelectorAll(".label, .voice, button")]
.filter((part) => !(part as HTMLElement).hidden)
.map((part) => part.textContent)
.join(" ");
const buttonOf = (mark: Shown) =>
mark.root.querySelector("button") as HTMLButtonElement;
/** happy-dom cannot make trusted events; the collector listens only for them, so these are marked trusted by hand. */
const trustedClick = () => {
const event = new MouseEvent("click", {
bubbles: true,
composed: true,
button: 0,
});
Object.defineProperty(event, "isTrusted", { value: true });
return event;
};
test("a recorded tab says OpenBot is recording it, with one button that leaves its site out", () => {
const choices: boolean[] = [];
const mark = show(RECORDING, (recorded) => choices.push(recorded));
expect(document.querySelector(INDICATOR_TAG)).toBe(mark.host);
expect(text(mark)).toBe(
"OpenBot is recording this tab Don't record example.test",
);
expect(mark.root.querySelector("[role=status]")?.textContent).toContain(
"OpenBot is recording this tab",
);
buttonOf(mark).click();
expect(choices).toEqual([false]);
});
test("while the Recorder hears the person, the mark says so", () => {
const mark = show({ ...RECORDING, narrating: true });
expect(text(mark)).toBe(
"OpenBot is recording this tab · Listening to you Don't record example.test",
);
mark.update({ ...RECORDING, narrating: false });
expect(text(mark)).toBe(
"OpenBot is recording this tab Don't record example.test",
);
});
test("a site left out says it is not recorded, and its button records it again", () => {
const choices: boolean[] = [];
const mark = show({ site: "example.test", excluded: true }, (recorded) =>
choices.push(recorded),
);
expect(text(mark)).toBe("Not recording example.test Record it");
buttonOf(mark).click();
expect(choices).toEqual([true]);
// The same mark turns back when the background says the site is recorded again.
mark.update(RECORDING);
expect(text(mark)).toBe(
"OpenBot is recording this tab Don't record example.test",
);
buttonOf(mark).click();
expect(choices).toEqual([true, false]);
});
test("a page script clicking the mark does nothing", () => {
const choices: boolean[] = [];
const mark = show(RECORDING, (recorded) => choices.push(recorded), false);
buttonOf(mark).click();
expect(choices).toEqual([]);
});
test("a click on the mark is never one of the person's steps", () => {
document.body.innerHTML = `<button id="b">Send</button>`;
const mark = show(RECORDING);
expect(mark.host.hasAttribute("data-copilotkit-ignore")).toBe(true);
const tracker = createSensitivityTracker({ isTrusted: () => true });
const uninstall = tracker.install();
const sent: CapturedEvent[] = [];
const capture = startEventCapture({
tracker,
send: (event) => sent.push(event),
});
try {
buttonOf(mark).dispatchEvent(trustedClick());
expect(sent.filter((event) => event.name === "click")).toEqual([]);
(document.getElementById("b") as HTMLElement).dispatchEvent(trustedClick());
expect(sent.filter((event) => event.name === "click")).toHaveLength(1);
} finally {
capture.stop();
uninstall();
}
});
test("the mark sits above the page, in OpenBot's colours, and removes itself", () => {
const mark = show(RECORDING);
const css = mark.root.querySelector("style")?.textContent ?? "";
expect(css).toContain("2147483647");
expect(css).toContain("position: fixed");
expect(css).toContain("prefers-color-scheme: dark");
expect(css).toContain("prefers-reduced-motion");
for (const value of [
...Object.values(NOTICE_TOKENS.light),
...Object.values(NOTICE_TOKENS.dark),
])
expect(css).toContain(value);
mark.remove();
expect(document.querySelector(INDICATOR_TAG)).toBeNull();
});
// ---- Where the mark hangs ------------------------------------------------------------------------------------------
//
// On OpenBot's own tab the mark, when it hung bottom-left, covered the sidebar's account row. What a person can use along the edges of OpenBot's home page, measured in Chromium 151 (headless, 2026-10-27,
// the self-host bar shown, as a new install shows it), at the sizes of a laptop and a desktop screen.
type Usable = { what: string; tag: "a" | "button" | "input"; box: Box };
const OPENBOT_HOME: Record<
string,
{ page: { width: number; height: number }; usable: Usable[] }
> = {
laptop: {
page: { width: 1280, height: 860 },
usable: [
{
what: "OpenBot",
tag: "a",
box: { left: 8, top: 8, width: 285, height: 34 },
},
{
what: "New conversation",
tag: "a",
box: { left: 299, top: 8, width: 32, height: 32 },
},
{
what: "Search conversations",
tag: "input",
box: { left: 33, top: 56, width: 297, height: 36 },
},
{
what: "Bots",
tag: "a",
box: { left: 8, top: 648, width: 323, height: 40 },
},
{
what: "Capabilities",
tag: "a",
box: { left: 8, top: 689, width: 323, height: 40 },
},
{
what: "Connections",
tag: "a",
box: { left: 8, top: 730, width: 323, height: 40 },
},
{
what: "Browser",
tag: "button",
box: { left: 8, top: 771, width: 323, height: 40 },
},
{
what: "account row",
tag: "button",
box: { left: 8, top: 812, width: 323, height: 40 },
},
{
what: "Toggle Sidebar",
tag: "button",
box: { left: 331, top: 0, width: 16, height: 860 },
},
{
what: "Talk to an engineer",
tag: "a",
box: { left: 611, top: 6, width: 118, height: 28 },
},
{
what: "Dismiss",
tag: "button",
box: { left: 737, top: 8, width: 24, height: 24 },
},
{
what: "Get a license",
tag: "a",
box: { left: 672, top: 824, width: 81, height: 20 },
},
],
},
desktop: {
page: { width: 1920, height: 1080 },
usable: [
{
what: "OpenBot",
tag: "a",
box: { left: 8, top: 8, width: 285, height: 34 },
},
{
what: "New conversation",
tag: "a",
box: { left: 299, top: 8, width: 32, height: 32 },
},
{
what: "account row",
tag: "button",
box: { left: 8, top: 1032, width: 323, height: 40 },
},
{
what: "Toggle Sidebar",
tag: "button",
box: { left: 331, top: 0, width: 16, height: 1080 },
},
{
what: "Talk to an engineer",
tag: "a",
box: { left: 611, top: 6, width: 118, height: 28 },
},
{
what: "Dismiss",
tag: "button",
box: { left: 737, top: 8, width: 24, height: 24 },
},
{
what: "Get a license",
tag: "a",
box: { left: 992, top: 1044, width: 81, height: 20 },
},
],
},
};
/** The mark on OpenBot's own tab, "OpenBot is recording this tab | Don't record 127.0.0.1", as Chromium drew it. */
const MARK_SIZE = { width: 361, height: 28 };
const inside = (box: Box, x: number, y: number) =>
x >= box.left &&
x < box.left + box.width &&
y >= box.top &&
y < box.top + box.height;
const overlaps = (a: Box, b: Box) =>
a.left < b.left + b.width &&
b.left < a.left + a.width &&
a.top < b.top + b.height &&
b.top < a.top + a.height;
/** The page as the browser's hit test answers it: the element drawn at a point, the smallest box there. */
function pageOf(usable: Usable[]) {
const elements = usable.map((thing) => {
const element = document.createElement(thing.tag);
if (thing.tag === "a") element.setAttribute("href", "#");
return { ...thing, element };
});
const background = document.createElement("main");
return (x: number, y: number): Element =>
elements
.filter((thing) => inside(thing.box, x, y))
.sort(
(a, b) => a.box.width * a.box.height - b.box.width * b.box.height,
)[0]?.element ?? background;
}
test("on OpenBot's own page the mark hangs where it covers nothing the person can use: not the account row, not the self-host bar's link or close button", () => {
for (const [size, { page, usable }] of Object.entries(OPENBOT_HOME)) {
const under = pageOf(usable);
const spot = chooseSpot((candidate) =>
coveredIn(spotBox(candidate, MARK_SIZE, page), under),
);
const box = spotBox(spot, MARK_SIZE, page);
expect({
size,
covered: usable
.filter((thing) => overlaps(box, thing.box))
.map((thing) => thing.what),
}).toEqual({
size,
covered: [],
});
expect(box.top).toBe(0);
expect(box.left).toBeGreaterThanOrEqual(16);
expect(box.left + box.width).toBeLessThanOrEqual(page.width - 16);
}
// On a laptop's screen the middle of the top edge is the self-host bar's link and close button, so it hangs right of
// the middle; on a wider screen the middle is clear.
const laptop = OPENBOT_HOME.laptop as (typeof OPENBOT_HOME)["laptop"];
expect(
chooseSpot((candidate) =>
coveredIn(
spotBox(candidate, MARK_SIZE, laptop.page),
pageOf(laptop.usable),
),
),
).toEqual({ edge: "top", at: 0.75 });
const desktop = OPENBOT_HOME.desktop as (typeof OPENBOT_HOME)["desktop"];
expect(
chooseSpot((candidate) =>
coveredIn(
spotBox(candidate, MARK_SIZE, desktop.page),
pageOf(desktop.usable),
),
),
).toEqual({ edge: "top", at: 0.5 });
});
test("the mark moves only when something usable comes under it and a spot is clear, and never back for nothing", () => {
const page = { width: 1280, height: 860 };
const bar: Usable = {
what: "banner link",
tag: "a",
box: { left: 560, top: 0, width: 160, height: 30 },
};
const clear = pageOf([]);
const covers =
(under: (x: number, y: number) => Element) => (spot: MarkSpot) =>
coveredIn(spotBox(spot, MARK_SIZE, page), under);
let spot = chooseSpot(covers(clear));
expect(spot).toEqual({ edge: "top", at: 0.5 });
// A banner draws late, its link under the mark: it moves to the first clear spot.
spot = chooseSpot(covers(pageOf([bar])), spot);
expect(spot).toEqual({ edge: "top", at: 0.75 });
// The banner goes: it stays where it is, which covers nothing.
spot = chooseSpot(covers(clear), spot);
expect(spot).toEqual({ edge: "top", at: 0.75 });
// Everything along both edges is usable: it goes where it covers least, and stays there on a tie.
const crowded = pageOf([
{
what: "top bar",
tag: "button",
box: { left: 0, top: 0, width: 1280, height: 40 },
},
{
what: "bottom bar",
tag: "button",
box: { left: 0, top: 820, width: 1280, height: 40 },
},
]);
expect(chooseSpot(covers(crowded), spot)).toEqual({ edge: "top", at: 0.75 });
});
test("the shown mark takes the spot the page leaves clear, hangs from that edge, and holds still under the pointer", () => {
const rect = HTMLElement.prototype.getBoundingClientRect;
HTMLElement.prototype.getBoundingClientRect = function (this: HTMLElement) {
return this.className === "mark"
? ({
...MARK_SIZE,
x: 0,
y: 0,
left: 0,
top: 0,
right: MARK_SIZE.width,
bottom: MARK_SIZE.height,
} as DOMRect)
: rect.call(this);
};
const width = document.documentElement.clientWidth || window.innerWidth;
const height = document.documentElement.clientHeight || window.innerHeight;
// The whole top edge is a toolbar: the mark rises from the middle of the bottom edge.
let usable: Usable[] = [
{
what: "toolbar",
tag: "button",
box: { left: 0, top: 0, width, height: 40 },
},
];
const looks: (() => void)[] = [];
try {
const mark = showIndicator({
view: RECORDING,
choose: () => {},
isTrusted: () => true,
under: (x, y) => pageOf(usable)(x, y),
timers: { set: (fn) => looks.push(fn), clear: () => {} },
});
shown = mark;
const inside = mark.root.querySelector<HTMLElement>(".mark");
const edge = () => inside?.dataset.edge;
expect(edge()).toBe("bottom");
expect(inside?.style.getPropertyValue("top")).toBe(
`${height - MARK_SIZE.height}px`,
);
expect(inside?.style.getPropertyValue("left")).toBe(
`${width / 2 - MARK_SIZE.width / 2}px`,
);
// The toolbar goes away while the person points at the mark: it does not move from under them.
usable = [];
mark.host.dispatchEvent(new Event("pointerenter"));
for (const look of looks) look();
expect(edge()).toBe("bottom");
// Once they move away and a cookie banner comes up under it, it hangs from the top edge again.
mark.host.dispatchEvent(new Event("pointerleave"));
usable = [
{
what: "cookie banner",
tag: "button",
box: { left: 0, top: height - 60, width, height: 60 },
},
];
for (const look of looks) look();
expect(edge()).toBe("top");
expect(inside?.style.getPropertyValue("top")).toBe("0px");
} finally {
HTMLElement.prototype.getBoundingClientRect = rect;
}
});
/*
* The mark is the sign that a tab is being recorded, and it lives in the page's DOM: the page could hide it with one
* CSS rule, or remove it, and nothing put it back. Its shadow root was open, so a page script could reach inside too.
*/
test("a page cannot reach inside the mark: its shadow root is closed to the page", () => {
const mark = show(RECORDING);
expect(mark.host.shadowRoot).toBeNull();
expect(mark.root.querySelector("button")).not.toBeNull();
});
test("what makes the mark seen is set on its host from inside, important, so no page rule or inline style outranks it", () => {
// A shadow tree's important :host rules outrank the page's important rules and an inline style on the host (CSS
// Cascade 5, context); measured in Chromium, which happy-dom does not cascade, so this pins the rule's words.
const mark = show(RECORDING);
const css = mark.root.querySelector("style")?.textContent ?? "";
const host = /:host \{([^}]*)\}/.exec(css)?.[1] ?? "";
for (const declaration of [
"display: block !important",
"position: fixed !important",
"visibility: visible !important",
"opacity: 1 !important",
"transform: none !important",
"filter: none !important",
"clip-path: none !important",
"pointer-events: auto !important",
"z-index: 2147483647 !important",
])
expect(host).toContain(declaration);
// Where it hangs is set inside, where the page cannot reach, never on the host.
expect(mark.host.getAttribute("style")).toBeNull();
});
test("a page that removes the mark, or hides it with hidden or inert, has it put back at once", async () => {
const mark = show(RECORDING);
mark.host.remove();
await Promise.resolve();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(mark.host.isConnected).toBe(true);
expect(document.documentElement.lastElementChild).toBe(mark.host);
mark.host.setAttribute("hidden", "");
mark.host.setAttribute("inert", "");
await new Promise((resolve) => setTimeout(resolve, 0));
expect(mark.host.hasAttribute("hidden")).toBe(false);
expect(mark.host.hasAttribute("inert")).toBe(false);
// Something the page adds after it, which would be drawn over it at the same z-index: the mark goes last again.
const cover = document.createElement("div");
document.documentElement.append(cover);
await new Promise((resolve) => setTimeout(resolve, 0));
expect(document.documentElement.lastElementChild).toBe(mark.host);
cover.remove();
// Taken away by the content script itself, it stays away.
mark.remove();
shown = undefined;
await new Promise((resolve) => setTimeout(resolve, 0));
expect(document.querySelector(INDICATOR_TAG)).toBeNull();
});
test("a script that also wants to be last trades places with the mark a few times a look, not as fast as the two can", async () => {
const mark = show(RECORDING);
const rival = document.createElement("div");
document.documentElement.append(rival);
const rivalWatch = new MutationObserver(() => {
if (document.documentElement.lastElementChild !== rival)
document.documentElement.append(rival);
});
rivalWatch.observe(document.documentElement, { childList: true });
try {
await new Promise((resolve) => setTimeout(resolve, 50));
// Both still on the page, and the trading has stopped: no mutation is left to come.
expect(mark.host.isConnected).toBe(true);
expect(rival.isConnected).toBe(true);
const settled = document.documentElement.lastElementChild;
await new Promise((resolve) => setTimeout(resolve, 50));
expect(document.documentElement.lastElementChild).toBe(settled);
} finally {
rivalWatch.disconnect();
rival.remove();
}
});