Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Show the Bot's screen when the Computer panel opens, and draw one Connect card per turn (#826)
Guido Vizoso636c6fe14c3b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bed2c0e3..9de5ea9f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -56,6 +56,18 @@ Composio Airtable", then "Airtable list bases". It now says "Used Airtable", andthe action, "List bases", with the app beside it. An Arcade app is named the same way, as the apprather than `arcade-Gmail`.+### An app a Bot cannot use yet asks once per turn++When a Bot's call to an unconnected app was refused, the Bot often also asked to show that app's+card, and the conversation drew the same Connect card twice. Each app's card is now drawn once per+turn; asking again in a new message can draw it again.++### The Computer panel shows the Bot's screen when it opens++Opening the Computer panel beside a conversation could leave it on "Waiting for the Bot's screen…"+while the Bot browsed, because the panel slid in from off-screen and the screen counted as out of+view for as long as it stayed open. The panel now starts showing the screen once it has opened.+### Plainer words, and one word for each thingOpenBot now says **Bot** everywhere a person reads it, where screens used to mix Bot, agent, coworkerdiff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsxindex 6cf6cc80..b439b5c5 100644--- a/app/src/components/channels/chat-transcript.tsx+++ b/app/src/components/channels/chat-transcript.tsx@@ -66,6 +66,7 @@ import {import { cn } from "@/lib/utils";import { VOICE_CHAT_ACTIVITY, type VoiceChatEntry } from "@/lib/voice/archive";import {+ dropRepeatedAppCards,groupToolSteps,ToolActivity,useDrawnTool,@@ -1560,7 +1561,10 @@ export function ChatTranscript({* cost was markdown parsing and chart SVGs, and those are skipped by the memoised children below,* which is where the 25x came from. This runs per render and is not worth guarding.*/- const items = groupToolSteps(toVisibleChatItems(messages), isDrawn);+ const items = groupToolSteps(+ dropRepeatedAppCards(toVisibleChatItems(messages)),+ isDrawn,+ );/** ONLY WHILE THERE IS NOTHING ELSE TO LOOK AT. Once a reply starts streaming, or a tool linediff --git a/app/src/components/channels/tool-activity.tsx b/app/src/components/channels/tool-activity.tsxindex 7e01601d..92fdc3f7 100644--- a/app/src/components/channels/tool-activity.tsx+++ b/app/src/components/channels/tool-activity.tsx@@ -141,6 +141,42 @@ function asksThePerson(result: string | undefined): boolean {return readTeamBotConsent(body) !== null || readConnectPrompt(body) !== null;}+/** The app a tool item draws a card for, or null when it draws none. */+function cardApp(item: ToolItem): string | null {+ if (item.result === undefined) return null;+ const answer = asText(item.result);+ const prompt = answer.startsWith(REFUSAL_MARKER)+ ? readConnectPrompt(answer.slice(REFUSAL_MARKER.length).trim())+ : item.toolCall.function.name === SHOW_APP_ACCESS_TOOL+ ? readConnectPrompt(answer)+ : null;+ return prompt?.serverId ?? null;+}++/**+ * One card per app per turn.+ *+ * A refused call already draws the app's card, and a Bot told to show the card for an app it cannot+ * use often calls `show_app_access` for the same app straight after, which drew the same card again.+ * The refused call is kept, because it is what the Bot did; a later `show_app_access` for an app+ * whose card that turn already drew is dropped. A person's next message starts a new turn.+ */+export function dropRepeatedAppCards(+ items: readonly VisibleChatItem[],+): VisibleChatItem[] {+ const drawn = new Set<string>();+ return items.filter((item) => {+ if (item.kind === "text" && item.role === "user") drawn.clear();+ if (item.kind !== "tool") return true;+ const app = cardApp(item);+ if (app === null) return true;+ const repeat =+ drawn.has(app) && item.toolCall.function.name === SHOW_APP_ACCESS_TOOL;+ drawn.add(app);+ return !repeat;+ });+}+type StepKind = "search" | "fetch" | "browser" | "connector" | "tool";export type StepDetails = {diff --git a/app/src/components/computer/preview-visibility.ts b/app/src/components/computer/preview-visibility.tsindex f8a48043..9ff32393 100644--- a/app/src/components/computer/preview-visibility.ts+++ b/app/src/components/computer/preview-visibility.ts@@ -34,8 +34,10 @@ export function useElementVisible<T extends Element>(): [setVisible(true);return;}- const observer = new IntersectionObserver(([entry]) => {- setVisible(Boolean(entry?.isIntersecting));+ // The newest entry is the element as it is now. A panel sliding open delivers the frame it began+ // off-screen and the frame it arrived in together, and reading the first left it "hidden".+ const observer = new IntersectionObserver((entries) => {+ setVisible(Boolean(entries.at(-1)?.isIntersecting));});observer.observe(node);return () => observer.disconnect();diff --git a/app/tests/preview-visibility.test.tsx b/app/tests/preview-visibility.test.tsxnew file mode 100644index 00000000..1c328f7f--- /dev/null+++ b/app/tests/preview-visibility.test.tsx@@ -0,0 +1,60 @@+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";+import { GlobalRegistrator } from "@happy-dom/global-registrator";+import { act, cleanup, renderHook } from "@testing-library/react";+import { useElementVisible } from "../src/components/computer/preview-visibility";++beforeAll(() => GlobalRegistrator.register());+afterEach(cleanup);+afterAll(() => GlobalRegistrator.unregister());++type Callback = (entries: Partial<IntersectionObserverEntry>[]) => void;++/** An observer the test fires by hand, as the browser would. */+function fakeObserver() {+ const callbacks: Callback[] = [];+ const original = window.IntersectionObserver;+ window.IntersectionObserver = class {+ constructor(callback: Callback) {+ callbacks.push(callback);+ }+ observe() {}+ unobserve() {}+ disconnect() {}+ takeRecords() {+ return [];+ }+ } as unknown as typeof IntersectionObserver;+ return {+ fire: (entries: Partial<IntersectionObserverEntry>[]) =>+ act(() => callbacks.at(-1)?.(entries)),+ restore: () => {+ window.IntersectionObserver = original;+ },+ };+}++test("an element that slid into view during one batch counts as visible", () => {+ // A panel animating open reports the frame it started off-screen and the frame it arrived in+ // together. The newest entry is the element as it is now; the first one is history.+ const observer = fakeObserver();+ try {+ const { result } = renderHook(() => useElementVisible<HTMLElement>());+ act(() => result.current[0](document.createElement("figure")));+ observer.fire([{ isIntersecting: false }, { isIntersecting: true }]);+ expect(result.current[1]).toBe(true);+ } finally {+ observer.restore();+ }+});++test("an element that left view during one batch counts as hidden", () => {+ const observer = fakeObserver();+ try {+ const { result } = renderHook(() => useElementVisible<HTMLElement>());+ act(() => result.current[0](document.createElement("figure")));+ observer.fire([{ isIntersecting: true }, { isIntersecting: false }]);+ expect(result.current[1]).toBe(false);+ } finally {+ observer.restore();+ }+});diff --git a/app/tests/tool-activity.test.tsx b/app/tests/tool-activity.test.tsxindex 123ad479..1e987b79 100644--- a/app/tests/tool-activity.test.tsx+++ b/app/tests/tool-activity.test.tsx@@ -6,6 +6,7 @@ import { CopilotKitContext } from "@copilotkit/react-core/v2/context";import type { ReactNode } from "react";import type { VisibleChatItem } from "../src/components/channels/chat-messages";import {+ dropRepeatedAppCards,groupToolSteps,stepDetails,summarise,@@ -300,3 +301,29 @@ test("while running, the line names the step in progress; opened, a step shows iexpect(view.getByText("query")).toBeDefined();expect(view.getAllByText("top 5 GDP 2025").length).toBeGreaterThan(0);});++test("an app's card is drawn once a turn, even when the Bot also asks to show it", () => {+ const refusal = JSON.stringify(+ 'Refused. [connect-app]{"serverId":"composio-airtable","title":"Airtable","agentId":"scout"} Connect Airtable and ask again.',+ );+ const shown = JSON.stringify(+ '[connect-app]{"serverId":"composio-airtable","title":"Airtable","agentId":"scout"} Connect Airtable.',+ );+ const items: VisibleChatItem[] = [+ { kind: "text", id: "q1", role: "user", text: "List my bases" },+ call("list", "mcp__composio-airtable__AIRTABLE_LIST_BASES", {}, refusal),+ call("show", "show_app_access", { app: "airtable" }, shown),+ text("a1"),+ { kind: "text", id: "q2", role: "user", text: "Try again" },+ call("show2", "show_app_access", { app: "airtable" }, shown),+ ];+ // The refused call keeps its card; the second card for the same app in that turn is dropped, and+ // a new turn may draw it again.+ expect(dropRepeatedAppCards(items).map((item) => item.id)).toEqual([+ "q1",+ "list",+ "a1",+ "q2",+ "show2",+ ]);+});