Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

12.5 KB
Raw Blame
import { useCopilotKit, useFrontendTool } from "@copilotkit/react-core/v2";
import { NOT_SHOWN, NOT_SHOWN_TAIL } from "@openbot/shared/component-markers";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import {
  useCallback,
  useEffect,
  useContext,
  useLayoutEffect,
  useMemo,
  useRef,
  useState,
  useSyncExternalStore,
} from "react";
import { RefusedCard } from "@/components/gallery/refused";
import {
  ArgumentsArrivingContext,
  GalleryStreamingContext,
} from "@/components/gallery/streaming";
import {
  agentComponentsQueryOptions,
  announceGallery,
  componentKeys,
  decideComponent,
  type GrantedComponent,
} from "@/lib/components/queries";
import { useActiveBotId, useDeclaredBotId } from "@/lib/copilot/active-bot";
import {
  GALLERY_COMPONENTS,
  type GalleryComponent,
  galleryManifest,
} from "@/lib/copilot/gallery-registry";
import {
  cardState,
  COULDNT_CHECK,
  createGrantStore,
  type GrantStore,
  grantsFrom,
} from "@/lib/copilot/grant-store";
import { settledArgs } from "@/lib/copilot/settled-args";

/**
 * Register compiled gallery components once per name, scoped to the active Bot with `available`.
 * Handlers still recheck grants at call time because a run's offered tool list is only a snapshot.
 */
export function GalleryTools() {
  const queryClient = useQueryClient();
  useEffect(() => {
    void announceGallery(galleryManifest()).then((added) => {
      if (added.length > 0) {
        void queryClient.invalidateQueries({ queryKey: componentKeys.all });
      }
    });
  }, [queryClient]);

  // Active Bot comes from the route/channel surface currently driving the provider.
  const grantsFor = useActiveBotId();
  // See sandboxed-tools: the placeholder Bot is not fetchable, so the grant query waits for a
  // surface to declare a real one.
  const declared = useDeclaredBotId();
  const grantsQuery = useQuery(agentComponentsQueryOptions(declared));
  const granted = grantsQuery.data;
  // One store per mount; every card reads its grants from here (see `grant-store.ts`).
  const [store] = useState(createGrantStore);
  useEffect(() => {
    store.set({
      grants: grantsFrom({
        data: granted,
        isError: grantsQuery.isError,
        enabled: declared !== undefined,
      }),
    });
  }, [store, granted, grantsQuery.isError, declared]);
  const held = useMemo(
    () =>
      new Map(
        (granted ?? []).map((component: GrantedComponent) => [
          component.name,
          component.description,
        ]),
      ),
    [granted],
  );

  return (
    <>
      {GALLERY_COMPONENTS.map((spec) =>
        spec.kind === "decision" ? (
          <GrantedDecision
            held={held}
            key={spec.name}
            spec={spec}
            store={store}
          />
        ) : (
          <GrantedTool
            grantsFor={grantsFor}
            held={held}
            key={spec.name}
            spec={spec}
            store={store}
          />
        ),
      )}
    </>
  );
}

function GrantedTool({
  grantsFor,
  spec,
  held,
  store,
}: {
  grantsFor: string;
  spec: GalleryComponent;
  held: Map<string, string>;
  store: GrantStore;
}) {
  const description = held.get(spec.name);
  const isHeld = description !== undefined;
  // Created ONCE: the SDK rebuilds a card whenever this identity changes. See `makeGalleryRender`.
  const [render] = useState(() => makeGalleryRender(spec, store));

  useFrontendTool({
    name: spec.name,
    // Published descriptions are model-facing runtime behavior.
    description: description ?? spec.description,
    parameters: spec.parameters,
    // Keep the hook mounted and hide revoked grants from the model to preserve hook order.
    available: isHeld,
    handler: async (
      args: Record<string, unknown>,
      context: { toolCall?: { id?: string } },
    ) => {
      const decision = await decideComponent(
        spec.name,
        grantsFor,
        spec.reads?.(args ?? {}) ?? [],
      );
      if (!decision.allowed) {
        const reason = decision.reason ?? "That component is not allowed here.";
        const id = context?.toolCall?.id;
        if (id) {
          store.set({
            refusals: new Map(store.get().refusals).set(id, reason),
          });
        }
        // Led with the same words the card shows, so the transcript and the card agree.
        return `${NOT_SHOWN}${spec.title}. ${reason}${NOT_SHOWN_TAIL}`;
      }
      return spec.confirmation ?? "It is now on screen for the person.";
    },
    render,
  });

  return null;
}

/**
 * The card a gallery tool call draws, made once per component.
 *
 * ONCE, AND THAT IS THE POINT. The SDK hands this function to `ToolCallRenderer` as the component
 * type, and remounts the card whenever it changes identity. It used to be a `useCallback` over the
 * grants, so grants arriving (they start unknown on every channel) rebuilt every card: drawn as the
 * refusal, then torn down and drawn again as the component. Now it reads the grants from the store
 * and only re-renders. Exported for its tests.
 */
export function makeGalleryRender(spec: GalleryComponent, store: GrantStore) {
  const Component = spec.Component;
  return function GalleryRender(props: {
    toolCallId?: string;
    args?: Record<string, unknown>;
    result?: string;
    status?: string;
  }) {
    const snapshot = useSyncExternalStore(
      store.subscribe,
      store.get,
      store.get,
    );
    const arriving = useContext(ArgumentsArrivingContext);
    const refusal = props.toolCallId
      ? snapshot.refusals.get(props.toolCallId)
      : undefined;
    if (refusal) {
      return <RefusedCard reason={refusal} title={spec.title} />;
    }
    /*
     * A stored refusal. The in-memory map above is gone once the page reloads, and a headless turn
     * never had one: the server answered the call itself (`server/src/components/headless.ts`).
     * The result is what survives, so a call that drew nothing is drawn as the refusal it was.
     */
    if (props.result?.startsWith(NOT_SHOWN)) {
      const lead = `${NOT_SHOWN}${spec.title}. `;
      const reason = (
        props.result.startsWith(lead)
          ? props.result.slice(lead.length)
          : props.result.slice(NOT_SHOWN.length)
      ).replace(NOT_SHOWN_TAIL, "");
      return <RefusedCard reason={reason} title={spec.title} />;
    }
    // From the polled grants, so a revocation shows before a new call starts. Unknown draws.
    const state = cardState(snapshot, spec.name, spec.title, props.toolCallId);
    if (typeof state === "object") {
      return <RefusedCard reason={state.refused} title={spec.title} />;
    }
    /*
     * A number, link or colour still arriving is held back; see `settledArgs`. Only while the
     * arguments are streaming (`inProgress`, not `executing`) in a row that can still receive them.
     */
    const complete = !(arriving && props.status === "inProgress");
    return (
      <GalleryStreamingContext.Provider value={!complete}>
        <Component {...settledArgs(props.args ?? {}, complete)} />
      </GalleryStreamingContext.Provider>
    );
  };
}

function GrantedDecision({
  spec,
  held,
  store,
}: {
  spec: GalleryComponent;
  held: Map<string, string>;
  store: GrantStore;
}) {
  useDecisionTool(spec, store, held);
  return null;
}

type FrontendTool = Parameters<typeof useFrontendTool>[0];

/**
 * The removal the SDK's `useHumanInTheLoop` does on unmount, which `useDecisionTool` copies the rest
 * of: without it a decision's renderer stayed registered after the gallery tools went away, drawing
 * from a grant store nobody updated any more.
 */
export function useForgetRenderOnUnmount(
  name: string,
  core: { removeHookRenderToolCall(name: string, agentId?: string): void },
) {
  useLayoutEffect(
    () => () => core.removeHookRenderToolCall(name),
    [core, name],
  );
}

/** `useFrontendTool`, and its renderer removed again on unmount, as `useHumanInTheLoop` does. */
function useHumanInTheLoopTool(tool: FrontendTool) {
  const { copilotkit } = useCopilotKit();
  useFrontendTool(tool);
  useForgetRenderOnUnmount(tool.name, copilotkit);
}

/**
 * A decision tool: the run waits until the person answers on the card.
 *
 * REGISTERED THROUGH `useFrontendTool`, NOT `useHumanInTheLoop`, AND THAT IS THE FIX. The SDK's
 * human-in-the-loop hook wraps the render in a component of its own that is rebuilt whenever the
 * tool's description changes (1.83.0: a `useCallback` over `tool.description`). Ours changes from
 * the built-in words to the published ones exactly when the grants load, so a decision on screen was
 * torn down at that moment, and again on a revoke. This is that hook's answer-and-wait handler, kept
 * as it is, with our render created once.
 *
 * `register` is the SDK hook (with its unmount cleanup) in the app and a recorder in the tests. Exported for its tests.
 */
export function useDecisionTool(
  spec: GalleryComponent,
  store: GrantStore,
  held: Map<string, string>,
  register: (tool: FrontendTool) => void = useHumanInTheLoopTool,
) {
  const description = held.get(spec.name);
  const isHeld = description !== undefined;
  const resolveRef = useRef<((result: unknown) => void) | null>(null);
  const stopListeningRef = useRef<(() => void) | null>(null);

  const respond = useCallback(async (result: unknown) => {
    if (!resolveRef.current) return;
    stopListeningRef.current?.();
    stopListeningRef.current = null;
    resolveRef.current(result);
    resolveRef.current = null;
  }, []);

  const handler = useCallback(
    async (_args: unknown, context?: { signal?: AbortSignal }) =>
      new Promise<unknown>((resolve, reject) => {
        const signal = context?.signal;
        if (signal?.aborted) {
          reject(new Error("Human-in-the-loop interaction aborted"));
          return;
        }
        resolveRef.current = resolve;
        if (signal) {
          const onAbort = () => {
            stopListeningRef.current = null;
            resolveRef.current = null;
            reject(new Error("Human-in-the-loop interaction aborted"));
          };
          signal.addEventListener("abort", onAbort, { once: true });
          stopListeningRef.current = () =>
            signal.removeEventListener("abort", onAbort);
        }
      }),
    [],
  );

  // Created once, for the same reason as `makeGalleryRender`.
  const [Render] = useState(() => makeDecisionRender(spec, store, respond));

  register({
    name: spec.name,
    description: description ?? spec.description,
    parameters: spec.parameters,
    available: isHeld,
    type: "human-in-the-loop",
    handler,
    render: Render,
  } as FrontendTool);
}

/**
 * The card a decision draws, made once per component.
 *
 * It can answer only while the call is executing, as the SDK's own decisions do. While the grants
 * are unknown it draws without `respond`, which the decision components read as "not yet": the
 * person sees what is being asked and cannot answer until the Bot is known to be allowed to ask.
 * Then the same card enables in place. Exported for its tests.
 */
export function makeDecisionRender(
  spec: GalleryComponent,
  store: GrantStore,
  respond: (result: unknown) => Promise<void>,
) {
  const Component = spec.Component;
  return function DecisionRender(props: Record<string, unknown>) {
    const snapshot = useSyncExternalStore(
      store.subscribe,
      store.get,
      store.get,
    );
    const answer = props.status === "executing" ? respond : undefined;
    const state = cardState(snapshot, spec.name, spec.title);
    if (state === "draw") return <Component {...props} respond={answer} />;
    if (state === "wait-for-grant") {
      return <Component {...props} respond={undefined} />;
    }
    return (
      <RefusedDecision
        {...(snapshot.grants.status === "failed"
          ? { reason: COULDNT_CHECK(spec.title) }
          : {})}
        respond={answer}
        title={spec.title}
      />
    );
  };
}

/**
 * A decision the Bot was not allowed to ask for.
 *
 * Decision tools suspend the run, so a refusal must also answer the tool call.
 */
function RefusedDecision({
  title,
  respond,
  reason = `${title} is not available to this Bot at the moment, so the person was not asked. An administrator grants components per Bot, and can unpublish one for every Bot at once.`,
}: {
  title: string;
  respond?: (result: unknown) => Promise<void>;
  reason?: string;
}) {
  useEffect(() => {
    if (!respond) return;
    void respond(reason).catch(() => {
      // The run being gone is not a failure worth reporting: there is nothing left to answer.
    });
  }, [respond, reason]);

  return <RefusedCard reason={reason} title={title} />;
}