Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

9.0 KB
Raw Blame
import type { BrowserRpcParams } from "./browser/BotBrowserTypes.ts";
import * as Predicate from "effect/Predicate";
import * as Schema from "effect/Schema";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/unstable/http";
import type { AkeruBrowserEndpoint } from "./botWorkspace.ts";

const Targets = Schema.Array(
  Schema.Struct({ type: Schema.String, webSocketDebuggerUrl: Schema.optional(Schema.String) }),
);

const Message = Schema.Struct({
  id: Schema.optional(Schema.Number),
  result: Schema.optional(Schema.Json),
  error: Schema.optional(Schema.Json),
});

const Document = Schema.Struct({ root: Schema.Struct({ nodeId: Schema.Number }) });

const Node = Schema.Struct({ nodeId: Schema.Number });

const Box = Schema.Struct({ model: Schema.Struct({ content: Schema.Array(Schema.Number) }) });

const AXTree = Schema.Struct({
  nodes: Schema.Array(
    Schema.Struct({
      ignored: Schema.optional(Schema.Boolean),
      role: Schema.optional(Schema.Struct({ value: Schema.optional(Schema.Unknown) })),
      name: Schema.optional(Schema.Struct({ value: Schema.optional(Schema.Unknown) })),
      backendDOMNodeId: Schema.optional(Schema.Number),
    }),
  ),
});

const decodeMessage = Schema.decodeUnknownSync(Message);

const decodeDocument = Schema.decodeUnknownSync(Document);

const decodeNode = Schema.decodeUnknownSync(Node);

const decodeBox = Schema.decodeUnknownSync(Box);

const decodeAXTree = Schema.decodeUnknownSync(AXTree);

class ComputerDiscoveryError extends Data.TaggedError("ComputerDiscoveryError") {
  override readonly message = "Graphical browser discovery failed.";
}

export const discoverComputerTargets = Effect.fn("discoverComputerTargets")(
  function* (url: string, headers: Readonly<Record<string, string>>) {
    const client = HttpClient.withScope(yield* HttpClient.HttpClient);
    const response = yield* client.get(url, { headers });

    if (response.status < 200 || response.status >= 300) return yield* new ComputerDiscoveryError();

    return yield* HttpClientResponse.schemaBodyJson(Targets)(response);
  },
  Effect.scoped,
  Effect.timeout(30_000),
);

/** Direct CDP connection to the workspace's existing graphical Chromium, never a second browser. */
export class ComputerCdp {
  private readonly socket: WebSocket;
  private nextId = 1;
  private readonly pending = new Map<
    number,
    { resolve: (value: typeof Message.Type.result) => void; reject: (error: Error) => void }
  >();
  private constructor(socket: WebSocket) {
    this.socket = socket;
    socket.addEventListener("message", (event) => {
      try {
        const message = decodeMessage(JSON.parse(String(event.data)));

        if (message.id === undefined) return;
        const pending = this.pending.get(message.id);
        this.pending.delete(message.id);

        if (message.error !== undefined)
          pending?.reject(new Error("Graphical browser command failed."));
        else pending?.resolve(message.result);
      } catch {
        this.close();
      }
    });
    socket.addEventListener("close", () => this.rejectPending());
    socket.addEventListener("error", () => this.rejectPending());
  }

  static async connect(endpoint: AkeruBrowserEndpoint) {
    const url = new URL(
      "json/list",
      endpoint.url.endsWith("/") ? endpoint.url : `${endpoint.url}/`,
    );

    const targets = await Effect.runPromise(
      discoverComputerTargets(url.toString(), endpoint.requestHeaders).pipe(
        Effect.provide(FetchHttpClient.layer),
      ),
    );

    const target = targets.find((target) => target.type === "page" && target.webSocketDebuggerUrl);

    if (!target?.webSocketDebuggerUrl) throw new Error("Graphical browser page is unavailable.");
    const address = new URL(endpoint.url);
    address.protocol = address.protocol === "https:" ? "wss:" : "ws:";
    address.pathname = new URL(target.webSocketDebuggerUrl).pathname;
    const token = endpoint.requestHeaders["x-daytona-preview-token"];

    if (token) address.searchParams.set("DAYTONA_SANDBOX_AUTH_KEY", token);
    const socket = new WebSocket(address);
    const client = new ComputerCdp(socket);
    await new Promise<void>((resolve, reject) => {
      const signal = AbortSignal.timeout(30_000);

      const settle = (error: Error | null) => {
        signal.removeEventListener("abort", onAbort);
        socket.removeEventListener("open", onOpen);
        socket.removeEventListener("error", onError);
        socket.removeEventListener("close", onClose);

        if (error === null) return resolve();
        client.close();
        reject(error);
      };

      const onAbort = () => settle(new Error("Graphical browser connection timed out."));
      const onOpen = () => settle(null);
      const onError = () => settle(new Error("Graphical browser connection failed."));
      // A handshake can close without an error event; fail fast so callers retry.
      const onClose = () => settle(new Error("Graphical browser connection closed."));
      signal.addEventListener("abort", onAbort, { once: true });
      socket.addEventListener("open", onOpen, { once: true });
      socket.addEventListener("error", onError, { once: true });
      socket.addEventListener("close", onClose, { once: true });
    });

    return client;
  }

  private rejectPending() {
    for (const pending of this.pending.values())
      pending.reject(new Error("Graphical browser disconnected."));
    this.pending.clear();
  }

  get connected() {
    return this.socket.readyState === WebSocket.OPEN;
  }

  close() {
    this.rejectPending();
    this.socket.close();
  }

  private request(
    method: string,
    params: BrowserRpcParams = {},
  ): Promise<typeof Message.Type.result> {
    if (this.socket.readyState !== WebSocket.OPEN)
      return Promise.reject(new Error("Graphical browser disconnected."));
    const id = this.nextId++;

    return new Promise((resolve, reject) => {
      const signal = AbortSignal.timeout(30_000);

      const abort = () => {
        this.close();
        reject(new Error("Graphical browser command timed out."));
      };

      signal.addEventListener("abort", abort, { once: true });
      this.pending.set(id, {
        resolve: (value) => {
          signal.removeEventListener("abort", abort);
          resolve(value);
        },
        reject: (error) => {
          signal.removeEventListener("abort", abort);
          reject(error);
        },
      });
      this.socket.send(JSON.stringify({ id, method, params }));
    });
  }

  async call(name: string, input: BrowserRpcParams) {
    if (name === "goto") {
      await this.request("Page.navigate", { url: input.url });

      return "Navigated.";
    }

    if (name === "tree") {
      const tree = decodeAXTree(await this.request("Accessibility.getFullAXTree"));

      return tree.nodes
        .filter((node) => !node.ignored)
        .map(
          (node) =>
            `${String(node.role?.value ?? "")} ${String(node.name?.value ?? "")} [backendNodeId=${node.backendDOMNodeId ?? ""}]`,
        )
        .join("\n")
        .slice(0, 50 * 1024);
    }

    if (name !== "click" && name !== "fill")
      throw new Error("Unsupported graphical browser operation.");
    let target: { nodeId: number } | { backendNodeId: number };

    if (Predicate.isString(input.selector)) {
      const document = decodeDocument(await this.request("DOM.getDocument"));
      target = decodeNode(
        await this.request("DOM.querySelector", {
          nodeId: document.root.nodeId,
          selector: input.selector,
        }),
      );

      if (target.nodeId === 0) throw new Error("Graphical browser selector did not match.");
    } else if (Predicate.isNumber(input.backendNodeId))
      target = { backendNodeId: input.backendNodeId };
    else throw new Error("Graphical browser target is required.");
    await this.request("DOM.scrollIntoViewIfNeeded", target);

    if (name === "click") {
      const box = decodeBox(await this.request("DOM.getBoxModel", target));
      const points = box.model.content;

      if (points.length !== 8) throw new Error("Graphical browser target has no bounds.");
      const x = (points[0]! + points[4]!) / 2;
      const y = (points[1]! + points[5]!) / 2;
      await this.request("Input.dispatchMouseEvent", {
        type: "mousePressed",
        x,
        y,
        button: "left",
        clickCount: 1,
      });
      await this.request("Input.dispatchMouseEvent", {
        type: "mouseReleased",
        x,
        y,
        button: "left",
        clickCount: 1,
      });
    } else {
      await this.request("DOM.focus", target);
      await this.request("Input.dispatchKeyEvent", {
        type: "keyDown",
        key: "a",
        code: "KeyA",
        modifiers: 2,
        windowsVirtualKeyCode: 65,
      });
      await this.request("Input.dispatchKeyEvent", {
        type: "keyUp",
        key: "a",
        code: "KeyA",
        modifiers: 2,
        windowsVirtualKeyCode: 65,
      });
      await this.request("Input.insertText", { text: input.value });
    }

    return "Completed.";
  }
}