Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

16.9 KB
Raw Blame
import * as Schema from "effect/Schema";
import * as Data from "effect/Data";
import type { PreviewSessionSnapshot } from "@akeru/contracts";
import * as Match from "effect/Match";
import {
  DEFAULT_VIEWPORT,
  browserbaseContexts,
  requireBrowserbaseApiKey,
} from "./BrowserbaseContext.ts";
import {
  FILL_PREVIEW_VIEWPORT,
  PreviewAutomationClickInput,
  PreviewAutomationEvaluateInput,
  PreviewAutomationNavigateInput,
  PreviewAutomationOpenInput,
  PreviewAutomationPressInput,
  type PreviewAutomationRequest,
  PreviewAutomationResizeInput,
  PreviewAutomationScrollInput,
  PreviewAutomationSetColorSchemeInput,
  PreviewAutomationSnapshot,
  type PreviewAutomationStatus,
  PreviewAutomationTypeInput,
  PreviewAutomationWaitForInput,
  type PreviewRenderedViewportSize,
  type PreviewTabId,
  type ThreadId,
} from "@akeru/contracts";
import { normalizePreviewUrl } from "@akeru/shared/preview";
import { resolvePreviewViewport } from "@akeru/shared/previewViewport";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Layer from "effect/Layer";
import * as RcMap from "effect/RcMap";
import * as Scope from "effect/Scope";
import { HttpClient } from "effect/unstable/http";
import { type BrowserContext, type Page } from "playwright-core";
import * as ServerSettings from "../serverSettings.ts";
import * as PreviewManager from "./Manager.ts";

interface BrowserTab {
  readonly threadId: ThreadId;
  readonly page: Page;
  loading: boolean;
}

const requestedUrl = (input: PreviewAutomationNavigateInput): string => {
  if (input.url) return normalizePreviewUrl(input.url);
  const target = input.target!;

  if (target.kind === "url") return normalizePreviewUrl(target.url);
  const path = target.path?.startsWith("/") ? target.path : `/${target.path ?? ""}`;

  return `${target.protocol ?? "http"}://127.0.0.1:${target.port}${path}`;
};

const viewportForSetting = (
  setting: ReturnType<typeof resolvePreviewViewport> | typeof FILL_PREVIEW_VIEWPORT,
): PreviewRenderedViewportSize =>
  Match.value(setting).pipe(
    Match.tag("fill", () => DEFAULT_VIEWPORT),
    Match.orElse((viewport) => ({ width: viewport.width, height: viewport.height })),
  );

const selectorFor = (input: {
  readonly locator?: string | undefined;
  readonly selector?: string | undefined;
}) => input.locator ?? input.selector ?? null;

export { BrowserConfigurationError } from "./BrowserbaseContext.ts";

const errorMessage = (cause: unknown): string =>
  cause instanceof Error ? cause.message : "Browser operation failed.";

export class ServerPreviewBrowser extends Context.Service<
  ServerPreviewBrowser,
  {
    readonly handle: (
      request: PreviewAutomationRequest,
    ) => Promise<import("@akeru/contracts").PreviewAutomationResponse["result"]>;
    readonly close: () => Promise<void>;
  }
>()("akeru-bot/preview/ServerPreviewBrowser") {}

export const make = Effect.gen(function* ServerPreviewBrowserMake() {
  const runtimeContext = yield* Effect.context<never>();
  const runPromise = Effect.runPromiseWith(runtimeContext);
  const runSync = Effect.runSyncWith(runtimeContext);
  const previewManager = yield* PreviewManager.PreviewManager;
  const httpClient = yield* HttpClient.HttpClient;
  const settingsService = yield* ServerSettings.ServerSettingsService;
  const requireApiKey = requireBrowserbaseApiKey(settingsService);
  const tabs = new Map<PreviewTabId, BrowserTab>();
  const activeByThread = new Map<string, PreviewTabId>();

  const contexts = yield* browserbaseContexts(httpClient, settingsService);

  // One lease holds the shared browser open. close() releases it, which closes the
  // Browserbase session; the next getContext() opens a fresh one.
  let lease: { readonly scope: Scope.Closeable; readonly context: Promise<BrowserContext> } | null =
    null;

  // Bumped by close() so an open that raced it does not leave a tab behind.
  let closeGeneration = 0;

  const getContext = (): Promise<BrowserContext> => {
    if (lease) return lease.context;
    const scope = runSync(Scope.make());

    const context = runPromise(
      RcMap.get(contexts, "browser").pipe(
        Effect.map(({ context }) => context),
        Scope.provide(scope),
      ),
    ).catch(async (cause: unknown) => {
      if (lease === current) lease = null;
      await runPromise(Scope.close(scope, Exit.void));
      throw cause;
    });

    const current = { scope, context };
    lease = current;

    return context;
  };

  // A request that raced close() must not open a new session: nothing would
  // close it again. getContext() claims the lease synchronously, so checking
  // right before the call leaves no gap.
  const contextFor = (generation: number): Promise<BrowserContext> => {
    if (generation !== closeGeneration) {
      return Promise.reject(new Error("The browser was closed while this request was running."));
    }

    return getContext();
  };

  const resolveTab = (request: PreviewAutomationRequest): BrowserTab => {
    const tabId = request.tabId ?? activeByThread.get(request.threadId);
    const tab = tabId ? tabs.get(tabId) : undefined;

    if (!tab || tab.threadId !== request.threadId) {
      throw new Error("No active browser tab exists for this chat.");
    }

    return tab;
  };

  const tabIdFor = (tab: BrowserTab): PreviewTabId => {
    for (const [tabId, candidate] of tabs) if (candidate === tab) return tabId;
    throw new Error("Browser tab is no longer active.");
  };

  const status = async (tab: BrowserTab | null): Promise<PreviewAutomationStatus> => {
    if (!tab) {
      return {
        available: true,
        visible: false,
        tabId: null,
        url: null,
        title: null,
        loading: false,
      };
    }

    return {
      available: true,
      visible: true,
      tabId: tabIdFor(tab),
      url: tab.page.url() === "about:blank" ? null : tab.page.url(),
      title: await tab.page.title(),
      loading: tab.loading,
      viewportSetting: FILL_PREVIEW_VIEWPORT,
      viewport: tab.page.viewportSize() ?? DEFAULT_VIEWPORT,
    };
  };

  const publishFrame = async (
    tab: BrowserTab,
  ): Promise<PreviewAutomationSnapshot["screenshot"]> => {
    const data = await tab.page.screenshot({ type: "png" });
    const viewport = tab.page.viewportSize() ?? DEFAULT_VIEWPORT;

    const screenshot = {
      mimeType: "image/png" as const,
      data: data.toString("base64"),
      width: viewport.width,
      height: viewport.height,
    };

    await runPromise(
      previewManager.reportFrame({
        threadId: tab.threadId,
        tabId: tabIdFor(tab),
        frame: screenshot,
      }),
    );

    return screenshot;
  };

  const reportPageStatus = async (tab: BrowserTab): Promise<void> => {
    const url = tab.page.url();
    await runPromise(
      previewManager.reportStatus({
        threadId: tab.threadId,
        tabId: tabIdFor(tab),
        navStatus: Navigation["Success"]({
          url: url === "about:blank" ? "about:blank" : url,
          title: await tab.page.title(),
        }),
        canGoBack: false,
        canGoForward: false,
      }),
    );
  };

  const navigate = async (
    tab: BrowserTab,
    input: PreviewAutomationNavigateInput,
    fallbackTimeout: number,
  ) => {
    const url = requestedUrl(input);
    const readiness = input.readiness ?? "load";
    tab.loading = true;

    try {
      await tab.page.goto(url, {
        timeout: input.timeoutMs ?? fallbackTimeout,
        waitUntil: Match.value(readiness).pipe(
          Match.when("domContentLoaded", (): "domcontentloaded" => "domcontentloaded"),
          Match.when("none", (): "commit" => "commit"),
          Match.orElse((): "load" => "load"),
        ),
      });
      tab.loading = false;
      await reportPageStatus(tab);
      await publishFrame(tab);

      return await status(tab);
    } catch (cause) {
      tab.loading = false;
      throw new Error(errorMessage(cause), { cause: cause });
    }
  };

  const handle = async (request: PreviewAutomationRequest) => {
    const generation = closeGeneration;
    await runPromise(requireApiKey);

    if (request.operation === "status") {
      const tabId = request.tabId ?? activeByThread.get(request.threadId);

      return await status(tabId ? (tabs.get(tabId) ?? null) : null);
    }

    if (request.operation === "open") {
      const input = decodeOpenInput(request.input);
      let tabId = request.tabId ?? activeByThread.get(request.threadId);
      let tab = tabId ? tabs.get(tabId) : undefined;

      if (!tab || input.reuseExistingTab === false) {
        const snapshot = await runPromise(
          previewManager.open({
            threadId: request.threadId,
            ...(input.url ? { url: input.url } : {}),
          }),
        );

        tabId = snapshot.tabId;
        const page = await (await contextFor(generation)).newPage();

        if (generation !== closeGeneration) {
          await page.close().catch(() => undefined);
          throw new Error("The browser was closed while this tab was opening.");
        }

        tab = { threadId: request.threadId, page, loading: false };
        tabs.set(tabId, tab);
      }

      activeByThread.set(request.threadId, tabId!);

      if (input.url) {
        return await navigate(tab, { url: input.url }, request.timeoutMs);
      }

      await reportPageStatus(tab);
      await publishFrame(tab);

      return await status(tab);
    }

    const tab = resolveTab(request);
    const tabId = tabIdFor(tab);

    switch (request.operation) {
      case "navigate":
        return await navigate(tab, decodeNavigateInput(request.input), request.timeoutMs);
      case "snapshot": {
        const [rawPageData, screenshot] = await Promise.all([
          tab.page.evaluate(`(() => {
            const nodes = Array.from(document.querySelectorAll("a,button,input,textarea,select,[role],[contenteditable='true']")).slice(0, 200);
            return {
              visibleText: document.body?.innerText ?? "",
              interactiveElements: nodes.map((element, index) => {
                const rect = element.getBoundingClientRect();
                return {
                  tag: element.tagName.toLowerCase(),
                  role: element.getAttribute("role"),
                  name: element.getAttribute("aria-label") ?? element.innerText ?? element.getAttribute("placeholder") ?? "",
                  selector: element.id ? "#" + CSS.escape(element.id) : element.tagName.toLowerCase() + ":nth-of-type(" + (index + 1) + ")",
                  x: rect.x,
                  y: rect.y,
                  width: rect.width,
                  height: rect.height,
                };
              }),
            };
          })()`),
          publishFrame(tab),
        ]);

        const pageData = decodePageData(rawPageData);

        return {
          url: tab.page.url(),
          title: await tab.page.title(),
          loading: tab.loading,
          ...pageData,
          accessibilityTree: await tab.page
            .locator("body")
            .ariaSnapshot()
            .catch(() => null),
          consoleEntries: [],
          networkEntries: [],
          actionTimeline: [],
          screenshot,
        } satisfies PreviewAutomationSnapshot;
      }

      case "click": {
        const input = decodeClickInput(request.input);
        const selector = selectorFor(input);

        if (selector)
          await tab.page.locator(selector).click({ timeout: input.timeoutMs ?? request.timeoutMs });
        else await tab.page.mouse.click(input.x!, input.y!);
        await publishFrame(tab);

        return undefined;
      }

      case "type": {
        const input = decodeTypeInput(request.input);
        const selector = selectorFor(input);

        if (selector) {
          const locator = tab.page.locator(selector);

          if (input.clear)
            await locator.fill(input.text, { timeout: input.timeoutMs ?? request.timeoutMs });
          else {
            await locator.focus({ timeout: input.timeoutMs ?? request.timeoutMs });
            await tab.page.keyboard.insertText(input.text);
          }
        } else {
          if (input.clear) await tab.page.keyboard.press("Meta+A");
          await tab.page.keyboard.insertText(input.text);
        }

        await publishFrame(tab);

        return undefined;
      }

      case "press": {
        const input = decodePressInput(request.input);
        await tab.page.keyboard.press([...(input.modifiers ?? []), input.key].join("+"));
        await publishFrame(tab);

        return undefined;
      }

      case "scroll": {
        const input = decodeScrollInput(request.input);
        const selector = selectorFor(input);

        if (selector) {
          await tab.page
            .locator(selector)
            .evaluate((element, delta) => element.scrollBy(delta.x, delta.y), {
              x: input.deltaX ?? 0,
              y: input.deltaY ?? 0,
            });
        } else await tab.page.mouse.wheel(input.deltaX ?? 0, input.deltaY ?? 0);
        await publishFrame(tab);

        return undefined;
      }

      case "evaluate": {
        const input = decodeEvaluateInput(request.input);
        const session = await (await contextFor(generation)).newCDPSession(tab.page);

        try {
          const result = await session.send("Runtime.evaluate", {
            expression: input.expression,
            awaitPromise: input.awaitPromise ?? true,
            returnByValue: input.returnByValue ?? true,
          });

          await publishFrame(tab);

          if (result.exceptionDetails) throw new Error(result.exceptionDetails.text);

          return result.result.value ?? result.result.description ?? null;
        } finally {
          await session.detach();
        }
      }

      case "waitFor": {
        const input = decodeWaitForInput(request.input);
        const timeout = input.timeoutMs ?? request.timeoutMs;
        const waits: Promise<void>[] = [];
        const selector = selectorFor(input);

        if (selector) waits.push(tab.page.locator(selector).waitFor({ state: "visible", timeout }));

        if (input.text)
          waits.push(tab.page.getByText(input.text, { exact: false }).first().waitFor({ timeout }));

        if (input.urlIncludes)
          waits.push(
            tab.page.waitForURL((url) => url.href.includes(input.urlIncludes!), { timeout }),
          );
        await Promise.all(waits);
        await publishFrame(tab);

        return undefined;
      }

      case "resize": {
        const input = decodeResizeInput(request.input);
        const setting = resolvePreviewViewport(input);
        const viewport = viewportForSetting(setting);
        await tab.page.setViewportSize(viewport);
        await runPromise(
          previewManager.resize({ threadId: request.threadId, tabId, viewport: setting }),
        );
        await publishFrame(tab);

        return { tabId, setting, viewport };
      }

      case "setColorScheme": {
        const input = decodeSetColorSchemeInput(request.input);
        await tab.page.emulateMedia({
          colorScheme: input.colorScheme === "system" ? null : input.colorScheme,
        });
        await publishFrame(tab);

        return { tabId, colorScheme: input.colorScheme };
      }

      case "recordingStart":
      case "recordingStop":
        throw new Error("Browser recording is not available in the web preview host.");
      default:
        throw new Error(`Unsupported browser operation: ${request.operation satisfies never}`);
    }
  };

  const close = async () => {
    closeGeneration += 1;
    const current = lease;
    lease = null;
    // Pages die with the browser session.
    tabs.clear();
    activeByThread.clear();

    if (!current) return;
    await current.context.catch(() => undefined);
    await runPromise(Scope.close(current.scope, Exit.void));
  };

  return ServerPreviewBrowser.of({ handle, close });
});

export const layer = Layer.effect(
  ServerPreviewBrowser,
  Effect.acquireRelease(make, (browser) =>
    Effect.promise(() => browser.close()).pipe(Effect.orDie),
  ),
);

const Navigation = Data.taggedEnum<PreviewSessionSnapshot["navStatus"]>();

const decodeOpenInput = Schema.decodeUnknownSync(PreviewAutomationOpenInput);

const decodeNavigateInput = Schema.decodeUnknownSync(PreviewAutomationNavigateInput);

const decodeClickInput = Schema.decodeUnknownSync(PreviewAutomationClickInput);

const decodeTypeInput = Schema.decodeUnknownSync(PreviewAutomationTypeInput);

const decodePressInput = Schema.decodeUnknownSync(PreviewAutomationPressInput);

const decodeScrollInput = Schema.decodeUnknownSync(PreviewAutomationScrollInput);

const decodeEvaluateInput = Schema.decodeUnknownSync(PreviewAutomationEvaluateInput);

const decodeWaitForInput = Schema.decodeUnknownSync(PreviewAutomationWaitForInput);

const decodeResizeInput = Schema.decodeUnknownSync(PreviewAutomationResizeInput);

const decodeSetColorSchemeInput = Schema.decodeUnknownSync(PreviewAutomationSetColorSchemeInput);

const decodePageData = Schema.decodeUnknownSync(
  Schema.Struct({
    visibleText: Schema.String,
    interactiveElements: PreviewAutomationSnapshot.fields.interactiveElements,
  }),
);