Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

20.8 KB
Raw Blame
import { PreviewAutomationElement } from "@akeru/contracts";
import { decodeEvaluationValue } from "./PreviewEvaluation.ts";
import * as Schema from "effect/Schema";

interface PreviewSelectorDiagnostics {
  readonly selectorKind: PreviewAutomationSelectorKind;
  readonly selectorLength?: number;
}

import * as Predicate from "effect/Predicate";

import { type BrowserWindow, desktopCapturer, nativeImage } from "electron";

import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";

import type * as Fiber from "effect/Fiber";

import * as Option from "effect/Option";

import * as Ref from "effect/Ref";

import { type PreviewAutomationSelectorKind, PreviewOperationError } from "./PreviewErrors.ts";
import type { createPreviewBrowserControl } from "./PreviewBrowserControl.ts";
import type { createPreviewState } from "./PreviewState.ts";
import {
  MAX_VISIBLE_TEXT_LENGTH,
  MAX_INTERACTIVE_ELEMENTS,
  MAX_SCREENSHOT_WIDTH,
  containsVisiblePngPixel,
  scaleCaptureRect,
  normalizeCaptureRect,
  type BrowserDiagnostics,
  type SendCommand,
} from "./PreviewModel.ts";

const decodePage = Schema.decodeUnknownEffect(
  Schema.Struct({
    url: Schema.String,
    title: Schema.String,
    loading: Schema.Boolean,
    visibleText: Schema.String,
    interactiveElements: Schema.Array(PreviewAutomationElement),
  }),
);

export const createPreviewAutomationSnapshot = ({
  evaluateWithDebugger,
  diagnosticsRef,
  actionTimelineRef,
  mainWindowRef,
  encodeJson,
  attemptPromise,
  attempt,
  runFork,
  requireWebContents,
  withControlSession,
}: {
  readonly evaluateWithDebugger: ReturnType<
    typeof createPreviewBrowserControl
  >["evaluateWithDebugger"];
  readonly diagnosticsRef: Ref.Ref<ReadonlyMap<number, BrowserDiagnostics>>;
  readonly actionTimelineRef: Ref.Ref<
    ReadonlyMap<
      string,
      readonly {
        readonly status: "running" | "succeeded" | "failed" | "interrupted";
        readonly id: string;
        readonly action: string;
        readonly startedAt: string;
        readonly completedAt?: string | undefined;
        readonly error?: string | undefined;
      }[]
    >
  >;
  readonly mainWindowRef: Ref.Ref<Option.Option<BrowserWindow>>;
  readonly encodeJson: ReturnType<typeof createPreviewState>["encodeJson"];
  readonly attemptPromise: ReturnType<typeof createPreviewState>["attemptPromise"];
  readonly attempt: ReturnType<typeof createPreviewState>["attempt"];
  readonly runFork: <A, E>(
    effect: Effect.Effect<A, E, never>,
    options?: Effect.RunOptions | undefined,
  ) => Fiber.Fiber<A, E>;
  readonly requireWebContents: ReturnType<typeof createPreviewState>["requireWebContents"];
  readonly withControlSession: ReturnType<typeof createPreviewBrowserControl>["withControlSession"];
}) => {
  const automationLocator = (input: {
    readonly selector?: string | undefined;
    readonly locator?: string | undefined;
  }): string | null => input.locator ?? (input.selector ? `css=${input.selector}` : null);

  const automationSelectorDiagnostics = (input: {
    readonly selector?: string | undefined;
    readonly locator?: string | undefined;
  }): PreviewSelectorDiagnostics => {
    if (input.locator !== undefined) {
      return { selectorKind: "locator", selectorLength: input.locator.length };
    }

    if (input.selector !== undefined) {
      return { selectorKind: "selector", selectorLength: input.selector.length };
    }

    return { selectorKind: "focused-element" };
  };

  const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")(
    function* (
      tabId: string,
      wc: Electron.WebContents,
      send: SendCommand,
      sendCleanup: SendCommand,
    ) {
      yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], {
        concurrency: 2,
        discard: true,
      });

      const page = yield* evaluateWithDebugger(
        tabId,
        send,
        `(() => {
          const selectorFor = (element) => {
            if (element.id) return "#" + CSS.escape(element.id);
            for (const attribute of ["data-testid", "name"]) {
              const value = element.getAttribute(attribute);
              if (value) return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]";
            }
            const buildParts = (current, parts = []) => {
              if (!current || current.nodeType !== Node.ELEMENT_NODE || parts.length >= 8) {
                return parts;
              }
              const parent = current.parentElement;
              const siblings = parent
                ? Array.from(parent.children).filter((child) => child.tagName === current.tagName)
                : [];
              const base = current.tagName.toLowerCase();
              const part = siblings.length > 1
                ? base + ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"
                : base;
              return buildParts(parent, [part, ...parts]);
            };
            return buildParts(element).join(" > ");
          };
          const visible = (element) => {
            const style = getComputedStyle(element);
            const rect = element.getBoundingClientRect();
            return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0;
          };
          const elements = Array.from(document.querySelectorAll(
            "a[href],button,input,textarea,select,[role],[tabindex]"
          )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => {
            const rect = element.getBoundingClientRect();
            return {
              tag: element.tagName.toLowerCase(),
              role: element.getAttribute("role"),
              name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "",
              selector: selectorFor(element),
              x: rect.x,
              y: rect.y,
              width: rect.width,
              height: rect.height
            };
          });
          return {
            url: location.href,
            title: document.title,
            loading: document.readyState !== "complete",
            visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}),
            interactiveElements: elements
          };
        })()`,
        true,
      ).pipe(Effect.flatMap(decodeEvaluationValue(tabId, decodePage)));

      const [accessibility, initialScreenshotResult, diagnostics, timelines] = yield* Effect.all([
        send("Accessibility.getFullAXTree"),
        send("Page.captureScreenshot", {
          format: "png",
          fromSurface: false,
          captureBeyondViewport: false,
        }),
        Ref.get(diagnosticsRef),
        Ref.get(actionTimelineRef),
      ]);

      const hostScreenshot = yield* Effect.gen(function* () {
        const mainWindow = yield* Ref.get(mainWindowRef);

        if (Option.isNone(mainWindow) || mainWindow.value.isDestroyed()) return null;
        const host = mainWindow.value.webContents;

        if (host.isDestroyed()) return null;

        const tabIdJson = yield* encodeJson(
          { operation: "automationSnapshot.encodeTabId", tabId, webContentsId: wc.id },
          tabId,
        );

        const rawRect = yield* attemptPromise(
          {
            operation: "automationSnapshot.measureHostPreview",
            tabId,
            webContentsId: wc.id,
          },
          () =>
            host.executeJavaScript(`new Promise((resolve) => {
              const deadline = performance.now() + 3000;
              const measure = () => {
                const preview = Array.from(document.querySelectorAll("webview")).find(
                  (element) =>
                    element.getWebContentsId?.() === ${wc.id} ||
                    element.dataset.previewServerTab === ${tabIdJson}
                );
                if (preview) {
                  const style = getComputedStyle(preview);
                  const rect = preview.getBoundingClientRect();
                  if (
                    style.display !== "none" &&
                    style.visibility !== "hidden" &&
                    Number(style.opacity) !== 0 &&
                    rect.width > 0 &&
                    rect.height > 0 &&
                    rect.right > 0 &&
                    rect.bottom > 0 &&
                    rect.left < innerWidth &&
                    rect.top < innerHeight
                  ) {
                    resolve({ x: rect.x, y: rect.y, width: rect.width, height: rect.height });
                    return;
                  }
                }
                if (performance.now() >= deadline) {
                  resolve(null);
                  return;
                }
                requestAnimationFrame(measure);
              };
              measure();
            })`),
        ).pipe(
          Effect.timeout("4 seconds"),
          Effect.orElseSucceed(() => null),
        );

        const rect = normalizeCaptureRect(rawRect);

        if (!rect) return null;

        const captured = yield* attemptPromise(
          {
            operation: "automationSnapshot.captureHostPreview",
            tabId,
            webContentsId: wc.id,
          },
          () => host.capturePage(rect, { stayAwake: true }),
        ).pipe(Effect.orElseSucceed(() => null));

        if (captured && containsVisiblePngPixel(captured.toPNG())) return captured;

        const mediaSourceId = yield* attempt(
          {
            operation: "automationSnapshot.createTabCaptureSource",
            tabId,
            webContentsId: wc.id,
          },
          () => wc.getMediaSourceId(host),
        ).pipe(Effect.orElseSucceed(() => null));

        if (mediaSourceId) {
          const mediaSourceIdJson = yield* encodeJson(
            {
              operation: "automationSnapshot.encodeTabCaptureSource",
              tabId,
              webContentsId: wc.id,
            },
            mediaSourceId,
          );

          const tabCapture = yield* attemptPromise(
            {
              operation: "automationSnapshot.captureTabStream",
              tabId,
              webContentsId: wc.id,
            },
            () =>
              host.executeJavaScript(`(async () => {
                let stream;
                let expired = false;
                const stop = (value) => value?.getTracks().forEach((track) => track.stop());
                try {
                  const capture = navigator.mediaDevices.getUserMedia({
                    audio: false,
                    video: { mandatory: {
                      chromeMediaSource: "tab",
                      chromeMediaSourceId: ${mediaSourceIdJson}
                    }}
                  }).then((value) => {
                    if (!expired) return value;
                    stop(value);
                    throw new Error("Tab capture timed out");
                  });
                  stream = await Promise.race([
                    capture,
                    new Promise((_, reject) => setTimeout(() => {
                      expired = true;
                      reject(new Error("Tab capture timed out"));
                    }, 3000))
                  ]);
                  const video = document.createElement("video");
                  video.muted = true;
                  video.playsInline = true;
                  video.srcObject = stream;
                  await video.play();
                  if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
                    await new Promise((resolve, reject) => {
                      const timer = setTimeout(() => reject(new Error("Tab frame timed out")), 3000);
                      video.addEventListener("loadeddata", () => {
                        clearTimeout(timer);
                        resolve();
                      }, { once: true });
                    });
                  }
                  const canvas = document.createElement("canvas");
                  canvas.width = video.videoWidth;
                  canvas.height = video.videoHeight;
                  const context = canvas.getContext("2d", { alpha: false });
                  if (!context || canvas.width <= 0 || canvas.height <= 0) {
                    throw new Error("Tab capture returned no frame");
                  }
                  context.drawImage(video, 0, 0);
                  return canvas.toDataURL("image/png").slice("data:image/png;base64,".length);
                } finally {
                  stop(stream);
                }
              })()`),
          ).pipe(
            Effect.timeout("4 seconds"),
            Effect.orElseSucceed(() => null),
          );

          if (Predicate.isString(tabCapture) && tabCapture.length > 0) {
            const tabData = Buffer.from(tabCapture, "base64");
            const tabImage = nativeImage.createFromBuffer(tabData);

            if (!tabImage.isEmpty() && containsVisiblePngPixel(tabData)) return tabImage;
          }
        }

        const windowBounds = mainWindow.value.getBounds();
        const contentBounds = mainWindow.value.getContentBounds();

        const sources = yield* attemptPromise(
          {
            operation: "automationSnapshot.captureDesktopWindow",
            tabId,
            webContentsId: wc.id,
          },
          () =>
            desktopCapturer.getSources({
              types: ["window"],
              thumbnailSize: { width: windowBounds.width, height: windowBounds.height },
            }),
        ).pipe(
          Effect.timeout("3 seconds"),
          Effect.orElseSucceed(() => []),
        );

        const source = sources.find(
          (candidate) => candidate.id === mainWindow.value.getMediaSourceId(),
        );

        if (!source || source.thumbnail.isEmpty()) return null;
        const thumbnailSize = source.thumbnail.getSize();
        const desktopRect = scaleCaptureRect(rect, windowBounds, contentBounds, thumbnailSize);

        if (
          desktopRect.x + desktopRect.width > thumbnailSize.width ||
          desktopRect.y + desktopRect.height > thumbnailSize.height
        ) {
          return null;
        }

        const desktopImage = source.thumbnail.crop(desktopRect);

        return !desktopImage.isEmpty() && containsVisiblePngPixel(desktopImage.toPNG())
          ? desktopImage
          : null;
      });

      const refreshedScreenshotResult = yield* send("Page.captureScreenshot", {
        format: "png",
        fromSurface: false,
        captureBeyondViewport: false,
      });

      const refreshedScreenshotData =
        Predicate.isObjectOrArray(refreshedScreenshotResult) &&
        refreshedScreenshotResult !== null &&
        "data" in refreshedScreenshotResult &&
        Predicate.isString(refreshedScreenshotResult.data)
          ? Buffer.from(refreshedScreenshotResult.data, "base64")
          : null;

      const hostScreenshotData = hostScreenshot?.toPNG() ?? null;

      const initialScreenshotData =
        Predicate.isObjectOrArray(initialScreenshotResult) &&
        initialScreenshotResult !== null &&
        "data" in initialScreenshotResult &&
        Predicate.isString(initialScreenshotResult.data)
          ? Buffer.from(initialScreenshotResult.data, "base64")
          : null;

      const fallbackScreenshotResult =
        refreshedScreenshotData && containsVisiblePngPixel(refreshedScreenshotData)
          ? refreshedScreenshotResult
          : hostScreenshotData && containsVisiblePngPixel(hostScreenshotData)
            ? { data: hostScreenshotData.toString("base64") }
            : initialScreenshotData && containsVisiblePngPixel(initialScreenshotData)
              ? initialScreenshotResult
              : null;

      const screencastScreenshotResult = fallbackScreenshotResult
        ? null
        : yield* Effect.gen(function* () {
            const screencastFrame = yield* Deferred.make<{ readonly data: string }>();

            const onScreencastMessage = (
              _event: Electron.Event,
              method: string,
              params: Record<string, Schema.Json | undefined>,
            ) => {
              if (method !== "Page.screencastFrame" || !Predicate.isString(params["data"])) return;
              const data = Buffer.from(params["data"], "base64");

              if (!containsVisiblePngPixel(data)) return;
              runFork(
                Deferred.succeed(screencastFrame, { data: params["data"] }).pipe(Effect.asVoid),
              );
            };

            return yield* Effect.acquireUseRelease(
              attempt(
                {
                  operation: "automationSnapshot.listenForScreencastFrame",
                  tabId,
                  webContentsId: wc.id,
                },
                () => wc.debugger.on("message", onScreencastMessage),
              ),
              () =>
                Effect.gen(function* () {
                  yield* send("Page.bringToFront");
                  yield* send("Page.startScreencast", {
                    format: "png",
                    maxWidth: MAX_SCREENSHOT_WIDTH,
                    everyNthFrame: 1,
                  });

                  return yield* Deferred.await(screencastFrame).pipe(Effect.timeout("3 seconds"));
                }).pipe(Effect.orElseSucceed(() => null)),
              () =>
                attempt(
                  {
                    operation: "automationSnapshot.removeScreencastListener",
                    tabId,
                    webContentsId: wc.id,
                  },
                  () => wc.debugger.off("message", onScreencastMessage),
                ).pipe(
                  Effect.ignore,
                  Effect.andThen(sendCleanup("Page.stopScreencast").pipe(Effect.ignore)),
                ),
            );
          });

      const screenshotResult =
        screencastScreenshotResult ?? fallbackScreenshotResult ?? initialScreenshotResult;

      const screenshot = yield* Effect.try({
        try: () => {
          if (
            !Predicate.isObjectOrArray(screenshotResult) ||
            screenshotResult === null ||
            !("data" in screenshotResult) ||
            !Predicate.isString(screenshotResult.data) ||
            screenshotResult.data.length === 0
          ) {
            throw new TypeError("Page.captureScreenshot returned no PNG data");
          }

          const sourceData = Buffer.from(screenshotResult.data, "base64");
          const image = nativeImage.createFromBuffer(sourceData);

          if (image.isEmpty()) {
            throw new TypeError("Page.captureScreenshot returned an invalid PNG");
          }

          if (!containsVisiblePngPixel(sourceData)) {
            throw new TypeError("Page.captureScreenshot returned an all-black PNG");
          }

          const sourceSize = image.getSize();

          if (
            !Number.isFinite(sourceSize.width) ||
            !Number.isFinite(sourceSize.height) ||
            sourceSize.width <= 0 ||
            sourceSize.height <= 0
          ) {
            throw new TypeError("Page.captureScreenshot returned invalid image dimensions");
          }

          const output =
            sourceSize.width > MAX_SCREENSHOT_WIDTH
              ? image.resize({ width: MAX_SCREENSHOT_WIDTH })
              : image;

          if (output.isEmpty()) {
            throw new TypeError("Page.captureScreenshot could not be resized");
          }

          const size = output.getSize();

          if (
            !Number.isFinite(size.width) ||
            !Number.isFinite(size.height) ||
            size.width <= 0 ||
            size.height <= 0
          ) {
            throw new TypeError("Page.captureScreenshot produced invalid output dimensions");
          }

          const data = sourceSize.width > MAX_SCREENSHOT_WIDTH ? output.toPNG() : sourceData;

          if (data.byteLength === 0) {
            throw new TypeError("Page.captureScreenshot produced an empty PNG");
          }

          return {
            mimeType: "image/png" as const,
            data: data.toString("base64"),
            width: size.width,
            height: size.height,
          };
        },
        catch: (cause) =>
          new PreviewOperationError({
            operation: "automationSnapshot.decodeScreenshot",
            tabId,
            webContentsId: wc.id,
            cause,
          }),
      });

      const browserDiagnostics = diagnostics.get(wc.id);

      return {
        ...page,
        accessibilityTree: accessibility,
        consoleEntries: [...(browserDiagnostics?.consoleEntries ?? [])],
        networkEntries: [...(browserDiagnostics?.networkEntries ?? [])],
        actionTimeline: [...(timelines.get(tabId) ?? [])],
        screenshot,
      };
    },
  );

  const automationSnapshot = Effect.fn("PreviewManager.automationSnapshot")(function* (
    tabId: string,
  ) {
    const wc = yield* requireWebContents(tabId);

    return yield* withControlSession(tabId, wc, "snapshot", (send, sendCleanup) =>
      captureAutomationSnapshot(tabId, wc, send, sendCleanup),
    );
  });

  return { automationLocator, automationSelectorDiagnostics, automationSnapshot };
};