Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import { decodeEvaluationValue } from "./PreviewEvaluation.ts";
import * as Schema from "effect/Schema";
import type {
DesktopPreviewPointerEvent,
PreviewAutomationClickInput,
PreviewAutomationPressInput,
PreviewAutomationTypeInput,
} from "@akeru/contracts";
import { webContents } from "electron";
import * as Effect from "effect/Effect";
import * as Ref from "effect/Ref";
import { previewAutomationKeySequence } from "./PreviewKeyboard.ts";
import {
PreviewAutomationTargetNotFoundError,
PreviewAutomationTargetNotEditableError,
PreviewAutomationCoordinatesOutsideViewportError,
PreviewAutomationInvalidSelectorError,
} from "./PreviewErrors.ts";
import type { createPreviewAutomationSnapshot } from "./PreviewAutomationSnapshot.ts";
import type { createPreviewBrowserControl } from "./PreviewBrowserControl.ts";
import type { createPreviewState } from "./PreviewState.ts";
import {
AGENT_CURSOR_MOVE_MS,
AGENT_CURSOR_CLICK_LEAD_MS,
type PointerEventListener,
type SendCommand,
} from "./PreviewModel.ts";
const decodeClickPoint = Schema.decodeUnknownEffect(
Schema.Union([
Schema.Struct({ x: Schema.Number, y: Schema.Number }),
Schema.Struct({ invalidSelector: Schema.Literal(true), message: Schema.String }),
Schema.Struct({ notFound: Schema.Literal(true) }),
]),
);
const decodeViewport = Schema.decodeUnknownEffect(
Schema.Struct({ width: Schema.Number, height: Schema.Number }),
);
const decodeTypeResult = Schema.decodeUnknownEffect(
Schema.Union([
Schema.Struct({ ok: Schema.Literal(true) }),
Schema.Struct({ invalidSelector: Schema.Literal(true), message: Schema.String }),
Schema.Struct({ notEditable: Schema.Literal(true) }),
Schema.Struct({ notFound: Schema.Literal(true) }),
]),
);
export const createPreviewAutomationInput = ({
automationLocator,
ensurePlaywrightInjected,
encodeJson,
evaluateWithDebugger,
automationSelectorDiagnostics,
pointerEventListenersRef,
deliverEvent,
prepareAutomationInput,
nextCounter,
pointerSequenceRef,
currentIso,
expectAgentInput,
requireWebContents,
withControlSession,
hostPlatform,
attempt,
}: {
readonly automationLocator: ReturnType<
typeof createPreviewAutomationSnapshot
>["automationLocator"];
readonly ensurePlaywrightInjected: ReturnType<
typeof createPreviewBrowserControl
>["ensurePlaywrightInjected"];
readonly encodeJson: ReturnType<typeof createPreviewState>["encodeJson"];
readonly evaluateWithDebugger: ReturnType<
typeof createPreviewBrowserControl
>["evaluateWithDebugger"];
readonly automationSelectorDiagnostics: ReturnType<
typeof createPreviewAutomationSnapshot
>["automationSelectorDiagnostics"];
readonly pointerEventListenersRef: Ref.Ref<ReadonlySet<PointerEventListener>>;
readonly deliverEvent: ReturnType<typeof createPreviewState>["deliverEvent"];
readonly prepareAutomationInput: ReturnType<
typeof createPreviewBrowserControl
>["prepareAutomationInput"];
readonly nextCounter: ReturnType<typeof createPreviewState>["nextCounter"];
readonly pointerSequenceRef: Ref.Ref<number>;
readonly currentIso: ReturnType<typeof createPreviewState>["currentIso"];
readonly expectAgentInput: ReturnType<typeof createPreviewBrowserControl>["expectAgentInput"];
readonly requireWebContents: ReturnType<typeof createPreviewState>["requireWebContents"];
readonly withControlSession: ReturnType<typeof createPreviewBrowserControl>["withControlSession"];
readonly hostPlatform: NodeJS.Platform;
readonly attempt: ReturnType<typeof createPreviewState>["attempt"];
}) => {
const resolveClickPoint = Effect.fn("PreviewManager.resolveClickPoint")(function* (
tabId: string,
send: SendCommand,
input: PreviewAutomationClickInput,
) {
if (!("selector" in input) && !("locator" in input)) {
return { x: input.x!, y: input.y! };
}
const locator = automationLocator(input)!;
yield* ensurePlaywrightInjected(tabId, send);
const locatorJson = yield* encodeJson(
{ operation: "automationClick.encodeLocator", tabId },
locator,
);
const point = yield* evaluateWithDebugger(
tabId,
send,
`(() => {
try {
const injected = globalThis.__t3PlaywrightInjected;
const parsed = injected.parseSelector(${locatorJson});
const element = injected.querySelector(parsed, document, true);
if (!element) return { notFound: true };
const visible = injected.elementState(element, "visible");
const enabled = injected.elementState(element, "enabled");
if (!visible.matches || !enabled.matches) return { notFound: true };
element.scrollIntoView({ block: "center", inline: "center" });
const rect = element.getBoundingClientRect();
return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
} catch (error) {
return { invalidSelector: true, message: String(error) };
}
})()`,
true,
).pipe(Effect.flatMap(decodeEvaluationValue(tabId, decodeClickPoint)));
if ("invalidSelector" in point) {
return yield* new PreviewAutomationInvalidSelectorError({
operation: "click",
tabId,
...automationSelectorDiagnostics(input),
reasonLength: point.message.length,
cause: point,
});
}
if ("notFound" in point) {
return yield* new PreviewAutomationTargetNotFoundError({
operation: "click",
tabId,
...automationSelectorDiagnostics(input),
});
}
return point;
});
const emitPointerEvent = Effect.fn("PreviewManager.emitPointerEvent")(function* (
event: DesktopPreviewPointerEvent,
) {
const listeners = yield* Ref.get(pointerEventListenersRef);
yield* Effect.forEach(
listeners,
(listener) => deliverEvent("pointer-event", event.tabId, () => listener(event)),
{ discard: true },
);
});
const performAutomationClick = Effect.fn("PreviewManager.performAutomationClick")(function* (
tabId: string,
input: PreviewAutomationClickInput,
send: SendCommand,
) {
yield* prepareAutomationInput(send, true);
const point = yield* resolveClickPoint(tabId, send, input);
const viewport = yield* evaluateWithDebugger(
tabId,
send,
"({ width: window.innerWidth, height: window.innerHeight })",
true,
).pipe(Effect.flatMap(decodeEvaluationValue(tabId, decodeViewport)));
if (point.x < 0 || point.y < 0 || point.x > viewport.width || point.y > viewport.height) {
return yield* new PreviewAutomationCoordinatesOutsideViewportError({
tabId,
x: point.x,
y: point.y,
viewportWidth: viewport.width,
viewportHeight: viewport.height,
});
}
const moveSequence = yield* nextCounter(pointerSequenceRef);
const moveCreatedAt = yield* currentIso;
yield* emitPointerEvent({
tabId,
phase: "move",
...point,
sequence: moveSequence,
createdAt: moveCreatedAt,
});
yield* Effect.sleep(AGENT_CURSOR_MOVE_MS);
const clickSequence = yield* nextCounter(pointerSequenceRef);
const clickCreatedAt = yield* currentIso;
yield* emitPointerEvent({
tabId,
phase: "click",
...point,
sequence: clickSequence,
createdAt: clickCreatedAt,
});
yield* Effect.sleep(AGENT_CURSOR_CLICK_LEAD_MS);
yield* expectAgentInput(tabId, { kind: "pointer", ...point, button: 0 });
yield* send("Input.dispatchMouseEvent", {
type: "mousePressed",
...point,
button: "left",
clickCount: 1,
});
yield* send("Input.dispatchMouseEvent", {
type: "mouseReleased",
...point,
button: "left",
clickCount: 1,
});
});
const automationClick = Effect.fn("PreviewManager.automationClick")(function* (
tabId: string,
input: PreviewAutomationClickInput,
) {
const wc = yield* requireWebContents(tabId);
yield* withControlSession(tabId, wc, "click", (send) =>
performAutomationClick(tabId, input, send),
);
});
const typeIntoAutomationTarget = Effect.fn("PreviewManager.typeIntoAutomationTarget")(function* (
tabId: string,
send: SendCommand,
input: PreviewAutomationTypeInput,
) {
const locator = automationLocator(input);
if (locator) yield* ensurePlaywrightInjected(tabId, send);
const locatorJson = locator
? yield* encodeJson({ operation: "automationType.encodeLocator", tabId }, locator)
: null;
const textJson = yield* encodeJson(
{ operation: "automationType.encodeText", tabId },
input.text,
);
const result = yield* evaluateWithDebugger(
tabId,
send,
`(() => {
try {
const element = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, true); })()` : "document.activeElement"};
if (!element) return { notFound: true };
const textControl =
element instanceof HTMLTextAreaElement ||
(element instanceof HTMLInputElement &&
!new Set(["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"]).has(element.type));
const editable = textControl || element.isContentEditable;
if (!editable || element.disabled || element.readOnly) return { notEditable: true };
element.focus();
if (document.activeElement !== element) return { notEditable: true };
const clear = ${input.clear ?? false};
if (clear) {
if (textControl) {
element.select();
} else {
const range = document.createRange();
range.selectNodeContents(element);
const selection = document.getSelection();
selection?.removeAllRanges();
selection?.addRange(range);
}
}
const text = ${textJson};
let inserted = true;
if (text.length > 0) {
inserted = document.execCommand("insertText", false, text);
} else if (clear) {
document.execCommand("delete", false);
const cleared = textControl
? element.value.length === 0
: (element.textContent ?? "").length === 0;
if (!cleared) {
if (textControl) {
const prototype = element instanceof HTMLTextAreaElement
? HTMLTextAreaElement.prototype
: HTMLInputElement.prototype;
const valueSetter = Object.getOwnPropertyDescriptor(prototype, "value")?.set;
if (valueSetter) valueSetter.call(element, "");
else element.value = "";
} else {
element.replaceChildren();
}
element.dispatchEvent(new InputEvent("input", {
bubbles: true,
inputType: "deleteContentBackward",
}));
}
}
if (!inserted) return { notEditable: true };
element.dispatchEvent(new Event("change", { bubbles: true }));
return { ok: true };
} catch (error) {
return { invalidSelector: true, message: String(error) };
}
})()`,
true,
).pipe(Effect.flatMap(decodeEvaluationValue(tabId, decodeTypeResult)));
if ("invalidSelector" in result) {
return yield* new PreviewAutomationInvalidSelectorError({
operation: "type",
tabId,
...automationSelectorDiagnostics(input),
reasonLength: result.message.length,
cause: result,
});
}
if ("notFound" in result) {
return yield* new PreviewAutomationTargetNotFoundError({
operation: "type",
tabId,
...automationSelectorDiagnostics(input),
});
}
if ("notEditable" in result) {
return yield* new PreviewAutomationTargetNotEditableError({
tabId,
...automationSelectorDiagnostics(input),
});
}
});
const performAutomationType = Effect.fn("PreviewManager.performAutomationType")(function* (
tabId: string,
input: PreviewAutomationTypeInput,
send: SendCommand,
) {
// CDP Input.insertText silently drops text until Electron has activated a hidden
// guest WebContents with a pointer event. Editing in the page runtime keeps
// background automation deterministic without stealing foreground app focus.
yield* typeIntoAutomationTarget(tabId, send, input);
});
const automationType = Effect.fn("PreviewManager.automationType")(function* (
tabId: string,
input: PreviewAutomationTypeInput,
) {
const wc = yield* requireWebContents(tabId);
yield* withControlSession(tabId, wc, "type", (send) =>
performAutomationType(tabId, input, send),
);
});
const performAutomationPress = Effect.fn("PreviewManager.performAutomationPress")(function* (
tabId: string,
wc: Electron.WebContents,
input: PreviewAutomationPressInput,
send: SendCommand,
sendCleanup: SendCommand,
) {
yield* prepareAutomationInput(send, false);
const keySequence = previewAutomationKeySequence(input, {
isMac: hostPlatform === "darwin",
});
const previouslyFocused = yield* attempt(
{ operation: "automationPress.getFocusedWebContents", tabId, webContentsId: wc.id },
() => webContents.getFocusedWebContents(),
);
let keyDownAttempted = false;
const releaseInput = Effect.gen(function* () {
if (keyDownAttempted) {
yield* sendCleanup("Input.dispatchKeyEvent", keySequence.keyUp).pipe(Effect.ignore);
}
yield* sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe(
Effect.ignore,
);
if (previouslyFocused && previouslyFocused.id !== wc.id && !previouslyFocused.isDestroyed()) {
yield* attempt(
{
operation: "automationPress.restoreFocusedWebContents",
tabId,
webContentsId: previouslyFocused.id,
},
() => previouslyFocused.focus(),
).pipe(Effect.ignore);
}
});
// Focus the guest WebContents itself, not its containing BrowserWindow. This
// activates native keyboard behavior for hidden/background previews without
// changing which thread is mounted in the UI. Restore the previous renderer
// after dispatch so automation never leaves the app's input focus behind.
yield* Effect.gen(function* () {
yield* attempt(
{ operation: "automationPress.focusWebContents", tabId, webContentsId: wc.id },
() => wc.focus(),
);
yield* send("Page.bringToFront");
yield* send("Emulation.setFocusEmulationEnabled", { enabled: true });
yield* expectAgentInput(tabId, keySequence.signal);
keyDownAttempted = true;
yield* send("Input.dispatchKeyEvent", keySequence.keyDown);
}).pipe(Effect.ensuring(releaseInput));
});
const automationPress = Effect.fn("PreviewManager.automationPress")(function* (
tabId: string,
input: PreviewAutomationPressInput,
) {
const wc = yield* requireWebContents(tabId);
yield* withControlSession(tabId, wc, "press", (send, sendCleanup) =>
performAutomationPress(tabId, wc, input, send, sendCleanup),
);
});
return { automationClick, automationType, automationPress };
};