Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

23.9 KB
Raw Blame
import { installGuestInputListeners } from "./GuestInput.ts";
import {
  createAnnotationOverlay,
  applyAnnotationTheme,
  PRIMARY_FILL,
} from "./AnnotationOverlay.ts";
import { createAnnotationStyleControls } from "./AnnotationStyleControls.ts";

import { ipcRenderer } from "electron";

import type {
  DesktopPreviewAnnotationTheme,
  PreviewAnnotationPayload,
  PreviewAnnotationPoint,
  PreviewAnnotationRect,
  PreviewAnnotationRegionTarget,
  PreviewAnnotationStrokeTarget,
  PreviewAnnotationStyleChange,
  PreviewAnnotationSubmission,
} from "@akeru/contracts";

import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts";

import { selectMarqueeElements } from "./MarqueeSelection.ts";

import {
  ANNOTATION_CAPTURED_CHANNEL,
  ANNOTATION_THEME_CHANNEL,
  CANCEL_PICK_CHANNEL,
  ELEMENT_PICKED_CHANNEL,
  START_PICK_CHANNEL,
} from "./GuestProtocol.ts";

import { PRIMARY, createButton } from "./AnnotationStyleControls.ts";

import {
  OVERLAY_ATTRIBUTE,
  type SelectedElement,
  rectFromDomRect,
  normalizeRect,
  isUsableRect,
  unionRects,
  isAnnotationNode,
  pickFromPoint,
  createBox,
  positionBox,
  createLabel,
  updateSelectedVisual,
  pathFromPoints,
  strokeBounds,
} from "./AnnotationGeometry.ts";

import { captureElement } from "./AnnotationElementCapture.ts";

const MAX_MARQUEE_ELEMENTS = 20;

type AnnotationTool = "select" | "marquee" | "draw" | "erase";

interface AnnotationSession {
  teardown: (notifyMain: boolean) => void;
  applyTheme: (theme: DesktopPreviewAnnotationTheme) => void;
}

let activeSession: AnnotationSession | null = null;

let idSequence = 0;

let annotationTheme: DesktopPreviewAnnotationTheme | null = null;

installGuestInputListeners();

const nextId = (prefix: string): string => {
  idSequence += 1;

  return `${prefix}_${idSequence.toString(36)}`;
};

function startAnnotation(): void {
  activeSession?.teardown(false);
  let finished = false;

  const {
    host,
    root,
    cursorStyle,
    hoverOutline,
    marqueeBox,
    svg,
    toolbar,
    editor,
    adjust,
    comment,
    dragHandle,
    submit,
    stylePanel,
  } = createAnnotationOverlay(annotationTheme);

  const selected = new Map<Element, SelectedElement>();
  const regions: PreviewAnnotationRegionTarget[] = [];
  const strokes: PreviewAnnotationStrokeTarget[] = [];
  const styleChanges = new Map<string, PreviewAnnotationStyleChange>();
  const toolButtons = new Map<AnnotationTool, HTMLButtonElement>();
  let tool: AnnotationTool = "select";
  let dragStart: PreviewAnnotationPoint | null = null;
  let activeStroke: { target: PreviewAnnotationStrokeTarget; path: SVGPathElement } | null = null;
  let pendingCapture = false;
  let editorExpanded = false;
  let editorWasShown = false;
  let editorPosition: { left: number; top: number } | null = null;
  let editorDrag: { pointerId: number; offsetX: number; offsetY: number } | null = null;
  let editorLayoutFrame: number | null = null;

  const resizeComment = (): void => {
    const maxHeight = 96;
    comment.style.height = "auto";
    const nextHeight = Math.min(comment.scrollHeight, maxHeight);
    comment.style.height = `${nextHeight}px`;
    comment.style.overflowY = comment.scrollHeight > maxHeight ? "auto" : "hidden";
    queueEditorLayout();
  };

  comment.addEventListener("input", resizeComment);

  const updateStatus = (): void => {
    const hasTargets = selected.size > 0 || regions.length > 0 || strokes.length > 0;
    editor.style.display = hasTargets ? "flex" : "none";
    submit.disabled = !hasTargets;
    submit.style.opacity = hasTargets ? "1" : "0.45";
    adjust.disabled = !hasTargets;
    stylePanel.style.display = editorExpanded && selected.size > 0 ? "grid" : "none";
    queueEditorLayout();

    if (hasTargets && !editorWasShown) {
      editorWasShown = true;
      window.setTimeout(() => comment.focus({ preventScroll: true }), 0);
    }
  };

  const refreshToolButtons = (): void => {
    for (const [candidate, button] of toolButtons) {
      const active = candidate === tool;
      button.classList.toggle("bg-primary/10", active);
      button.classList.toggle("text-primary", active);
      button.classList.toggle("text-foreground", !active);
    }

    if (tool !== "select") hoverOutline.style.display = "none";

    if (tool !== "marquee") marqueeBox.style.display = "none";
    document.documentElement.setAttribute("data-t3code-annotation-tool", tool);
  };

  const removeSelected = (target: SelectedElement): void => {
    if (target.element instanceof HTMLElement || target.element instanceof SVGElement) {
      for (const [property, baseline] of target.baselineStyles) {
        if (baseline) target.element.style.setProperty(property, baseline);
        else target.element.style.removeProperty(property);
      }
    }

    selected.delete(target.element);
    target.outline.remove();
    target.label.remove();

    for (const [key, change] of styleChanges) {
      if (change.targetId === target.id) styleChanges.delete(key);
    }

    updateStatus();
  };

  const addSelected = (element: Element): void => {
    if (selected.has(element)) return;

    const target: SelectedElement = {
      id: nextId("element"),
      element,
      outline: createBox(PRIMARY, PRIMARY_FILL),
      label: createLabel(),
      baselineStyles: new Map(),
    };

    selected.set(element, target);
    root.append(target.outline, target.label);
    updateSelectedVisual(target);
    updateStatus();

    if (editorExpanded) {
      stylePanel.style.display = "grid";
      syncStyleControls();
    }
  };

  const toggleSelected = (element: Element, additive: boolean): void => {
    const existing = selected.get(element);

    if (existing) {
      removeSelected(existing);

      return;
    }

    if (!additive) {
      for (const target of Array.from(selected.values())) removeSelected(target);
    }

    addSelected(element);
  };

  const setStyleForSelected = (property: string, value: string): void => {
    for (const target of selected.values()) {
      if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement))
        continue;

      if (!target.baselineStyles.has(property)) {
        target.baselineStyles.set(property, target.element.style.getPropertyValue(property));
      }

      const key = `${target.id}:${property}`;

      const previousValue =
        styleChanges.get(key)?.previousValue ??
        getComputedStyle(target.element).getPropertyValue(property).trim();

      target.element.style.setProperty(property, value, "important");
      styleChanges.set(key, {
        targetId: target.id,
        selector: null,
        property,
        previousValue,
        value,
      });
      updateSelectedVisual(target);
    }
  };

  const syncStyleControls = createAnnotationStyleControls({
    panel: stylePanel,
    selected,
    setStyleForSelected,
  });

  const tools: ReadonlyArray<[AnnotationTool, string, string]> = [
    ["select", "Select", "Select elements (V)"],
    ["marquee", "Region", "Draw a region or marquee-select elements (R)"],
    ["draw", "Draw", "Draw freehand (D)"],
    ["erase", "Erase", "Remove an annotation target (E)"],
  ];

  for (const [candidate, label, title] of tools) {
    const button = createButton(label, title);
    button.className += " h-8 px-2.5 text-sm";
    button.addEventListener("click", () => {
      tool = candidate;
      refreshToolButtons();
    });
    toolButtons.set(candidate, button);
    toolbar.appendChild(button);
  }

  const clampEditorPosition = (left: number, top: number) => {
    const margin = 8;
    const rect = editor.getBoundingClientRect();

    return {
      left: Math.min(
        Math.max(margin, left),
        Math.max(margin, window.innerWidth - rect.width - margin),
      ),
      top: Math.min(
        Math.max(margin, top),
        Math.max(margin, window.innerHeight - rect.height - margin),
      ),
    };
  };

  const applyEditorPosition = (position: { left: number; top: number }): void => {
    const clamped = clampEditorPosition(position.left, position.top);
    editor.style.left = `${clamped.left}px`;
    editor.style.top = `${clamped.top}px`;
    editor.style.right = "auto";
    editor.style.bottom = "auto";

    if (editorExpanded) editorPosition = clamped;
  };

  const getAnnotationBounds = (): PreviewAnnotationRect | null =>
    unionRects(
      [
        ...Array.from(selected.values(), (target) =>
          rectFromDomRect(target.element.getBoundingClientRect()),
        ),
        ...regions.map((region) => region.rect),
        ...strokes.map((stroke) => stroke.bounds),
      ],
      0,
    );

  const positionCompactEditor = (): void => {
    const bounds = getAnnotationBounds();

    if (!bounds) return;
    const editorRect = editor.getBoundingClientRect();
    const gap = 8;

    const candidates = [
      { left: bounds.x + bounds.width + gap, top: bounds.y },
      { left: bounds.x - editorRect.width - gap, top: bounds.y },
      {
        left: bounds.x + bounds.width - editorRect.width,
        top: bounds.y + bounds.height + gap,
      },
      {
        left: bounds.x + bounds.width - editorRect.width,
        top: bounds.y - editorRect.height - gap,
      },
    ];

    const overflow = (position: { left: number; top: number }): number =>
      Math.max(0, -position.left) +
      Math.max(0, -position.top) +
      Math.max(0, position.left + editorRect.width - window.innerWidth) +
      Math.max(0, position.top + editorRect.height - window.innerHeight);

    const best = candidates.reduce((current, candidate) =>
      overflow(candidate) < overflow(current) ? candidate : current,
    );

    applyEditorPosition(best);
  };

  function queueEditorLayout(): void {
    if (editorLayoutFrame !== null) window.cancelAnimationFrame(editorLayoutFrame);
    editorLayoutFrame = window.requestAnimationFrame(() => {
      editorLayoutFrame = null;

      if (editor.style.display === "none") return;

      if (editorExpanded && editorPosition) applyEditorPosition(editorPosition);
      else positionCompactEditor();
    });
  }

  adjust.addEventListener("click", () => {
    if (selected.size === 0) return;

    if (!editorExpanded) {
      const rect = editor.getBoundingClientRect();
      editorExpanded = true;
      editorPosition = { left: rect.left, top: rect.top };
      stylePanel.style.display = selected.size > 0 ? "grid" : "none";
      dragHandle.style.display = "block";
      adjust.setAttribute("aria-expanded", "true");
      adjust.title = "Collapse annotation editor";
      adjust.setAttribute("aria-label", "Collapse annotation editor");

      if (selected.size > 0) syncStyleControls();
    } else {
      editorExpanded = false;
      editorPosition = null;
      stylePanel.style.display = "none";
      dragHandle.style.display = "none";
      adjust.setAttribute("aria-expanded", "false");
      adjust.title = "Expand annotation editor";
      adjust.setAttribute("aria-label", "Expand annotation editor");
    }

    queueEditorLayout();
  });

  const onEditorPointerDown = (event: PointerEvent): void => {
    if (event.button !== 0 || !editorExpanded) return;
    const rect = editor.getBoundingClientRect();
    editorDrag = {
      pointerId: event.pointerId,
      offsetX: event.clientX - rect.left,
      offsetY: event.clientY - rect.top,
    };
    dragHandle.setPointerCapture(event.pointerId);
    dragHandle.style.cursor = "grabbing";
    event.preventDefault();
    event.stopPropagation();
  };

  const onEditorPointerMove = (event: PointerEvent): void => {
    if (!editorDrag || editorDrag.pointerId !== event.pointerId) return;
    applyEditorPosition({
      left: event.clientX - editorDrag.offsetX,
      top: event.clientY - editorDrag.offsetY,
    });
    event.preventDefault();
    event.stopPropagation();
  };

  const onEditorPointerUp = (event: PointerEvent): void => {
    if (!editorDrag || editorDrag.pointerId !== event.pointerId) return;
    editorDrag = null;
    dragHandle.style.cursor = "grab";

    if (dragHandle.hasPointerCapture(event.pointerId))
      dragHandle.releasePointerCapture(event.pointerId);
    event.preventDefault();
    event.stopPropagation();
  };

  dragHandle.addEventListener("pointerdown", onEditorPointerDown);
  dragHandle.addEventListener("pointermove", onEditorPointerMove);
  dragHandle.addEventListener("pointerup", onEditorPointerUp);
  dragHandle.addEventListener("pointercancel", onEditorPointerUp);

  const repaint = (): void => {
    for (const target of selected.values()) updateSelectedVisual(target);
    queueEditorLayout();
  };

  const removeTargetAtPoint = (x: number, y: number): boolean => {
    for (const target of Array.from(selected.values()).toReversed()) {
      const rect = target.element.getBoundingClientRect();

      if (x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) {
        removeSelected(target);

        return true;
      }
    }

    const regionIndex = regions.findIndex(
      (region) =>
        x >= region.rect.x &&
        x <= region.rect.x + region.rect.width &&
        y >= region.rect.y &&
        y <= region.rect.y + region.rect.height,
    );

    if (regionIndex >= 0) {
      const [removed] = regions.splice(regionIndex, 1);
      root.querySelector(`[data-region-id="${removed?.id}"]`)?.remove();
      updateStatus();

      return true;
    }

    const strokeIndex = strokes.findIndex(
      (stroke) =>
        x >= stroke.bounds.x &&
        x <= stroke.bounds.x + stroke.bounds.width &&
        y >= stroke.bounds.y &&
        y <= stroke.bounds.y + stroke.bounds.height,
    );

    if (strokeIndex >= 0) {
      const [removed] = strokes.splice(strokeIndex, 1);
      svg.querySelector(`[data-stroke-id="${removed?.id}"]`)?.remove();
      updateStatus();

      return true;
    }

    return false;
  };

  const selectElementsInRect = (rect: PreviewAnnotationRect): number => {
    const candidates = selectMarqueeElements({
      elements: document.querySelectorAll("body *"),
      rect,
      limit: MAX_MARQUEE_ELEMENTS,
      eligible: (element) =>
        !isAnnotationNode(element) &&
        (element.children.length === 0 ||
          element instanceof HTMLButtonElement ||
          element instanceof HTMLAnchorElement ||
          element.getAttribute("role") === "button"),
      measure: (element) => element.getBoundingClientRect(),
    });

    for (const candidate of candidates) addSelected(candidate);

    return candidates.length;
  };

  const clearHoverOutline = (): void => {
    hoverOutline.style.display = "none";
  };

  const onPointerMove = (event: PointerEvent): void => {
    if (isAnnotationNode(event.target)) {
      clearHoverOutline();

      return;
    }

    if (tool === "select" && dragStart === null) {
      const target = pickFromPoint(event.clientX, event.clientY);

      if (target) positionBox(hoverOutline, rectFromDomRect(target.getBoundingClientRect()));
      else clearHoverOutline();

      return;
    }

    clearHoverOutline();

    if (tool === "marquee" && dragStart) {
      positionBox(
        marqueeBox,
        normalizeRect(dragStart.x, dragStart.y, event.clientX, event.clientY),
      );

      return;
    }

    if (tool === "draw" && activeStroke) {
      activeStroke.target.points = [
        ...activeStroke.target.points,
        { x: event.clientX, y: event.clientY },
      ];
      activeStroke.target.bounds = strokeBounds(
        activeStroke.target.points,
        activeStroke.target.width,
      );
      activeStroke.path.setAttribute("d", pathFromPoints(activeStroke.target.points));
    }
  };

  const onPointerDown = (event: PointerEvent): void => {
    if (event.button !== 0 || isAnnotationNode(event.target)) return;
    event.preventDefault();
    event.stopPropagation();

    if (tool === "select") {
      const target = pickFromPoint(event.clientX, event.clientY);

      if (target) toggleSelected(target, event.shiftKey);

      return;
    }

    if (tool === "erase") {
      removeTargetAtPoint(event.clientX, event.clientY);

      return;
    }

    dragStart = { x: event.clientX, y: event.clientY };

    if (tool === "draw") {
      const stroke: PreviewAnnotationStrokeTarget = {
        id: nextId("stroke"),
        color: annotationTheme?.primary ?? "#2563eb",
        width: 4,
        points: [dragStart],
        bounds: { x: dragStart.x, y: dragStart.y, width: 1, height: 1 },
      };

      const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
      path.setAttribute(OVERLAY_ATTRIBUTE, "");
      path.setAttribute("data-stroke-id", stroke.id);
      path.setAttribute("fill", "none");
      path.setAttribute("stroke", stroke.color);
      path.setAttribute("stroke-width", String(stroke.width));
      path.setAttribute("stroke-linecap", "round");
      path.setAttribute("stroke-linejoin", "round");
      svg.appendChild(path);
      activeStroke = { target: stroke, path };
    }
  };

  const onPointerUp = (event: PointerEvent): void => {
    if (!dragStart) return;
    event.preventDefault();
    event.stopPropagation();

    if (tool === "marquee") {
      const rect = normalizeRect(dragStart.x, dragStart.y, event.clientX, event.clientY);
      marqueeBox.style.display = "none";

      if (isUsableRect(rect)) {
        const found = selectElementsInRect(rect);

        if (found === 0) {
          const region: PreviewAnnotationRegionTarget = { id: nextId("region"), rect };
          regions.push(region);

          const regionBox = createBox(
            PRIMARY,
            "color-mix(in srgb, var(--t3-primary) 6%, transparent)",
          );

          regionBox.setAttribute("data-region-id", region.id);
          positionBox(regionBox, rect);
          root.appendChild(regionBox);
        }
      }
    } else if (tool === "draw" && activeStroke) {
      if (activeStroke.target.points.length > 1) strokes.push(activeStroke.target);
      else activeStroke.path.remove();
      activeStroke = null;
    }

    dragStart = null;
    updateStatus();
  };

  const onClick = (event: MouseEvent): void => {
    if (isAnnotationNode(event.target)) return;
    event.preventDefault();
    event.stopPropagation();
  };

  const onPointerOut = (event: PointerEvent): void => {
    if (event.relatedTarget === null) clearHoverOutline();
  };

  const onWindowBlur = (): void => {
    clearHoverOutline();
  };

  const restoreStyles = (): void => {
    for (const target of selected.values()) {
      if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement))
        continue;

      for (const [property, baseline] of target.baselineStyles) {
        if (baseline) target.element.style.setProperty(property, baseline);
        else target.element.style.removeProperty(property);
      }
    }
  };

  const teardown = (notifyMain: boolean): void => {
    if (finished) return;
    finished = true;
    restoreStyles();
    window.removeEventListener("pointermove", onPointerMove, true);
    window.removeEventListener("pointerdown", onPointerDown, true);
    window.removeEventListener("pointerup", onPointerUp, true);
    window.removeEventListener("pointerout", onPointerOut, true);
    window.removeEventListener("click", onClick, true);
    window.removeEventListener("blur", onWindowBlur);
    window.removeEventListener("keydown", onKeyDown, true);
    window.removeEventListener("scroll", repaint, true);
    window.removeEventListener("resize", repaint);
    dragHandle.removeEventListener("pointerdown", onEditorPointerDown);
    dragHandle.removeEventListener("pointermove", onEditorPointerMove);
    dragHandle.removeEventListener("pointerup", onEditorPointerUp);
    dragHandle.removeEventListener("pointercancel", onEditorPointerUp);

    if (editorLayoutFrame !== null) window.cancelAnimationFrame(editorLayoutFrame);
    ipcRenderer.off(CANCEL_PICK_CHANNEL, onCancel);
    ipcRenderer.off(ANNOTATION_CAPTURED_CHANNEL, onCaptured);
    document.documentElement.removeAttribute("data-t3code-annotation-tool");
    cursorStyle.remove();
    host.remove();
    activeSession = null;

    if (notifyMain) ipcRenderer.send(ELEMENT_PICKED_CHANNEL, null);
  };

  const onCancel = (): void => teardown(false);
  const onCaptured = (): void => teardown(false);

  const onKeyDown = (event: KeyboardEvent): void => {
    if (isAnnotationNode(event.target) && event.key !== "Escape") return;

    if (event.key === "Escape") {
      event.preventDefault();
      event.stopPropagation();
      teardown(true);

      return;
    }

    if (event.key === "v") tool = "select";
    else if (event.key === "r") tool = "marquee";
    else if (event.key === "d") tool = "draw";
    else if (event.key === "e") tool = "erase";
    else return;
    refreshToolButtons();
  };

  const submitAnnotation = (submission: PreviewAnnotationSubmission): void => {
    if (pendingCapture || (selected.size === 0 && regions.length === 0 && strokes.length === 0))
      return;
    pendingCapture = true;
    submit.disabled = true;
    submit.textContent = "Capturing…";
    void Promise.all(
      Array.from(selected.values()).map(async (target) => {
        const element = await captureElement(target.element);

        if (!element) return null;

        for (const change of styleChanges.values()) {
          if (change.targetId === target.id) change.selector = element.selector;
        }

        return {
          id: target.id,
          element,
          rect: rectFromDomRect(target.element.getBoundingClientRect()),
        };
      }),
    ).then((captured) => {
      const elements = captured.filter((target) => target !== null);

      const annotation: PreviewAnnotationPayload = {
        id: nextId("annotation"),
        pageUrl: location.href,
        pageTitle: document.title?.trim() || null,
        comment: comment.value.trim(),
        elements,
        regions: [...regions],
        strokes: [...strokes],
        styleChanges: Array.from(styleChanges.values()),
        screenshot: null,
        createdAt: new Date().toISOString(),
      };

      editor.style.display = "none";
      toolbar.style.display = "none";
      hoverOutline.style.display = "none";

      const screenshotRect = unionRects([
        ...elements.map((target) => target.rect),
        ...regions.map((region) => region.rect),
        ...strokes.map((stroke) => stroke.bounds),
      ]);

      ipcRenderer.send(ELEMENT_PICKED_CHANNEL, annotation, screenshotRect, submission);
    });
  };

  submit.addEventListener("click", () => submitAnnotation("attach"));
  root.addEventListener("keydown", (event) => {
    const submission = event.target === comment ? resolveAnnotationSubmission(event) : null;
    // Keep this in the bubble phase so editor inputs receive the event before
    // it is isolated from listeners installed by the inspected page.
    event.stopImmediatePropagation();

    if (!submission) return;
    event.preventDefault();
    submitAnnotation(submission);
  });

  window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false });
  window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: false });
  window.addEventListener("pointerup", onPointerUp, { capture: true, passive: false });
  window.addEventListener("pointerout", onPointerOut, { capture: true, passive: true });
  window.addEventListener("click", onClick, { capture: true, passive: false });
  window.addEventListener("blur", onWindowBlur);
  window.addEventListener("keydown", onKeyDown, { capture: true });
  window.addEventListener("scroll", repaint, { capture: true, passive: true });
  window.addEventListener("resize", repaint, { passive: true });
  ipcRenderer.on(CANCEL_PICK_CHANNEL, onCancel);
  ipcRenderer.on(ANNOTATION_CAPTURED_CHANNEL, onCaptured);
  document.documentElement.appendChild(host);
  refreshToolButtons();
  updateStatus();
  activeSession = {
    teardown,
    applyTheme: (theme) => applyAnnotationTheme(host, theme),
  };
}

ipcRenderer.on(START_PICK_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme | undefined) => {
  if (theme) annotationTheme = theme;
  startAnnotation();
});

ipcRenderer.on(ANNOTATION_THEME_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme) => {
  annotationTheme = theme;
  activeSession?.applyTheme(theme);
});

ipcRenderer.on(CANCEL_PICK_CHANNEL, () => activeSession?.teardown(false));