Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

12.1 KB
Raw Blame
import { describeDemonstrationTarget } from "@openbot/shared/demonstration-rules";
import {
  holdsValue,
  isSensitiveTarget,
  MIN_DIGEST_LENGTH,
  normaliseSecret,
  sha256HexSync,
} from "@openbot/shared/recorder-masking";

const FIELD = "input, textarea, select, [contenteditable]";

export type SensitivityTracker = {
  /** Capture-phase pointerdown, focusin, click, input and change on window; install it BEFORE the collector so a digest
   * exists, and a click's field is known, before the collector's own listener builds the event it masks. */
  install(): () => void;
  /** input and change are judged by the field the event came from (else the focused field), a click by the field it
   * landed on: the field a label forwards its click to, never an earlier pointer target. */
  sensitiveForEvent(name: string): boolean;
  /** Values of at least MIN_DIGEST_LENGTH characters typed or filled into sensitive fields (a password manager's untrusted
   * input events count too), and the code a row of one-character boxes holds together, normalised then hashed, once per
   * value. */
  onSensitiveValue(listener: (digest: string) => void): void;
  /** This frame's own digests, for masking this frame's events. */
  digests(): ReadonlySet<string>;
};

const fieldOf = (target: EventTarget | null): Element | null =>
  target instanceof Element ? target.closest(FIELD) : null;

// The collector describes a click that a re-render moved to <body> or <html>, or off a detached node, by its pointerdown
// target (`clicks.ts` `installClickCapture` > `onClick`), so such a click keeps the pointerdown field.
const describedByPointerdown = (target: EventTarget | null): boolean =>
  !(target instanceof Element) ||
  target.localName === "body" ||
  target.localName === "html" ||
  !target.isConnected;

/** The longest label text read for a field: an aria-labelledby can point at a whole region. */
const LABEL_TEXT_LIMIT = 1000;

/** The text of the field's own labels and of what its aria-labelledby names: a label that is not a <label for> (a span
 * named by aria-labelledby) still names the field. */
const labelText = (field: Element): string => {
  const parts: string[] = [];
  if (
    field instanceof HTMLInputElement ||
    field instanceof HTMLTextAreaElement ||
    field instanceof HTMLSelectElement
  )
    for (const label of Array.from(field.labels ?? []))
      parts.push(label.textContent ?? "");
  const root = field.getRootNode() as Partial<NonElementParentNode>;
  for (const id of (field.getAttribute("aria-labelledby") ?? "")
    .split(/\s+/)
    .filter(Boolean)) {
    const named = root.getElementById?.(id) ?? document.getElementById(id);
    if (named) parts.push(named.textContent ?? "");
  }
  return parts.join(" ").slice(0, LABEL_TEXT_LIMIT);
};

/** The field as the collector describes it (its tag and every attribute), with its label text and what it holds: what
 * isSensitiveTarget reads. */
const describeField = (field: Element) => ({
  tag: field.localName,
  attributes: Object.fromEntries(
    Array.from(field.attributes, ({ name, value }) => [name, value]),
  ),
  label: labelText(field),
  value: fieldValue(field),
});

/** A one-time code typed a character per box: a row of 4 to 8 boxes, each one character long. */
const CODE_BOXES_MIN = 4;
const CODE_BOXES_MAX = 8;
/** How far above a box its row is looked for: each box can sit in its own wrapper, inside a row, inside a form. */
const CODE_BOX_DEPTH = 3;
/** Inputs a code box can be: text-like, never a checkbox or a button. */
const BOX_TYPES: ReadonlySet<string> = new Set([
  "",
  "text",
  "tel",
  "number",
  "password",
]);

const isBoxType = (element: Element): element is HTMLInputElement =>
  element instanceof HTMLInputElement &&
  BOX_TYPES.has((element.getAttribute("type") ?? "").toLowerCase());
/** A box that takes one character: maxlength=1. */
const isOneCharBox = (element: Element): element is HTMLInputElement =>
  isBoxType(element) && element.maxLength === 1;
/** A box of that shape without maxlength=1: numeric (inputmode numeric, type tel or a digits pattern) and no longer than one
 * character, which counts only beside its siblings in one parent. */
const isNumericBox = (element: Element): element is HTMLInputElement =>
  isBoxType(element) &&
  element.maxLength <= 1 &&
  (element.getAttribute("inputmode")?.toLowerCase() === "numeric" ||
    element.getAttribute("type")?.toLowerCase() === "tel" ||
    /^\^?(?:\\d|\[0-9)/.test(element.getAttribute("pattern") ?? ""));
/** Each box among `elements`, in document order. */
const boxesAmong = (
  elements: Iterable<Element>,
  isBox: (element: Element) => element is HTMLInputElement,
): HTMLInputElement[] => Array.from(elements).filter(isBox);

const inRow = (
  field: HTMLInputElement,
  boxes: HTMLInputElement[],
): HTMLInputElement[] | null =>
  boxes.length >= CODE_BOXES_MIN &&
  boxes.length <= CODE_BOXES_MAX &&
  boxes.includes(field)
    ? boxes
    : null;

/** The row of code boxes `field` is one of, in document order (the order the code is typed), or null: the nearest parent,
 * wrapper or form holding 4 to 8 one-character boxes, or 4 to 8 numeric boxes side by side in one parent. Found with no label,
 * since a row of boxes usually has none. */
const codeBoxes = (field: Element): HTMLInputElement[] | null => {
  if (!isBoxType(field)) return null;
  if (field.maxLength === 1) {
    let container: ParentNode | null = field.parentNode;
    for (let depth = 0; container && depth <= CODE_BOX_DEPTH; depth++) {
      const boxes = boxesAmong(
        container.querySelectorAll("input"),
        isOneCharBox,
      );
      if (boxes.length >= CODE_BOXES_MIN) return inRow(field, boxes);
      if (
        container instanceof HTMLFormElement ||
        !(container instanceof Element)
      )
        break;
      container = container.parentNode;
    }
    return field.form
      ? inRow(field, boxesAmong(field.form.elements, isOneCharBox))
      : null;
  }
  const parent = field.parentNode;
  return parent && isNumericBox(field)
    ? inRow(field, boxesAmong(parent.children, isNumericBox))
    : null;
};

/** The code a row of boxes holds together, in order. */
const rowValue = (boxes: HTMLInputElement[]): string =>
  boxes.map((box) => box.value.trim()).join("");

const fieldValue = (field: Element): string =>
  field instanceof HTMLInputElement ||
  field instanceof HTMLTextAreaElement ||
  field instanceof HTMLSelectElement
    ? field.value
    : (field.textContent ?? "");

export function createSensitivityTracker(options?: {
  isTrusted?: (event: Event) => boolean;
}): SensitivityTracker {
  const isTrusted = options?.isTrusted ?? ((event: Event) => event.isTrusted);
  const cache = new WeakMap<Element, boolean>();
  const digests = new Set<string>();
  const listeners: Array<(digest: string) => void> = [];
  let pointerField: Element | null = null;
  let valueField: Element | null = null;

  const classify = (field: Element): boolean => {
    const described = describeField(field);
    // Only a field that holds what the person enters can hold a secret: a checkbox or a button input that says "Show
    // password" only names one.
    const sensitive =
      holdsValue(described) &&
      ((field instanceof HTMLInputElement && field.type === "password") ||
        describeDemonstrationTarget({ point: null, element: field })
          .sensitive ||
        // The masking's own rule, with the label text: a one-time code field without autocomplete=one-time-code, by its
        // id, placeholder, aria-labelledby text, test id or shape, or by a label that says the code was texted or sent to
        // the person and the digits typed into it, so its value is hashed and masked wherever it reappears. Judged again
        // on every edit, as the value changes; once sensitive, always sensitive.
        isSensitiveTarget(described, { url: location.href }) ||
        // One box of a code typed a character per box, with no label: every box's value is masked.
        codeBoxes(field) !== null);
    // A field that was ever sensitive (a show-password toggle flips its type) stays sensitive.
    const known = cache.get(field) === true || sensitive;
    cache.set(field, known);
    return known;
  };
  const cached = (field: Element | null): boolean =>
    field === null ? false : (cache.get(field) ?? classify(field));

  const onPointer = (event: Event) => {
    if (!isTrusted(event)) return;
    pointerField = fieldOf(event.composedPath()[0] ?? null);
    if (pointerField) classify(pointerField);
  };
  const onClick = (event: Event) => {
    if (!isTrusted(event)) return;
    // A click on a label reaches its field as a second click, after a pointerdown on the label itself.
    const target = event.composedPath()[0] ?? null;
    if (describedByPointerdown(target)) return;
    pointerField = fieldOf(target);
    if (pointerField) classify(pointerField);
  };
  const onFocus = (event: Event) => {
    if (!isTrusted(event)) return;
    const field = fieldOf(event.composedPath()[0] ?? null);
    if (field) classify(field);
  };
  const report = (value: string) => {
    const normalised = normaliseSecret(value);
    if (normalised.length < MIN_DIGEST_LENGTH) return;
    const digest = sha256HexSync(normalised);
    if (digests.has(digest)) return;
    digests.add(digest);
    for (const listener of listeners) listener(digest);
  };
  /** The code a row of boxes makes together is hashed, since no single box holds it. */
  const reportRow = (field: Element | null) => {
    const boxes = field ? codeBoxes(field) : null;
    if (boxes) report(rowValue(boxes));
  };
  const onValue = (event: Event) => {
    // A field inside a shadow root: document.activeElement is its host, composedPath()[0] the field itself.
    const own = fieldOf(event.composedPath()[0] ?? null);
    // A password manager fills a field with untrusted events: what it put there is hashed too, but the field the collector
    // judges its next event by moves only on a trusted one, the only kind the collector records.
    if (isTrusted(event)) valueField = own;
    const field = own ?? fieldOf(document.activeElement);
    if (!field || !classify(field)) return;
    report(fieldValue(field));
    reportRow(field);
  };
  // A row of boxes is often filled without an input event per box: a paste the page spreads across them itself, or digits
  // moved along by script. The row is hashed again once the page has handled a paste, and when a box loses focus.
  const pending = new Set<ReturnType<typeof setTimeout>>();
  const onPaste = (event: Event) => {
    const field = fieldOf(event.composedPath()[0] ?? null);
    if (!field || codeBoxes(field) === null) return;
    const timer = setTimeout(() => {
      pending.delete(timer);
      reportRow(field);
    }, 0);
    pending.add(timer);
  };
  const onBlur = (event: Event) =>
    reportRow(fieldOf(event.composedPath()[0] ?? null));

  return {
    install() {
      const capture = { capture: true, passive: true } as const;
      window.addEventListener("pointerdown", onPointer, capture);
      window.addEventListener("focusin", onFocus, capture);
      window.addEventListener("click", onClick, capture);
      window.addEventListener("input", onValue, capture);
      window.addEventListener("change", onValue, capture);
      window.addEventListener("paste", onPaste, capture);
      window.addEventListener("focusout", onBlur, capture);
      return () => {
        window.removeEventListener("pointerdown", onPointer, capture);
        window.removeEventListener("focusin", onFocus, capture);
        window.removeEventListener("click", onClick, capture);
        window.removeEventListener("input", onValue, capture);
        window.removeEventListener("change", onValue, capture);
        window.removeEventListener("paste", onPaste, capture);
        window.removeEventListener("focusout", onBlur, capture);
        for (const timer of pending) clearTimeout(timer);
        pending.clear();
      };
    },
    sensitiveForEvent(name) {
      if (name === "input" || name === "change")
        return cached(valueField ?? fieldOf(document.activeElement));
      if (name === "click") return cached(pointerField);
      return false;
    },
    onSensitiveValue(listener) {
      listeners.push(listener);
    },
    digests: () => digests,
  };
}