Skip to content

Automate browser based workflows with AI

33.9 KB
Raw Blame
// @novnc/novnc is CJS with __esModule marker. Vite 8 (Rollup 5) changed
// CJS interop so the default import may be the namespace object instead of
// exports.default.  This guard works across bundler versions.
import _RFB, { type RfbEvent } from "@novnc/novnc/lib/rfb.js";
type RFB = _RFB;
const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;
import { useEffect, useState, useRef, useCallback } from "react";
import { useQuery } from "@tanstack/react-query";

import { getClient } from "@/api/AxiosClient";
import { isForbiddenError } from "@/api/forbidden";
import {
  Status,
  type TaskApiResponse,
  type WorkflowRunStatusApiResponse,
} from "@/api/types";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useLogging } from "@/hooks/useLogging";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { useSettingsStore } from "@/store/SettingsStore";
import {
  useDemonstrationControl,
  useManualSignInControl,
} from "@/store/useManualSignInStore";
import { useRecordingLauncherStore } from "@/store/useRecordingLauncherStore";
import { wssBaseUrl, newWssBaseUrl } from "@/util/env";
import { installNoVncGestureCrashGuard } from "@/util/novncGestureCrashGuard";
import { cn } from "@/util/utils";
import {
  StreamStatusPanel,
  type StreamDiagnostic,
} from "@/routes/streaming/StreamDiagnostics";
import { streamReconnectDelayMs } from "@/routes/streaming/streamLifecycle";
import type {
  StreamState,
  StreamStateChangeHandler,
} from "@/routes/streaming/streamState";
import {
  VNC_SUPER_L_KEYSYM,
  handleVncClipboardPasteShortcut,
  pasteTextIntoVnc,
  type HeldMetaSides,
} from "@/components/browserStreamClipboard";
import {
  STREAM_CONTAINER_CLASS,
  toastClipboardReadFailed,
  toastNothingToPaste,
  usePastedNotice,
} from "@/routes/streaming/pasteFeedback";
import {
  PastedNotice,
  StreamControlBar,
  TakeControlButton,
} from "@/routes/streaming/StreamControlBar";
import { useRecordingMessageChannel } from "@/routes/streaming/useRecordingMessageChannel";
import { useWebSocketParams } from "@/routes/streaming/webSocketParams";

import "./browser-stream.css";

installNoVncGestureCrashGuard();

const MESSAGE_RECONNECT_DELAY_MS = 1000;
const MESSAGE_MAX_RECONNECT_ATTEMPTS = 20;
const VNC_MAX_RECONNECT_ATTEMPTS = 8;
const STREAM_GAVE_UP_DIAGNOSTIC: StreamDiagnostic = {
  title: "Browser stream connection lost",
  detail:
    "The browser session stopped responding after several reconnect attempts.",
  hint: "Refresh the page to try again.",
};

interface BrowserSession {
  browser_session_id: string;
  status?: string | null;
  browser_address?: string | null;
  started_at?: string | null;
  completed_at?: string | null;
}

type BrowserSessionLiveness = Pick<
  BrowserSession,
  "browser_address" | "started_at" | "completed_at"
>;

function getBrowserSessionState(browserSession: BrowserSessionLiveness | null) {
  const hasBrowserSession = Boolean(
    browserSession && !browserSession.completed_at,
  );
  return {
    hasBrowserSession,
    isBrowserSessionStarted:
      hasBrowserSession &&
      Boolean(browserSession?.started_at || browserSession?.browser_address),
    isBrowserSessionEnded: !hasBrowserSession,
  };
}

type Props = {
  browserSessionId?: string;
  // The session behind browserSessionId, from a parent that already polls it.
  // The stream then reads this instead of polling the session a second time.
  browserSession?: BrowserSessionLiveness;
  interactive?: boolean;
  showControlButtons?: boolean;
  task?: {
    run: TaskApiResponse;
  };
  workflow?: {
    run: WorkflowRunStatusApiResponse;
  };
  resizeTrigger?: number;
  isVisible?: boolean;
  isExecuting?: boolean;
  onReadyChange?: (isReady: boolean, browserSessionId: string | null) => void;
  onStreamStateChange?: StreamStateChangeHandler;
  onActivity?: () => void;
  // --
  onClose?: () => void;
};

type RfbWithFrameUpdates = RFB & {
  _framebufferUpdate?: () => boolean;
};

function BrowserStream({
  browserSessionId = undefined,
  browserSession: parentBrowserSession = undefined,
  interactive = true,
  showControlButtons = undefined,
  task = undefined,
  workflow = undefined,
  resizeTrigger,
  isVisible = true,
  isExecuting = false,
  onReadyChange,
  onStreamStateChange,
  onActivity,
  // --
  onClose,
}: Props) {
  let showStream: boolean = false;
  let runId: string | null;
  let entity: "browserSession" | "task" | "workflow" | null;

  if (browserSessionId) {
    runId = browserSessionId;
    entity = "browserSession";
    showStream = true;
  } else if (task) {
    runId = task.run.task_id;
    showStream = statusIsNotFinalized(task.run);
    entity = "task";
  } else if (workflow) {
    runId = workflow.run.workflow_run_id;
    browserSessionId = workflow.run.browser_session_id ?? undefined;
    showStream = statusIsNotFinalized(workflow.run);
    entity = "workflow";
  } else {
    entity = null;
    runId = null;
  }

  const parentSessionState = parentBrowserSession
    ? getBrowserSessionState(parentBrowserSession)
    : null;

  useQuery({
    queryKey: ["hasBrowserSession", browserSessionId],
    queryFn: async () => {
      const client = await getClient(credentialGetter, "sans-api-v1");

      try {
        const response = await client.get<BrowserSession | null>(
          `/browser_sessions/${browserSessionId}`,
        );
        const sessionState = getBrowserSessionState(response.data);

        setHasBrowserSession(sessionState.hasBrowserSession);
        setIsBrowserSessionStarted(sessionState.isBrowserSessionStarted);
        setIsBrowserSessionEnded(sessionState.isBrowserSessionEnded);
        return sessionState.isBrowserSessionStarted;
      } catch (error) {
        setHasBrowserSession(false);
        setIsBrowserSessionStarted(false);
        // A forbidden session (expired, or owned by another org) never becomes
        // allowed, so surface it as an error and stop the poll instead of
        // swallowing it into the 1/s "not started yet" branch below.
        if (isForbiddenError(error)) {
          setIsBrowserSessionEnded(true);
          throw error;
        }
        return false;
      }
    },
    enabled:
      entity === "browserSession" && !!browserSessionId && !parentSessionState,
    refetchInterval: (query) =>
      query.state.status === "error" && isForbiddenError(query.state.error)
        ? false
        : query.state.data
          ? 5000
          : 1000,
  });

  const [polledHasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
  const [polledIsBrowserSessionStarted, setIsBrowserSessionStarted] =
    useState(false);
  const [polledIsBrowserSessionEnded, setIsBrowserSessionEnded] =
    useState(false);
  const hasBrowserSession =
    parentSessionState?.hasBrowserSession ?? polledHasBrowserSession;
  const isBrowserSessionStarted =
    parentSessionState?.isBrowserSessionStarted ??
    polledIsBrowserSessionStarted;
  const isBrowserSessionEnded =
    parentSessionState?.isBrowserSessionEnded ?? polledIsBrowserSessionEnded;
  const [hasGivenUp, setHasGivenUp] = useState(false);
  const [userIsControlling, setUserIsControlling] = useState(false);
  useManualSignInControl(
    showControlButtons ? browserSessionId : undefined,
    setUserIsControlling,
  );
  const demonstrating = useRecordingLauncherStore(
    (state) => state.demonstrationBrowserSessionId === browserSessionId,
  );
  const recordingDemonstration = useRecordingLauncherStore(
    (state) =>
      state.demonstrationRecordingBrowserSessionId === browserSessionId,
  );
  const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
  const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
  // The message socket must open after VNC's handshake has set the ALB
  // stickiness cookie, or the two sockets can land on different API tasks and
  // lose their client_id pairing. Latched so a VNC blip doesn't close it.
  const [hasVncConnected, setHasVncConnected] = useState(false);
  const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);
  const [terminalDiagnostic, setTerminalDiagnostic] =
    useState<StreamDiagnostic | null>(null);
  const [isReady, setIsReady] = useState(false);
  const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] =
    useState(0);
  const prevMessageConnectedRef = useRef<boolean>(false);
  const messageDroppedRef = useRef(false);
  const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
    null,
  );
  const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => {
    setCanvasContainer(node);
  }, []);
  const rfbRef = useRef<RFB | null>(null);
  const onActivityRef = useRef(onActivity);
  const userCanSendVncInputRef = useRef(false);
  const heldMetaSidesRef = useRef<HeldMetaSides>({
    left: false,
    right: false,
  });
  const leftCmdSentAsSuperRef = useRef(false);
  const observerRef = useRef<MutationObserver | null>(null);
  const vncReconnectAttemptsRef = useRef(0);
  const vncReconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(
    null,
  );
  const messageReconnectAttemptsRef = useRef(0);
  const messageReconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(
    null,
  );
  const gaveUpLoggedRef = useRef(false);
  const streamLogFieldsRef = useRef({ browserSessionId, entity, runId });
  streamLogFieldsRef.current = { browserSessionId, entity, runId };
  const settingsStore = useSettingsStore();
  const credentialGetter = useCredentialGetter();
  const logging = useLogging();
  const getWebSocketParams = useWebSocketParams();
  const isBrowserSessionAvailable =
    entity !== "browserSession" || hasBrowserSession;
  const isBrowserSessionBackendReady =
    entity !== "browserSession" || isBrowserSessionStarted;
  const handleMessageConnectionChange = useCallback(
    (connected: boolean, event?: CloseEvent) => {
      if (connected) {
        setTerminalDiagnostic(null);
        return;
      }
      if (!event) {
        return;
      }
      const { code, reason } = event;
      setTerminalDiagnostic(
        (prev) =>
          prev ??
          (code === 1006
            ? {
                title: "The messages channel slipped away",
                detail: "The messages channel dropped before sending a frame.",
                hint: "Check that the API server is reachable from the UI.",
              }
            : {
                title: "The messages channel packed up and left",
                detail: `Messages channel closed with code ${code}${reason ? ` (${reason})` : ""}.`,
              }),
      );
    },
    [],
  );
  const legacyMessageSocketUrl =
    entity === "task" && runId
      ? `${wssBaseUrl}/stream/messages/task/${runId}`
      : entity === "workflow" && runId
        ? `${wssBaseUrl}/stream/messages/workflow_run/${runId}`
        : undefined;
  const { isMessageConnected, sendCommand } = useRecordingMessageChannel({
    browserSessionId: runId,
    enabled:
      showStream &&
      Boolean(canvasContainer) &&
      Boolean(runId) &&
      isBrowserSessionAvailable &&
      isBrowserSessionBackendReady &&
      hasVncConnected,
    clipboard: "vnc",
    socketUrl: legacyMessageSocketUrl,
    reconnectTrigger: messagesDisconnectedTrigger,
    onConnectionChange: handleMessageConnectionChange,
  });

  useEffect(() => {
    onActivityRef.current = onActivity;
  }, [onActivity]);

  useEffect(() => {
    setIsBrowserSessionStarted(false);
    setIsReady(false);
    setIsVncConnected(false);
    setHasVncConnected(false);
    messageDroppedRef.current = false;
    setIsCanvasReady(false);
    setHasBrowserSession(true);
    setIsBrowserSessionEnded(false);
    setHasGivenUp(false);
    setTerminalDiagnostic(null);
    gaveUpLoggedRef.current = false;
    messageReconnectAttemptsRef.current = 0;
    if (messageReconnectTimerRef.current) {
      clearTimeout(messageReconnectTimerRef.current);
      messageReconnectTimerRef.current = null;
    }
    vncReconnectAttemptsRef.current = 0;
    if (vncReconnectTimerRef.current) {
      clearTimeout(vncReconnectTimerRef.current);
      vncReconnectTimerRef.current = null;
    }
    if (rfbRef.current) {
      rfbRef.current.disconnect();
      rfbRef.current = null;
    }
  }, [browserSessionId]);

  // browser is ready
  useEffect(() => {
    setIsReady(
      isVncConnected &&
        isCanvasReady &&
        isBrowserSessionAvailable &&
        isBrowserSessionBackendReady,
    );
  }, [
    isBrowserSessionAvailable,
    isBrowserSessionBackendReady,
    isCanvasReady,
    isVncConnected,
  ]);

  useEffect(() => {
    // browserSessionId intentionally not a dep: re-firing on prop change
    // before isReady resets would spuriously report (true, newSessionId).
    onReadyChange?.(isReady, isReady ? (browserSessionId ?? null) : null);
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [isReady, onReadyChange]);

  useEffect(() => {
    return () => {
      onReadyChange?.(false, null);
    };
  }, [onReadyChange]);

  const runEnded = (entity === "task" || entity === "workflow") && !showStream;
  const streamState: StreamState = isReady
    ? "live"
    : hasGivenUp || isBrowserSessionEnded || runEnded
      ? "stopped"
      : "connecting";

  useEffect(() => {
    // Same reason as onReadyChange: browserSessionId is read, not a dep.
    onStreamStateChange?.(streamState, browserSessionId ?? null);
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [streamState, onStreamStateChange]);

  useEffect(() => {
    return () => {
      onStreamStateChange?.("connecting", null);
    };
  }, [onStreamStateChange]);

  // `isUsingABrowser` is tied to local `isReady`, so this component owns it.
  // `isLoadingABrowser` is owned by the route instead (SKY-9777).
  useEffect(() => {
    settingsStore.setIsUsingABrowser(isReady);
    settingsStore.setBrowserSessionId(
      isReady ? (browserSessionId ?? null) : null,
    );
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [isReady, browserSessionId]);

  // message channel reconnect policy
  useEffect(() => {
    const messageJustClosed =
      prevMessageConnectedRef.current && !isMessageConnected;
    prevMessageConnectedRef.current = isMessageConnected;
    if (messageJustClosed) {
      messageDroppedRef.current = true;
    }

    if (isMessageConnected) {
      messageDroppedRef.current = false;
      return;
    }

    // A live VNC stream proves the session is real: reconnect now and drop the cap (also recovers a late VNC connect).
    // Skipped for a socket still opening, which the first VNC connect enables.
    if (isVncConnected && messageDroppedRef.current) {
      messageReconnectAttemptsRef.current = 0;
      if (messageReconnectTimerRef.current) {
        clearTimeout(messageReconnectTimerRef.current);
        messageReconnectTimerRef.current = null;
      }
      setMessagesDisconnectedTrigger((x) => x + 1);
      return;
    }

    if (!messageJustClosed) {
      return;
    }

    // No stream is live; a session the backend can't find would respin forever, so cap it.
    if (messageReconnectAttemptsRef.current >= MESSAGE_MAX_RECONNECT_ATTEMPTS) {
      setTerminalDiagnostic((prev) => prev ?? STREAM_GAVE_UP_DIAGNOSTIC);
      setHasGivenUp(true);
      if (!gaveUpLoggedRef.current) {
        gaveUpLoggedRef.current = true;
        logging.warn("Stream gave up", {
          stream: "vnc",
          browser_session_id:
            streamLogFieldsRef.current.browserSessionId ?? null,
          workflow_run_id:
            streamLogFieldsRef.current.entity === "workflow"
              ? streamLogFieldsRef.current.runId
              : null,
          reason: "message_reconnect_exhausted",
          reconnect_attempts: messageReconnectAttemptsRef.current,
        });
      }
      return;
    }

    messageReconnectAttemptsRef.current += 1;
    if (messageReconnectTimerRef.current) {
      clearTimeout(messageReconnectTimerRef.current);
    }
    messageReconnectTimerRef.current = setTimeout(() => {
      messageReconnectTimerRef.current = null;
      setMessagesDisconnectedTrigger((x) => x + 1);
    }, MESSAGE_RECONNECT_DELAY_MS);
  }, [isMessageConnected, isVncConnected, logging]);

  useEffect(() => {
    return () => {
      if (messageReconnectTimerRef.current) {
        clearTimeout(messageReconnectTimerRef.current);
      }
    };
  }, []);

  // vnc socket
  useEffect(
    () => {
      if (!showStream || !canvasContainer || !runId) {
        if (rfbRef.current) {
          rfbRef.current.disconnect();
          rfbRef.current = null;
          setIsVncConnected(false);
        }
        return;
      }

      let cancelled = false;
      let didDisconnect = false;

      async function setupVnc() {
        if (rfbRef.current && isVncConnected) {
          return;
        }

        const wsParams = await getWebSocketParams();
        if (cancelled) {
          return;
        }
        const vncUrl =
          entity === "browserSession"
            ? `${newWssBaseUrl}/stream/vnc/browser_session/${runId}?${wsParams}`
            : entity === "task"
              ? `${wssBaseUrl}/stream/vnc/task/${runId}?${wsParams}`
              : entity === "workflow"
                ? `${wssBaseUrl}/stream/vnc/workflow_run/${runId}?${wsParams}`
                : null;

        if (!vncUrl) {
          throw new Error("No vnc url");
        }

        if (rfbRef.current) {
          rfbRef.current.disconnect();
        }

        if (!isBrowserSessionAvailable || !isBrowserSessionBackendReady) {
          setIsVncConnected(false);
          return;
        }

        const canvas = canvasContainer;

        if (!canvas) {
          throw new Error("Canvas element not found");
        }

        observerRef.current = new MutationObserver(() => {
          const canvasElement = canvasContainer.querySelector("canvas");
          if (canvasElement) {
            setIsCanvasReady(true);
            observerRef.current?.disconnect();
          }
        });

        observerRef.current.observe(canvasContainer, {
          childList: true,
          subtree: true,
        });

        const rfb = new RFB(canvas, vncUrl);

        rfb.scaleViewport = true;
        rfb.compressionLevel = 2;
        rfb.qualityLevel = 6;

        const frameUpdateRfb = rfb as RfbWithFrameUpdates;
        // noVNC does not expose a public framebuffer-update event in 1.5.x.
        // Hook the internal method defensively so activity tracking degrades
        // to no-op if the private API changes.
        const originalFrameUpdate =
          frameUpdateRfb._framebufferUpdate?.bind(rfb);
        if (originalFrameUpdate) {
          frameUpdateRfb._framebufferUpdate = () => {
            const didCompleteFrameUpdate = originalFrameUpdate();
            if (didCompleteFrameUpdate) {
              onActivityRef.current?.();
            }
            return didCompleteFrameUpdate;
          };
        }

        rfbRef.current = rfb;

        const canvasElement = canvasContainer.querySelector("canvas");

        if (canvasElement) {
          setIsCanvasReady(true);
          observerRef.current?.disconnect();
        }

        rfb.addEventListener("connect", () => {
          setIsVncConnected(true);
          setHasVncConnected(true);
          setHasGivenUp(false);
          setTerminalDiagnostic(null);
          messageReconnectAttemptsRef.current = 0;
          vncReconnectAttemptsRef.current = 0;
        });

        rfb.addEventListener("disconnect", (e: RfbEvent) => {
          if (cancelled || didDisconnect) return;
          didDisconnect = true;
          setIsVncConnected(false);
          setIsCanvasReady(false);
          // Exponential backoff with jitter, capped attempts: a session with no
          // reachable VNC endpoint dies in ~150ms per dial, and an immediate
          // retrigger hammered the API at ~4 dials/second.
          if (vncReconnectAttemptsRef.current < VNC_MAX_RECONNECT_ATTEMPTS) {
            const delay = streamReconnectDelayMs(
              vncReconnectAttemptsRef.current,
            );
            vncReconnectAttemptsRef.current += 1;
            vncReconnectTimerRef.current = setTimeout(
              () => {
                vncReconnectTimerRef.current = null;
                setVncDisconnectedTrigger((x) => x + 1);
              },
              delay + Math.random() * delay * 0.5,
            );
          } else {
            setHasGivenUp(true);
            if (!gaveUpLoggedRef.current) {
              gaveUpLoggedRef.current = true;
              logging.warn("Stream gave up", {
                stream: "vnc",
                browser_session_id:
                  streamLogFieldsRef.current.browserSessionId ?? null,
                workflow_run_id:
                  streamLogFieldsRef.current.entity === "workflow"
                    ? streamLogFieldsRef.current.runId
                    : null,
                reason: "reconnect_exhausted",
                reconnect_attempts: vncReconnectAttemptsRef.current,
              });
            }
          }
          onClose?.();
          const clean = Boolean(e.detail?.clean);
          setTerminalDiagnostic(
            (prev) =>
              prev ??
              (clean
                ? {
                    title: "The browser stream packed up and left",
                    detail: "The browser stream closed cleanly.",
                  }
                : {
                    title: "The browser stream slipped away",
                    hint: "Refresh the page or switch to local browser streaming.",
                  }),
          );
        });
      }

      setupVnc();

      return () => {
        cancelled = true;
        // Every re-run redials via setupVnc, so a pending retry is redundant.
        if (vncReconnectTimerRef.current) {
          clearTimeout(vncReconnectTimerRef.current);
          vncReconnectTimerRef.current = null;
        }
        if (observerRef.current) {
          observerRef.current.disconnect();
          observerRef.current = null;
        }
        if (rfbRef.current) {
          rfbRef.current.disconnect();
          rfbRef.current = null;
        }
        setIsVncConnected(false);
        setIsCanvasReady(false);
      };
    },
    // cannot include isVncConnected in deps as it will cause infinite loop
    // eslint-disable-next-line react-hooks/exhaustive-deps
    [
      browserSessionId,
      entity,
      canvasContainer,
      isBrowserSessionAvailable,
      isBrowserSessionBackendReady,
      runId,
      showStream,
      vncDisconnectedTrigger, // will re-run on disconnects
      logging,
    ],
  );

  // effect to send a message when the user is controlling, vs not controlling.
  // Waits for VNC: the backend drops control messages until its VNC channel
  // exists, and every VNC (re)connect starts that channel agent-controlled.
  useEffect(() => {
    if (!isMessageConnected || !isVncConnected) {
      return;
    }

    if (interactive || userIsControlling) {
      sendCommand({ kind: "take-control" });
    } else {
      sendCommand({ kind: "cede-control" });
    }
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [interactive, isMessageConnected, isVncConnected, userIsControlling]);

  // noVNC (1.5.0) only rescales via its own observer, which gets swallowed on
  // re-parent; re-asserting scaleViewport on resize forces a recompute (skip 0×0).
  useEffect(() => {
    if (!canvasContainer || typeof ResizeObserver === "undefined") {
      return;
    }
    const rescale = () => {
      const rect = canvasContainer.getBoundingClientRect();
      if (rect.width > 0 && rect.height > 0 && rfbRef.current) {
        rfbRef.current.scaleViewport = true;
      }
    };
    rescale();
    const observer = new ResizeObserver(rescale);
    observer.observe(canvasContainer);
    return () => observer.disconnect();
  }, [canvasContainer, resizeTrigger]);

  // Effect to show toast when task or workflow reaches a final state based on hook updates
  useEffect(() => {
    const run = task ? task.run : workflow ? workflow.run : null;

    if (!run) {
      return;
    }

    const name = task ? "task" : workflow ? "agent" : null;

    if (!name) {
      return;
    }

    if (run.status === Status.Failed || run.status === Status.Terminated) {
      // Only show toast if VNC is not connected or was never connected,
      // to avoid double toasting if disconnect handler also triggers similar logic.
      // However, the disconnect handler now primarily invalidates queries.
      toast({
        title: "Run Ended",
        description: `The ${name} run has ${run.status}.`,
        variant: "destructive",
      });
    } else if (run.status === Status.Completed) {
      toast({
        title: "Run Completed",
        description: `The ${name} run has been completed.`,
        variant: "success",
      });
    }
  }, [task, workflow]);

  useEffect(() => {
    if (!interactive) {
      setUserIsControlling(false);
    }
  }, [interactive]);

  // When control can no longer be offered (buttons hidden and not inherently
  // interactive), a prior grab must be released or its input keeps flowing.
  // Recording and demonstration modes hold control while their control buttons are hidden.
  useEffect(() => {
    if (!interactive && !showControlButtons && !demonstrating) {
      setUserIsControlling(false);
    }
  }, [interactive, showControlButtons, demonstrating]);

  useDemonstrationControl(recordingDemonstration, true, setUserIsControlling);

  const theUserIsControlling =
    userIsControlling || (interactive && !showControlButtons);

  const [pastedCharacters, showPasted] = usePastedNotice();

  const sendTextToVnc = useCallback(
    async (text: string) => {
      if (!text) {
        toastNothingToPaste();
        return;
      }
      // The clipboard read can wait on a permission prompt; the stream or the
      // user's control may have changed by the time it resolves.
      const rfb = rfbRef.current;
      if (!rfb || !userCanSendVncInputRef.current) {
        return;
      }
      await pasteTextIntoVnc(rfb, text);
      if (rfbRef.current !== rfb) {
        return;
      }
      showPasted(text);
      // Keystrokes after a paste belong to the page, not the button.
      rfb.focus({ preventScroll: true });
    },
    [showPasted],
  );

  const pasteIntoVnc = useCallback(async () => {
    let text: string;
    try {
      text = await navigator.clipboard.readText();
    } catch (err) {
      console.error("Failed to read the clipboard for VNC:", err);
      toastClipboardReadFailed();
      return;
    }
    await sendTextToVnc(text);
  }, [sendTextToVnc]);

  useEffect(() => {
    userCanSendVncInputRef.current = theUserIsControlling;
  }, [theUserIsControlling]);

  useEffect(() => {
    if (!canvasContainer) {
      return;
    }

    // Same platform test noVNC uses for its Cmd/Alt keysym swap.
    const isApplePlatform = /mac|iphone|ipad|ipod/i.test(navigator.platform);

    const handleKeyDown = (event: KeyboardEvent) => {
      // Track only Meta keydowns noVNC's canvas receives: restoring a side noVNC never tracked would strand the modifier remotely, since noVNC drops keyups for keys it never saw down.
      if (event.key === "Meta" && event.target instanceof HTMLCanvasElement) {
        if (event.code === "MetaLeft") {
          heldMetaSidesRef.current = {
            ...heldMetaSidesRef.current,
            left: true,
          };
          // noVNC sends Apple left Cmd as Alt_L, so the remote page saw Cmd+P as Alt+P. Send Super_L, as noVNC already does for right Cmd.
          if (isApplePlatform && userCanSendVncInputRef.current) {
            event.preventDefault();
            event.stopImmediatePropagation();
            rfbRef.current?.sendKey(VNC_SUPER_L_KEYSYM, "MetaLeft", true);
            leftCmdSentAsSuperRef.current = true;
            return;
          }
        } else if (event.code === "MetaRight") {
          heldMetaSidesRef.current = {
            ...heldMetaSidesRef.current,
            right: true,
          };
        }
      }

      if (!userCanSendVncInputRef.current) {
        return;
      }

      void handleVncClipboardPasteShortcut(event, rfbRef.current, {
        getHeldMetaSides: () => heldMetaSidesRef.current,
        onPasteError: toastClipboardReadFailed,
        onPasted: showPasted,
        onEmptyClipboard: toastNothingToPaste,
      });
    };

    const releaseLeftCmd = () => {
      if (leftCmdSentAsSuperRef.current) {
        rfbRef.current?.sendKey(VNC_SUPER_L_KEYSYM, "MetaLeft", false);
        leftCmdSentAsSuperRef.current = false;
      }
    };

    const handleKeyUp = (event: KeyboardEvent) => {
      if (event.key === "Meta" && event.code === "MetaLeft") {
        heldMetaSidesRef.current = {
          ...heldMetaSidesRef.current,
          left: false,
        };
        releaseLeftCmd();
      } else if (event.key === "Meta" && event.code === "MetaRight") {
        heldMetaSidesRef.current = {
          ...heldMetaSidesRef.current,
          right: false,
        };
      }
    };

    const handleBlur = () => {
      heldMetaSidesRef.current = { left: false, right: false };
      releaseLeftCmd();
    };

    // Edit-menu paste arrives only as a paste event; Cmd/Ctrl+V never does,
    // because the shortcut handler prevents its default.
    const handlePaste = (event: ClipboardEvent) => {
      if (!userCanSendVncInputRef.current) {
        return;
      }
      event.preventDefault();
      void sendTextToVnc(event.clipboardData?.getData("text/plain") ?? "");
    };

    canvasContainer.addEventListener("keydown", handleKeyDown, true);
    canvasContainer.addEventListener("paste", handlePaste);
    window.addEventListener("keyup", handleKeyUp, true);
    window.addEventListener("blur", handleBlur);
    return () => {
      canvasContainer.removeEventListener("keydown", handleKeyDown, true);
      canvasContainer.removeEventListener("paste", handlePaste);
      window.removeEventListener("keyup", handleKeyUp, true);
      window.removeEventListener("blur", handleBlur);
    };
  }, [canvasContainer, showPasted, sendTextToVnc]);

  const streamDiagnostic: StreamDiagnostic =
    !showStream || !runId
      ? {
          title: "Starting browser session",
          detail: "Waiting for a live browser session to attach.",
        }
      : entity === "browserSession" && browserSessionId && !hasBrowserSession
        ? {
            title: "This browser session has wandered off",
            detail: "Looks like it slipped away mid-stream.",
            hint: "Refresh the page or spin up a fresh browser session.",
          }
        : terminalDiagnostic
          ? terminalDiagnostic
          : !isBrowserSessionBackendReady
            ? {
                title: "Warming up your browser",
                detail:
                  "The session is here — we're just waiting for the backend to give the green light.",
                pending: true,
              }
            : !isVncConnected
              ? {
                  title: "Reaching out to your browser",
                  detail: "Opening up the live stream and message channels...",
                  hint: "If this sticks around, check VNC support for the session or switch to local browser streaming.",
                  pending: true,
                }
              : !isCanvasReady
                ? {
                    title: "Setting the stage",
                    detail:
                      "The connection is open — now we're waiting for the browser to paint its first frame.",
                    pending: true,
                  }
                : {
                    title: "Tuning in to your browser...",
                    pending: true,
                  };

  return (
    <>
      <div
        className={cn(
          "browser-stream relative flex flex-col items-center justify-center",
          STREAM_CONTAINER_CLASS,
          {
            "user-is-controlling": theUserIsControlling,
          },
        )}
        ref={setCanvasContainerRef}
      >
        {isReady && isVisible && (
          // Same as InteractiveStreamView: while the take-control button is
          // offered, a click anywhere on the picture takes control instead of
          // being swallowed by this layer.
          <div
            data-testid="browser-stream-overlay"
            className={cn(
              "overlay z-10 flex items-center justify-center overflow-hidden",
              { "can-take-control": showControlButtons && !userIsControlling },
            )}
            onClick={
              showControlButtons && !userIsControlling
                ? () => setUserIsControlling(true)
                : undefined
            }
          >
            <PastedNotice characters={pastedCharacters} />
            {showControlButtons && (
              <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
                <TakeControlButton
                  onClick={() => setUserIsControlling(true)}
                  className={cn("control-button pointer-events-auto", {
                    hide: userIsControlling,
                  })}
                />
                {userIsControlling && !demonstrating && (
                  <StreamControlBar
                    onStop={() => setUserIsControlling(false)}
                    onPaste={pasteIntoVnc}
                  />
                )}
              </div>
            )}
          </div>
        )}
        {isExecuting && (
          <div className="pointer-events-none absolute flex aspect-video w-full animate-glow items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-yellow-500">
            <div className="relative h-full w-full">
              <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-yellow-500">
                <div className="truncate">Agent is working</div>
              </div>
            </div>
          </div>
        )}
        {!isReady && (
          <div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-neutral-200 bg-white text-sm text-neutral-600 dark:border-slate-800 dark:bg-transparent dark:text-slate-400">
            <StreamStatusPanel diagnostic={streamDiagnostic} />
          </div>
        )}
      </div>
    </>
  );
}

export { BrowserStream };