Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

feat: render sandbox dev servers natively in the side browser (#162)

crispinhu2741e6a07aa554
diff --git a/chatbot/src/AgentBrowser.jsx b/chatbot/src/AgentBrowser.jsx
index 1be66026..dc3205af 100644
--- a/chatbot/src/AgentBrowser.jsx
+++ b/chatbot/src/AgentBrowser.jsx
@@ -1,6 +1,8 @@
-import React, { useEffect, useRef, useState } from "react";
+import React, { useCallback, useEffect, useRef, useState } from "react";
import HarnessIcon from "./HarnessIcon";
import { api } from "./api";
+import { devServerAddress, devServerLabel, devServerOrigin, devServerTarget, devServerURL } from "./agent-browser-links.js";
+import { elementLabel, elementReference, inspectionReference } from "./element-context.js";
import "./agent-browser.css";
const modifiers = (e) =>
@@ -10,12 +12,595 @@ const buttonName = (b) => (b === 1 ? "middle" : b === 2 ? "right" : "left");
const deniedError = "无权操作";
const startingText = "正在启动 Agent 浏览器…";
-// AgentBrowser shows the Agent browser of the conversation's workspace: the
-// browser Main Bot drives with workbench_browser_*, running on the execution
-// machine next to the workspace. The user watches it live and can take over.
-// Protocol and input handling follow the task panel in web/collaboration.js:
-// one WebSocket carries frames and input, plain polling is the fallback.
-export default function AgentBrowser({ runId, request = null, onRequestHandled = () => {} }) {
+// AgentBrowser is the side panel's 浏览器 tab. Dev server addresses
+// (http://localhost:PORT) and workspace files (file:///workspace/…) open
+// natively: the platform serves them and the user's own browser renders the
+// page. Everything else shows the Agent's
+// browser on the execution machine as a live screencast.
+// onAddElement, when given, lets the user pick an element of a native page
+// and add it to the composer as context for the Agent.
+export default function AgentBrowser({ runId, request = null, onRequestHandled = () => {}, onAddElement = null }) {
+ // Decided before the first render, so a native page never starts a screencast.
+ const [native, setNative] = useState(() => {
+ const target = request?.url ? devServerTarget(request.url) : null;
+ return target ? { ...target, key: request.id } : null;
+ });
+ const [forward, setForward] = useState(null);
+ useEffect(() => {
+ if (!request?.url) return;
+ const target = devServerTarget(request.url);
+ if (target) {
+ setNative({ ...target, key: request.id });
+ onRequestHandled(request.id);
+ return;
+ }
+ setNative(null);
+ setForward(request);
+ }, [request?.id, request?.url]);
+ const forwarded = useCallback(
+ (id) => {
+ setForward((current) => (current?.id === id ? null : current));
+ onRequestHandled(id);
+ },
+ [onRequestHandled],
+ );
+ if (native)
+ return (
+ <DevServerFrame
+ key={`${native.port}:${native.key}`}
+ runId={runId}
+ port={native.port}
+ path={native.path}
+ onOpen={(url) => {
+ const target = devServerTarget(url);
+ if (target) setNative({ ...target, key: `${Date.now()}` });
+ else {
+ setNative(null);
+ setForward({ id: `${Date.now()}-${Math.random()}`, url });
+ }
+ }}
+ onScreencast={() => setNative(null)}
+ onAddElement={onAddElement}
+ />
+ );
+ return (
+ <ScreencastBrowser
+ runId={runId}
+ request={forward}
+ onRequestHandled={forwarded}
+ onNative={(target) => setNative({ ...target, key: `${Date.now()}` })}
+ onAddElement={onAddElement}
+ />
+ );
+}
+
+// DevServerFrame renders a dev server or a file of the workspace in an iframe
+// served by the platform under /devpreview/: native input, scrolling and
+// devtools.
+function DevServerFrame({ runId, port, path, onOpen, onScreencast, onAddElement = null }) {
+ const frameRef = useRef(null);
+ const addressRef = useRef(null);
+ const [address, setAddress] = useState(`${devServerOrigin(port)}${path}`);
+ const [src] = useState(() => devServerURL(runId, port, path));
+ const [loading, setLoading] = useState(true);
+ const [loads, setLoads] = useState(0);
+ const pick = useElementPicker(frameRef, loads);
+ const current = () => {
+ try {
+ return frameRef.current?.contentWindow?.location || null;
+ } catch {
+ return null;
+ }
+ };
+ // Client-side routing changes the address without a load event.
+ useEffect(() => {
+ const timer = setInterval(() => {
+ const location = current();
+ if (!location || location.href === "about:blank" || document.activeElement === addressRef.current) return;
+ setAddress(devServerAddress(runId, port, location));
+ }, 800);
+ return () => clearInterval(timer);
+ }, [runId, port]);
+ const history = (step) => {
+ try {
+ frameRef.current?.contentWindow?.history.go(step);
+ } catch {
+ /* cross-origin page: ignore */
+ }
+ };
+ const reload = () => {
+ setLoading(true);
+ try {
+ frameRef.current.contentWindow.location.reload();
+ } catch {
+ frameRef.current.src = src;
+ }
+ };
+ const openWindow = () => {
+ const location = current();
+ const href = location && location.href !== "about:blank" ? location.href : src;
+ window.open(href, "_blank", "noopener");
+ };
+ const button = (icon, label, onClick, extra = "") => (
+ <button type="button" className={`agent-browser-btn${extra}`} title={label} aria-label={label} onClick={onClick}>
+ <HarnessIcon name={icon} size={15} />
+ </button>
+ );
+ return (
+ <div className="agent-browser is-native">
+ <div className="agent-browser-bar">
+ {button("arrow-left", "后退", () => history(-1))}
+ {button("arrow-right", "前进", () => history(1))}
+ {button("refresh", "刷新", reload)}
+ <form
+ className="agent-browser-address"
+ onSubmit={(event) => {
+ event.preventDefault();
+ const target = address.trim();
+ if (!target) return;
+ const next = devServerTarget(/^[a-z][a-z0-9+.-]*:/i.test(target) ? target : `http://${target}`);
+ if (next && next.port === port) {
+ setLoading(true);
+ frameRef.current.src = devServerURL(runId, port, next.path);
+ addressRef.current?.blur();
+ } else onOpen(target);
+ }}
+ >
+ <HarnessIcon name="globe" size={13} />
+ <input
+ ref={addressRef}
+ type="text"
+ spellCheck={false}
+ aria-label="网址"
+ value={address}
+ onChange={(event) => setAddress(event.target.value)}
+ />
+ {onAddElement ? <AnnotateButton picking={pick.picking} onClick={pick.toggle} /> : null}
+ </form>
+ {button("preview-expand", "在新窗口打开", openWindow)}
+ {button("monitor", "切换到 Agent 浏览器画面", onScreencast)}
+ </div>
+ <div className="agent-browser-stage agent-browser-native">
+ <iframe
+ ref={frameRef}
+ className="agent-browser-frame"
+ title={port === "file" ? "工作区文件" : `开发服务器 localhost:${port}`}
+ src={src}
+ onLoad={() => {
+ setLoading(false);
+ setLoads((n) => n + 1);
+ const location = current();
+ if (location && document.activeElement !== addressRef.current)
+ setAddress(devServerAddress(runId, port, location));
+ }}
+ />
+ {loading ? (
+ <div className="agent-browser-overlay agent-browser-loading" data-tone="idle">
+ <span>正在打开 {devServerLabel(port)}…</span>
+ </div>
+ ) : null}
+ <ElementPickLayer
+ pick={pick}
+ frameRef={frameRef}
+ onAdd={(el) => {
+ const location = current();
+ onAddElement?.(elementReference(el, location ? devServerAddress(runId, port, location) : address));
+ pick.clear();
+ }}
+ />
+ </div>
+ <div className="agent-browser-foot">
+ <span>原生预览 · {port === "file" ? "工作区文件" : `端口 ${port}`}</span>
+ <small role="status">
+ {port === "file"
+ ? "页面在你的浏览器里直接渲染;文件改动后点刷新"
+ : "页面在你的浏览器里直接渲染;暂不支持热更新,代码改动后点刷新"}
+ </small>
+ </div>
+ </div>
+ );
+}
+
+// useElementPicker drives element selection inside a same-origin preview
+// frame: hovering outlines the element under the pointer, a click selects
+// it without reaching the page. The page's own DOM is left alone apart from a
+// temporary crosshair cursor; outlines are drawn above the frame.
+function useElementPicker(frameRef, loads) {
+ const [picking, setPicking] = useState(false);
+ const [hover, setHover] = useState(null);
+ const [picked, setPicked] = useState(null);
+ const [error, setError] = useState("");
+ const [, setTick] = useState(0);
+ const hovered = useRef(null);
+ const frameDocument = () => {
+ try {
+ return frameRef.current?.contentDocument || null;
+ } catch {
+ return null;
+ }
+ };
+ // Selection and outline follow the page while it scrolls or resizes.
+ useEffect(() => {
+ if (!picked && !picking) return undefined;
+ const doc = frameDocument();
+ const win = doc?.defaultView;
+ if (!win) return undefined;
+ const refresh = () => setTick((n) => n + 1);
+ win.addEventListener("scroll", refresh, true);
+ win.addEventListener("resize", refresh);
+ return () => {
+ win.removeEventListener("scroll", refresh, true);
+ win.removeEventListener("resize", refresh);
+ };
+ }, [picked, picking, loads]);
+ useEffect(() => {
+ if (!picking) return undefined;
+ const doc = frameDocument();
+ if (!doc?.documentElement) {
+ setError("当前页面不支持标记元素");
+ setPicking(false);
+ return undefined;
+ }
+ setError("");
+ const cursor = doc.createElement("style");
+ cursor.setAttribute("data-af-pick", "");
+ cursor.textContent = "*{cursor:crosshair!important}";
+ (doc.head || doc.documentElement).appendChild(cursor);
+ const target = (event) => {
+ const el = event.target;
+ return el && el.nodeType === 1 && el !== doc.documentElement ? el : null;
+ };
+ const move = (event) => {
+ const el = target(event);
+ if (el === hovered.current) return;
+ hovered.current = el;
+ setHover(el ? { el } : null);
+ };
+ const block = (event) => {
+ event.preventDefault();
+ event.stopPropagation();
+ event.stopImmediatePropagation?.();
+ };
+ const click = (event) => {
+ block(event);
+ const el = target(event);
+ if (!el) return;
+ setPicked({ el });
+ setPicking(false);
+ };
+ const key = (event) => {
+ if (event.key !== "Escape") return;
+ block(event);
+ setPicking(false);
+ };
+ const leave = () => {
+ hovered.current = null;
+ setHover(null);
+ };
+ const blocked = ["mousedown", "mouseup", "pointerdown", "pointerup", "dblclick", "contextmenu", "auxclick"];
+ doc.addEventListener("mousemove", move, true);
+ doc.addEventListener("click", click, true);
+ doc.addEventListener("keydown", key, true);
+ doc.documentElement.addEventListener("mouseleave", leave);
+ blocked.forEach((name) => doc.addEventListener(name, block, true));
+ return () => {
+ doc.removeEventListener("mousemove", move, true);
+ doc.removeEventListener("click", click, true);
+ doc.removeEventListener("keydown", key, true);
+ doc.documentElement.removeEventListener("mouseleave", leave);
+ blocked.forEach((name) => doc.removeEventListener(name, block, true));
+ cursor.remove();
+ hovered.current = null;
+ setHover(null);
+ };
+ }, [picking, loads]);
+ // A navigation replaces the document the selection belonged to.
+ useEffect(() => setPicked(null), [loads]);
+ useEffect(() => {
+ if (!picking && !picked) return undefined;
+ const key = (event) => {
+ if (event.key !== "Escape") return;
+ setPicking(false);
+ setPicked(null);
+ };
+ document.addEventListener("keydown", key);
+ return () => document.removeEventListener("keydown", key);
+ }, [picking, picked]);
+ return {
+ picking,
+ hover,
+ picked: picked?.el?.isConnected ? picked : null,
+ error,
+ toggle: () => {
+ setPicked(null);
+ setPicking((on) => !on);
+ },
+ repick: () => {
+ setPicked(null);
+ setPicking(true);
+ },
+ clear: () => setPicked(null),
+ };
+}
+
+const outlineOf = (el, frame) => {
+ const rect = el.getBoundingClientRect();
+ return {
+ left: frame.offsetLeft + rect.left,
+ top: frame.offsetTop + rect.top,
+ width: rect.width,
+ height: rect.height,
+ };
+};
+
+// AnnotateButton sits at the end of the address bar and turns element
+// marking on and off.
+function AnnotateButton({ picking, disabled = false, onClick }) {
+ return (
+ <button
+ type="button"
+ className={`agent-browser-annotate${picking ? " is-on" : ""}`}
+ title={picking ? "退出标记(Esc)" : "在页面上标记元素,加入对话框"}
+ aria-pressed={picking}
+ disabled={disabled}
+ onClick={onClick}
+ >
+ <HarnessIcon name="crosshair" size={14} />
+ <span>标记元素</span>
+ </button>
+ );
+}
+
+// ElementPickLayer outlines elements of the native preview frame.
+function ElementPickLayer({ pick, frameRef, onAdd }) {
+ const frame = frameRef.current;
+ const stage = frame?.parentElement;
+ if (!frame || !stage) return null;
+ const hover = pick.picking && pick.hover?.el?.isConnected ? pick.hover.el : null;
+ const picked = pick.picked?.el || null;
+ const box = (el) => (el ? { ...outlineOf(el, frame), label: elementLabel(el) } : null);
+ return (
+ <PickOverlay
+ stage={stage}
+ hover={box(hover)}
+ picked={box(picked)}
+ picking={pick.picking}
+ error={pick.error}
+ onAdd={() => onAdd(picked)}
+ onRepick={pick.repick}
+ onCancel={pick.clear}
+ />
+ );
+}
+
+// PickOverlay draws the hover outline and, once an element is picked, a
+// popover that adds it to the composer. Boxes are in stage pixels.
+function PickOverlay({ stage, hover, picked, picking, error, busy = false, onAdd, onRepick, onCancel }) {
+ if (!stage) return null;
+ const box = (o, className) => (
+ <div className={className} style={{ left: o.left, top: o.top, width: o.width, height: o.height }} aria-hidden="true">
+ <span
+ className={`agent-browser-pick-tag${o.top < 22 ? " is-inside" : ""}${
+ o.left + 260 > stage.clientWidth ? " is-end" : ""
+ }`}
+ >
+ <span className="agent-browser-pick-label">{o.label}</span>
+ <span className="agent-browser-pick-size">
+ {Math.round(o.width)}×{Math.round(o.height)}
+ </span>
+ </span>
+ </div>
+ );
+ let popover = null;
+ if (picked) {
+ const width = Math.min(280, stage.clientWidth - 16);
+ const below = picked.top + picked.height + 8;
+ const top = below + 92 <= stage.clientHeight ? below : Math.max(8, picked.top - 92 - 8);
+ const left = Math.max(8, Math.min(picked.left, stage.clientWidth - width - 8));
+ popover = (
+ <div className="agent-browser-pick-popover" role="dialog" aria-label="选中的元素" style={{ top, left, width }}>
+ <div className="agent-browser-pick-name">{picked.label}</div>
+ <div className="agent-browser-pick-actions">
+ <button type="button" className="agent-browser-pick-primary" disabled={busy} onClick={onAdd}>
+ <HarnessIcon name="chat" size={14} />
+ {busy ? "读取中…" : "加入对话框"}
+ </button>
+ <button type="button" className="agent-browser-pick-secondary" onClick={onRepick}>
+ 重新选择
+ </button>
+ <button type="button" className="agent-browser-pick-secondary" onClick={onCancel}>
+ 取消
+ </button>
+ </div>
+ </div>
+ );
+ }
+ return (
+ <>
+ {hover ? box(hover, "agent-browser-pick-outline") : null}
+ {picked ? box(picked, "agent-browser-pick-outline is-picked") : null}
+ {popover}
+ {picking ? (
+ <div className="agent-browser-pick-hint" role="status">
+ 点击要标记的元素,Esc 退出
+ </div>
+ ) : null}
+ {error ? (
+ <div className="agent-browser-pick-hint" role="alert">
+ {error}
+ </div>
+ ) : null}
+ </>
+ );
+}
+
+// useScreencastPicker selects elements on the Agent browser screencast. The
+// page is inspected over CDP (operation "inspect"): hovering asks for the
+// element under the pointer, one request at a time with only the latest point
+// kept; a click asks again with the full description the composer sends.
+function useScreencastPicker(runId, stageRef, screenRef) {
+ const [picking, setPicking] = useState(false);
+ const [hover, setHover] = useState(null);
+ const [picked, setPicked] = useState(null);
+ const [inspection, setInspection] = useState(null);
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState("");
+ const generation = useRef(0);
+ const inflight = useRef(false);
+ const pending = useRef(null);
+
+ const inspect = useCallback(
+ async (point, detail) => {
+ const state = await api(`/api/runs/${encodeURIComponent(runId)}/browser`, {
+ method: "POST",
+ body: JSON.stringify({ operation: "inspect", point, detail }),
+ });
+ return state?.inspect || null;
+ },
+ [runId],
+ );
+ // box maps an inspection (page CSS px) onto the stage.
+ const box = useCallback(
+ (found) => {
+ const stage = stageRef.current, screen = screenRef.current;
+ if (!found?.rect || !found.viewport?.width || !stage || !screen) return null;
+ const s = stage.getBoundingClientRect(), img = screen.getBoundingClientRect();
+ const scale = img.width / found.viewport.width;
+ return {
+ left: img.left - s.left + (found.rect.x || 0) * scale,
+ top: img.top - s.top + (found.rect.y || 0) * scale,
+ width: found.rect.width * scale,
+ height: found.rect.height * scale,
+ label: found.label || "",
+ };
+ },
+ [stageRef, screenRef],
+ );
+ const pointOf = (e) => {
+ const rect = screenRef.current?.getBoundingClientRect();
+ if (!rect?.width || !rect.height) return null;
+ const x = (e.clientX - rect.left) / rect.width, y = (e.clientY - rect.top) / rect.height;
+ return x < 0 || y < 0 || x > 1 || y > 1 ? null : { x, y };
+ };
+ const pump = useCallback(async () => {
+ if (inflight.current || !pending.current) return;
+ const { point, gen } = pending.current;
+ pending.current = null;
+ inflight.current = true;
+ try {
+ const found = await inspect(point, false);
+ if (gen === generation.current) setHover(box(found));
+ } catch (err) {
+ if (gen === generation.current) setError(err.message || "无法检查页面元素");
+ } finally {
+ inflight.current = false;
+ pump();
+ }
+ }, [inspect, box]);
+
+ const clear = useCallback(() => {
+ generation.current += 1;
+ pending.current = null;
+ setPicking(false);
+ setHover(null);
+ setPicked(null);
+ setInspection(null);
+ setBusy(false);
+ setError("");
+ }, []);
+ const repick = useCallback(() => {
+ clear();
+ setPicking(true);
+ }, [clear]);
+ const active = picking || !!picked;
+ useEffect(() => {
+ if (!active) return undefined;
+ // Capture phase: Escape leaves picking before the screencast forwards it.
+ const onKey = (e) => {
+ if (e.key !== "Escape") return;
+ e.preventDefault();
+ e.stopPropagation();
+ clear();
+ };
+ document.addEventListener("keydown", onKey, true);
+ return () => document.removeEventListener("keydown", onKey, true);
+ }, [active, clear]);
+ useEffect(() => clear, [runId, clear]);
+
+ return {
+ picking,
+ active,
+ hover,
+ picked,
+ inspection,
+ busy,
+ error,
+ clear,
+ repick,
+ toggle: () => (picking ? clear() : repick()),
+ // The page scrolled under an outline: drop it until the pointer moves.
+ scrolled: () => {
+ generation.current += 1;
+ setHover(null);
+ },
+ leave: () => {
+ generation.current += 1;
+ pending.current = null;
+ setHover(null);
+ },
+ move: (e) => {
+ const point = pointOf(e);
+ if (!point) return;
+ setError("");
+ pending.current = { point, gen: generation.current };
+ pump();
+ },
+ select: async (e) => {
+ const point = pointOf(e);
+ if (!point) return;
+ generation.current += 1;
+ const gen = generation.current;
+ pending.current = null;
+ setPicking(false);
+ setBusy(true);
+ const s = stageRef.current?.getBoundingClientRect() || { left: 0, top: 0 };
+ setPicked(hover ? { ...hover } : { left: e.clientX - s.left, top: e.clientY - s.top, width: 0, height: 0, label: "…" });
+ try {
+ const found = await inspect(point, true);
+ if (gen !== generation.current) return;
+ if (!found?.element) {
+ setPicked(null);
+ setPicking(true);
+ setError("这个位置没有可选的元素");
+ } else {
+ setInspection(found);
+ setPicked(box(found));
+ }
+ } catch (err) {
+ if (gen !== generation.current) return;
+ setPicked(null);
+ setPicking(true);
+ setError(err.message || "无法检查页面元素");
+ } finally {
+ if (gen === generation.current) setBusy(false);
+ }
+ },
+ };
+}
+
+// ScreencastBrowser shows the Agent browser of the conversation's workspace:
+// the browser Main Bot drives with workbench_browser_*, running on the
+// execution machine next to the workspace. The user watches it live and can
+// take over. Protocol and input handling follow the task panel in
+// web/collaboration.js: one WebSocket carries frames and input, plain polling
+// is the fallback.
+function ScreencastBrowser({
+ runId,
+ request = null,
+ onRequestHandled = () => {},
+ onNative = () => {},
+ onAddElement = null,
+}) {
const stageRef = useRef(null);
const screenRef = useRef(null);
const imeRef = useRef(null);
@@ -27,6 +612,9 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
const [notice, setNotice] = useState("");
const [address, setAddress] = useState("");
const [denied, setDenied] = useState(false);
+ const pick = useScreencastPicker(runId, stageRef, screenRef);
+ const picking = useRef(false);
+ picking.current = pick.active;
useEffect(() => {
if (!runId) return undefined;
@@ -267,53 +855,24 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
queue.push(ev);
flush();
}
- // Scrolling is shown locally first: the current frame slides by the pending
- // delta until a frame captured after that scroll replaces it.
- let wheel = null, wheelTimer = 0, settleTimer = 0, scrolls = [], shiftX = 0, shiftY = 0, frameLag = 80;
+ // Scrolls are not previewed by sliding the current frame: that moved the
+ // page's fixed parts and scrollbar with it and bared the stage below. The
+ // stream delivers the scrolled frame in well under 100ms instead.
+ let wheel = null, wheelTimer = 0;
function point(e) {
const rect = screen.getBoundingClientRect();
return {
- x: (e.clientX - rect.left + shiftX) / (rect.width || 1),
- y: (e.clientY - rect.top + shiftY) / (rect.height || 1),
+ x: (e.clientX - rect.left) / (rect.width || 1),
+ y: (e.clientY - rect.top) / (rect.height || 1),
};
}
- function applyShift() {
- const scale = (screen.getBoundingClientRect().width || 1) / (state?.width || 1);
- const limit = stage.clientHeight || 0;
- let dx = wheel?.deltaX || 0, dy = wheel?.deltaY || 0;
- for (const b of scrolls) {
- dx += b.dx;
- dy += b.dy;
- }
- shiftX = Math.max(-limit, Math.min(limit, -dx * scale));
- shiftY = Math.max(-limit, Math.min(limit, -dy * scale));
- screen.style.transform = shiftX || shiftY ? `translate(${shiftX}px,${shiftY}px)` : "";
- }
- function settleScroll(age) {
- const now = Date.now();
- scrolls = scrolls.filter((b) => now - b.at < age);
- applyShift();
- }
function flushWheel() {
wheelTimer = 0;
if (!wheel) return;
const ev = wheel;
wheel = null;
- scrolls.push({ dx: ev.deltaX, dy: ev.deltaY, at: Date.now() });
push(ev);
- applyShift();
- clearTimeout(settleTimer);
- settleTimer = setTimeout(() => settleScroll(400), 500);
}
- // A frame newer than the round trip already includes the scroll.
- const onFrameLoad = () => {
- const now = Date.now(), first = scrolls.find((b) => !b.seen);
- if (first) {
- first.seen = true;
- frameLag = Math.round(frameLag * 0.7 + (now - first.at) * 0.3);
- }
- settleScroll(Math.max(50, Math.min(400, frameLag)));
- };
const mouse = (e, action) => ({
type: "mouse",
action,
@@ -323,18 +882,21 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
clickCount: e.detail || 1,
modifiers: modifiers(e),
});
+ // While picking, the capture layer owns the pointer: nothing reaches the page.
+ const onScreen = (e) => !picking.current && (e.target === screen || e.target === stage);
const onPointerDown = (e) => {
- if (!state?.available || (e.target !== screen && e.target !== stage)) return;
+ if (!state?.available || !onScreen(e)) return;
e.preventDefault();
stage.setPointerCapture?.(e.pointerId);
ime.focus({ preventScroll: true });
+ claim();
push(mouse(e, "down"));
};
const onPointerUp = (e) => {
- if (state?.available) push(mouse(e, "up"));
+ if (state?.available && onScreen(e)) push(mouse(e, "up"));
};
const onPointerMove = (e) => {
- if (!state?.available || readOnly) return;
+ if (!state?.available || readOnly || !onScreen(e)) return;
pendingMove = {
type: "mouse",
action: "move",
@@ -354,6 +916,7 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
};
const onWheel = (e) => {
if (!state?.available || readOnly) return;
+ pick.scrolled();
e.preventDefault();
const unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? state.height || 600 : 1;
wheel = {
@@ -363,13 +926,14 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
deltaY: (wheel?.deltaY || 0) + e.deltaY * unit,
modifiers: modifiers(e),
};
- applyShift();
+ claim();
if (!wheelTimer) wheelTimer = setTimeout(flushWheel, 40);
};
const onContextMenu = (e) => e.preventDefault();
const onStageFocus = () => ime.focus({ preventScroll: true });
const onKeyDown = (e) => {
if (e.isComposing || e.keyCode === 229) return;
+ if (picking.current && e.key === "Escape") return;
if ((e.metaKey || e.ctrlKey) && ["v", "c", "x"].includes(e.key.toLowerCase())) return;
// Handled here so Escape or Tab reach the page instead of closing the panel.
e.preventDefault();
@@ -398,20 +962,43 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
if (ime.value) push({ type: "text", text: ime.value });
ime.value = "";
};
+ // wantedSize is the Agent browser viewport that fills this panel.
+ function wantedSize() {
+ const rect = stage.getBoundingClientRect();
+ if (!state?.available || rect.width < 120 || rect.height < 120) return null;
+ const width = Math.round(Math.min(1600, Math.max(1024, rect.width)));
+ const height = Math.round(Math.min(2000, Math.max(500, (width * rect.height) / rect.width)));
+ // Frames carry up to the panel's device pixels: the Agent browser
+ // renders at 2x, so a high-DPI panel is not shown upscaled frames.
+ const frameWidth = Math.round(rect.width * Math.min(window.devicePixelRatio || 1, 2));
+ return { width, height, frameWidth };
+ }
+ function sendSize(force = false) {
+ if (!alive || readOnly) return;
+ const size = wantedSize();
+ if (!size) return;
+ const key = `${size.width}x${size.height}@${size.frameWidth}`;
+ if (key === sentSize && !force) return;
+ sentSize = key;
+ command({ operation: "resize", ...size });
+ }
function fit() {
clearTimeout(sizeTimer);
- sizeTimer = setTimeout(() => {
- if (!alive || readOnly) return;
- const rect = stage.getBoundingClientRect();
- if (!state?.available || rect.width < 120 || rect.height < 120) return;
- const width = Math.round(Math.min(1600, Math.max(1024, rect.width)));
- const height = Math.round(Math.min(2000, Math.max(500, (width * rect.height) / rect.width)));
- const frameWidth = Math.round(rect.width * Math.min(window.devicePixelRatio || 1, 1.5));
- const key = `${width}x${height}@${frameWidth}`;
- if (key === sentSize) return;
- sentSize = key;
- command({ operation: "resize", width, height, frameWidth });
- }, 400);
+ sizeTimer = setTimeout(sendSize, 400);
+ }
+ // Every panel watching the run shares one Agent browser, and the last one
+ // to resize it wins. The panel being used takes the size back when it no
+ // longer fits, but only on input, so two open panels never take turns.
+ let claimedAt = 0;
+ function claim() {
+ const size = wantedSize();
+ if (!size || readOnly || Date.now() - claimedAt < 1000) return;
+ // Frames report the viewport with the page's scrollbar (about 15px)
+ // while resize sets it without, so only a clearly different size counts.
+ const off = (a, b) => !(a > 0) || Math.abs(a - b) > 24;
+ if (!off(state.width, size.width) && !off(state.height, size.height)) return;
+ claimedAt = Date.now();
+ sendSize(true);
}
const listeners = [
@@ -421,7 +1008,6 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
[stage, "wheel", onWheel, { passive: false }],
[stage, "contextmenu", onContextMenu],
[stage, "focus", onStageFocus],
- [screen, "load", onFrameLoad],
[ime, "keydown", onKeyDown],
[ime, "keyup", onKeyUp],
[ime, "compositionstart", onCompositionStart],
@@ -441,7 +1027,7 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
controller.current = null;
for (const [target, type, fn, options] of listeners) target.removeEventListener(type, fn, options);
observer?.disconnect();
- for (const timer of [noticeTimer, reconnectTimer, sizeTimer, moveTimer, wheelTimer, settleTimer]) clearTimeout(timer);
+ for (const timer of [noticeTimer, reconnectTimer, sizeTimer, moveTimer, wheelTimer]) clearTimeout(timer);
screen.onload = screen.onerror = null;
ws?.close();
ws = null;
@@ -495,8 +1081,22 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
readOnly={denied}
onChange={(event) => setAddress(event.target.value)}
/>
+ {onAddElement && !denied ? (
+ <AnnotateButton picking={pick.picking} disabled={!page.available} onClick={pick.toggle} />
+ ) : null}
</form>
{navButton("plus", "新标签页", "newTab")}
+ {devServerTarget(address) ? (
+ <button
+ type="button"
+ className="agent-browser-btn"
+ title="在原生预览中打开(你的浏览器直接渲染,操作更流畅)"
+ aria-label="在原生预览中打开"
+ onClick={() => onNative(devServerTarget(address))}
+ >
+ <HarnessIcon name="preview-expand" size={15} />
+ </button>
+ ) : null}
</div>
{page.tabs.length > 1 ? (
<div className="agent-browser-tabs" role="tablist" aria-label="浏览器标签页">
@@ -555,6 +1155,32 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =
) : null}
</div>
) : null}
+ {pick.picking ? (
+ <div
+ className="agent-browser-pick-capture"
+ data-testid="agent-browser-pick-capture"
+ onPointerMove={pick.move}
+ onPointerLeave={pick.leave}
+ onPointerDown={(e) => e.preventDefault()}
+ onClick={pick.select}
+ />
+ ) : null}
+ {pick.active ? (
+ <PickOverlay
+ stage={stageRef.current}
+ hover={pick.picking ? pick.hover : null}
+ picked={pick.picked}
+ picking={pick.picking}
+ error={pick.error}
+ busy={pick.busy}
+ onAdd={() => {
+ if (pick.inspection?.element) onAddElement?.(inspectionReference(pick.inspection));
+ pick.clear();
+ }}
+ onRepick={pick.repick}
+ onCancel={pick.clear}
+ />
+ ) : null}
</div>
<div className="agent-browser-foot">
<span>{status}</span>
diff --git a/chatbot/src/App.jsx b/chatbot/src/App.jsx
index e61278a9..a70dee72 100644
--- a/chatbot/src/App.jsx
+++ b/chatbot/src/App.jsx
@@ -54,6 +54,7 @@ import MentionPicker from "./MentionPicker";
import Participants, { Avatar, authorFor } from "./Participants";
import IssueContext from "./IssueContext";
import { mentionAt, insertMention } from "./mentions.js";
+import { elementTitle } from "./element-context.js";
import HarnessIcon from "./HarnessIcon";
import QueuedMessages from "./QueuedMessages";
import "./message-queue.css";
@@ -814,6 +815,16 @@ export default function App() {
closeMenu();
input.current?.focus();
}
+ // addElementReference takes an element picked in the native preview: it
+ // rides along as a reference chip, without a token in the text.
+ function addElementReference(ref) {
+ if (references.length >= 12) {
+ chat.setError("每轮最多引用 12 个对象。");
+ return;
+ }
+ setReferences((old) => [...old, ref]);
+ requestAnimationFrame(() => input.current?.focus());
+ }
function addReference(ref) {
if (
references.length >= 12 &&
@@ -1509,8 +1520,16 @@ export default function App() {
{message.references?.length ? (
<div className="message-references">
{message.references.map((ref) => (
- <span key={ref.type + ref.id}>
- @{ref.name || ref.id}
+ <span
+ key={ref.type + ref.id}
+ title={ref.type === "element" ? elementTitle(ref) : undefined}
+ >
+ {ref.type === "element" ? (
+ <HarnessIcon name="crosshair" size={12} />
+ ) : (
+ "@"
+ )}
+ {ref.name || ref.id}
</span>
))}
</div>
@@ -1843,9 +1862,15 @@ export default function App() {
{references.map((ref) => (
<span
key={ref.type + ref.id}
- className="composer-attachment-file"
+ className={`composer-attachment-file${ref.type === "element" ? " is-element" : ""}`}
+ title={ref.type === "element" ? elementTitle(ref) : undefined}
>
- @{ref.name || ref.id}
+ {ref.type === "element" ? (
+ <HarnessIcon name="crosshair" size={13} />
+ ) : (
+ "@"
+ )}
+ {ref.name || ref.id}
<button
aria-label={`移除引用 ${ref.name || ref.id}`}
onClick={() => {
@@ -2134,6 +2159,7 @@ export default function App() {
workspaceRunId={workspaceRunId}
browserRequest={browserRequest}
onBrowserRequestHandled={browserRequestHandled}
+ onAddElement={chat.readOnly ? null : addElementReference}
onClose={() => setPanel(false)}
/>
) : null}
diff --git a/chatbot/src/ArtifactPanel.jsx b/chatbot/src/ArtifactPanel.jsx
index ba3c546a..536d0cb9 100644
--- a/chatbot/src/ArtifactPanel.jsx
+++ b/chatbot/src/ArtifactPanel.jsx
@@ -192,6 +192,7 @@ export default function ArtifactPanel({
workspaceRunId = "",
browserRequest = null,
onBrowserRequestHandled = () => {},
+ onAddElement = null,
}) {
const root = useRef(null);
const searchInput = useRef(null);
@@ -577,6 +578,15 @@ export default function ArtifactPanel({
runId={workspaceRunId}
request={browserRequest}
onRequestHandled={onBrowserRequestHandled}
+ onAddElement={
+ onAddElement &&
+ ((ref) => {
+ onAddElement(ref);
+ // A compact panel covers the composer: go back so the
+ // new chip is in view.
+ if (layout.compact) onClose();
+ })
+ }
/>
) : (
<div className="agent-browser-empty" role="status">
diff --git a/chatbot/src/agent-browser-links.js b/chatbot/src/agent-browser-links.js
index 37dd0c91..08685a22 100644
--- a/chatbot/src/agent-browser-links.js
+++ b/chatbot/src/agent-browser-links.js
@@ -68,3 +68,49 @@ export function remarkWorkspaceLinks() {
};
return walk;
}
+
+// devServerTarget returns the port and path of a plain-HTTP address on the
+// execution machine (a dev server), or null. Those pages are shown natively:
+// the platform proxies the port, so the user's own browser renders them.
+// Workspace files (file:///workspace/…) are served the same way; their
+// "port" is "file".
+export function devServerTarget(raw) {
+ let url;
+ try {
+ url = new URL(String(raw || "").trim());
+ } catch {
+ return null;
+ }
+ if (url.protocol === "file:") {
+ if (url.host || !url.pathname || url.pathname === "/") return null;
+ return { port: devFile, path: `${url.pathname}${url.search}${url.hash}` };
+ }
+ if (url.protocol !== "http:" || !localHost.test(url.hostname)) return null;
+ const port = Number(url.port || 80);
+ if (!Number.isInteger(port) || port <= 0 || port > 65535) return null;
+ return { port, path: `${url.pathname || "/"}${url.search}${url.hash}` };
+}
+
+const devFile = "file";
+
+// devServerOrigin is how the page's address starts on the execution machine.
+export const devServerOrigin = (port) => (port === devFile ? "file://" : `http://localhost:${port}`);
+
+// devServerLabel names the target in the panel.
+export const devServerLabel = (port) => (port === devFile ? "工作区文件" : `localhost:${port}`);
+
+export const devServerPrefix = (runId, port) =>
+ `/devpreview/${encodeURIComponent(runId)}/${port}`;
+
+// devServerURL is where the platform serves the dev server page.
+export const devServerURL = (runId, port, path = "/") =>
+ `${devServerPrefix(runId, port)}${path.startsWith("/") ? path : `/${path}`}`;
+
+// devServerAddress maps a proxied page location back to the address the
+// dev server has on the execution machine.
+export function devServerAddress(runId, port, location) {
+ const prefix = devServerPrefix(runId, port);
+ const path = location?.pathname || "/";
+ const rest = path === prefix ? "/" : path.startsWith(`${prefix}/`) ? path.slice(prefix.length) : path;
+ return `${devServerOrigin(port)}${rest}${location?.search || ""}${location?.hash || ""}`;
+}
diff --git a/chatbot/src/agent-browser.css b/chatbot/src/agent-browser.css
index 4bc22237..54715d3b 100644
--- a/chatbot/src/agent-browser.css
+++ b/chatbot/src/agent-browser.css
@@ -55,7 +55,8 @@
background: var(--subtle);
color: var(--muted);
}
-.agent-browser-address:focus-within {
+/* Only typing an address highlights the bar, not its 标记元素 button. */
+.agent-browser-address:has(input:focus) {
border-color: var(--accent);
}
.agent-browser-address input {
@@ -154,6 +155,21 @@
filter: grayscale(1);
opacity: 0.35;
}
+/* Native preview: the dev server page itself, rendered by this browser. */
+.agent-browser-native {
+ display: flex;
+ background: #fff;
+}
+.agent-browser-frame {
+ flex: 1;
+ width: 100%;
+ height: 100%;
+ border: 0;
+ background: #fff;
+}
+.agent-browser-loading {
+ background: var(--subtle);
+}
.agent-browser-ime {
position: absolute;
top: 0;
@@ -263,3 +279,186 @@
[data-theme="dark"] .markdown .agent-browser-link code {
color: #8ab4f8;
}
+/* Element picking in a native preview: outlines above the frame, a popover
+ * that adds the picked element to the composer. */
+.agent-browser-btn.is-on {
+ background: var(--hover);
+ color: var(--text);
+}
+.agent-browser-pick-outline {
+ position: absolute;
+ z-index: 2;
+ box-sizing: border-box;
+ border: 1.5px solid #3b82f6;
+ border-radius: 3px;
+ background: rgb(59 130 246 / 0.12);
+ pointer-events: none;
+}
+.agent-browser-pick-outline.is-picked {
+ border-width: 2px;
+ background: rgb(59 130 246 / 0.08);
+}
+.agent-browser-pick-tag {
+ position: absolute;
+ display: flex;
+ gap: 6px;
+ bottom: 100%;
+ left: -1.5px;
+ max-width: 260px;
+ margin-bottom: 4px;
+ padding: 2px 6px;
+ overflow: hidden;
+ border-radius: 4px;
+ background: #1f2937;
+ color: #fff;
+ font-size: 11px;
+ line-height: 16px;
+ white-space: nowrap;
+ text-overflow: ellipsis;
+}
+/* The size stays visible; a long label is the part that gets cut. */
+.agent-browser-pick-label {
+ min-width: 0;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+.agent-browser-pick-size {
+ flex: none;
+ opacity: 0.7;
+}
+.agent-browser-pick-tag.is-end {
+ right: -1.5px;
+ left: auto;
+}
+.agent-browser-pick-tag.is-inside.is-end {
+ right: 0;
+ left: auto;
+ border-radius: 0 0 0 4px;
+}
+.agent-browser-pick-tag.is-inside {
+ top: 0;
+ bottom: auto;
+ left: 0;
+ margin: 0;
+ border-radius: 0 0 4px 0;
+}
+.agent-browser-pick-popover {
+ position: absolute;
+ z-index: 3;
+ box-sizing: border-box;
+ padding: 10px;
+ border: 1px solid var(--line);
+ border-radius: 12px;
+ background: var(--surface);
+ box-shadow: 0 8px 28px rgb(0 0 0 / 0.14), 0 2px 6px rgb(0 0 0 / 0.06);
+ color: var(--text);
+ font-size: 12.5px;
+ animation: agent-browser-pick-in 160ms cubic-bezier(0.2, 0.7, 0.3, 1);
+}
+.agent-browser-pick-name {
+ margin-bottom: 8px;
+ overflow: hidden;
+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+ font-size: 12px;
+ white-space: nowrap;
+ text-overflow: ellipsis;
+}
+.agent-browser-pick-actions {
+ display: flex;
+ gap: 6px;
+}
+.agent-browser-pick-actions button {
+ display: inline-flex;
+ align-items: center;
+ gap: 5px;
+ height: 28px;
+ padding: 0 10px;
+ border-radius: 7px;
+ font-size: 12.5px;
+ cursor: pointer;
+ transition: background-color 120ms, color 120ms;
+}
+.agent-browser-pick-primary {
+ border: 0;
+ background: var(--accent);
+ color: var(--surface);
+}
+.agent-browser-pick-primary:hover {
+ background: color-mix(in srgb, var(--accent) 86%, var(--surface));
+}
+.agent-browser-pick-secondary {
+ border: 1px solid var(--line);
+ background: transparent;
+ color: var(--muted);
+}
+.agent-browser-pick-secondary:hover {
+ background: var(--hover);
+ color: var(--text);
+}
+.agent-browser-pick-hint {
+ position: absolute;
+ bottom: 10px;
+ left: 50%;
+ z-index: 3;
+ padding: 4px 10px;
+ transform: translateX(-50%);
+ border-radius: 999px;
+ background: rgb(31 41 55 / 0.88);
+ color: #fff;
+ font-size: 12px;
+ pointer-events: none;
+}
+@keyframes agent-browser-pick-in {
+ from {
+ opacity: 0;
+ transform: translateY(-4px);
+ }
+}
+@media (prefers-reduced-motion: reduce) {
+ .agent-browser-pick-popover {
+ animation: none;
+ }
+}
+/* Element picking on the screencast: a transparent layer takes the pointer
+ * so hovering and the pick click never reach the Agent browser. */
+.agent-browser-pick-capture {
+ position: absolute;
+ inset: 0;
+ z-index: 1;
+ cursor: crosshair;
+}
+.agent-browser-pick-primary:disabled {
+ opacity: 0.6;
+ cursor: progress;
+}
+/* "标记元素" sits at the end of the address bar, like the page's own tool. */
+.agent-browser-annotate {
+ display: inline-flex;
+ flex: none;
+ align-items: center;
+ gap: 4px;
+ height: 22px;
+ margin-right: -6px;
+ padding: 0 7px;
+ border: 0;
+ border-radius: 999px;
+ background: transparent;
+ color: var(--muted);
+ font: inherit;
+ font-size: 12px;
+ white-space: nowrap;
+ cursor: pointer;
+ transition: background-color 120ms, color 120ms;
+}
+.agent-browser-annotate:hover:not(:disabled) {
+ background: var(--hover);
+ color: var(--text);
+}
+.agent-browser-annotate.is-on {
+ background: color-mix(in srgb, var(--accent) 14%, transparent);
+ color: var(--accent);
+}
+.agent-browser-annotate:disabled {
+ opacity: 0.4;
+ cursor: default;
+}
diff --git a/chatbot/src/element-context.js b/chatbot/src/element-context.js
new file mode 100644
index 00000000..dbb4fa7b
--- /dev/null
+++ b/chatbot/src/element-context.js
@@ -0,0 +1,239 @@
+// Element context: what the user points at in a native preview, described so
+// the Agent can find it in code (component and source file when the dev build
+// exposes them) and in its own browser (a unique selector).
+
+const styleKeys = [
+ "display",
+ "position",
+ "width",
+ "height",
+ "margin",
+ "padding",
+ "gap",
+ "flex-direction",
+ "justify-content",
+ "align-items",
+ "color",
+ "background-color",
+ "font-size",
+ "font-weight",
+ "line-height",
+ "border",
+ "border-radius",
+ "box-shadow",
+ "opacity",
+];
+
+// Values that only restate the default are left out.
+const boringStyle = /^(none|normal|auto|0px|0px none.*|rgba\(0, 0, 0, 0\))$/;
+const styleDefaults = { position: "static", opacity: "1", "font-weight": "400" };
+const layoutKeys = new Set(["gap", "flex-direction", "justify-content", "align-items"]);
+
+const agentRefAttr = / data-agentflow-ref="[^"]*"/g;
+const textLimit = 200;
+const htmlLimit = 3000;
+
+const clip = (value, limit) => {
+ const text = String(value || "").replace(/\s+/g, " ").trim();
+ return text.length > limit ? `${text.slice(0, limit)}…` : text;
+};
+
+const escapeIdent = (value) =>
+ typeof CSS !== "undefined" && CSS.escape ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+
+// Hashed or utility classes say little about where the element is; skip
+// classes that look generated (css-1x2y3z, sc-abc123) and keep at most two.
+const meaningfulClasses = (el) =>
+ [...(el.classList || [])]
+ .filter((c) => !/^(css|sc|jsx|svelte|emotion)-|^[a-z]{1,3}[-_]?(?=[0-9a-z]*[0-9])[0-9a-z]{5,}$/i.test(c) && c.length <= 40)
+ .slice(0, 2);
+
+function unique(doc, selector, el) {
+ try {
+ const found = doc.querySelectorAll(selector);
+ return found.length === 1 && found[0] === el;
+ } catch {
+ return false;
+ }
+}
+
+function step(el) {
+ const tag = el.tagName.toLowerCase();
+ const testId = el.getAttribute("data-testid") || el.getAttribute("data-test-id");
+ if (testId) return `${tag}[data-testid="${testId.replace(/"/g, '\\"')}"]`;
+ const classes = meaningfulClasses(el).map((c) => `.${escapeIdent(c)}`).join("");
+ return `${tag}${classes}`;
+}
+
+function nth(el, base) {
+ const parent = el.parentElement;
+ if (!parent) return base;
+ const same = [...parent.children].filter((c) => c.tagName === el.tagName);
+ return same.length > 1 ? `${base}:nth-of-type(${same.indexOf(el) + 1})` : base;
+}
+
+// elementSelector returns a selector that matches only el in its document.
+export function elementSelector(el) {
+ const doc = el.ownerDocument;
+ if (el.id && unique(doc, `#${escapeIdent(el.id)}`, el)) return `#${escapeIdent(el.id)}`;
+ const parts = [];
+ let node = el;
+ while (node && node.nodeType === 1 && node !== doc.documentElement) {
+ if (node !== el && node.id && unique(doc, `#${escapeIdent(node.id)}`, node)) {
+ parts.unshift(`#${escapeIdent(node.id)}`);
+ break;
+ }
+ parts.unshift(step(node));
+ const selector = parts.join(" > ");
+ if (unique(doc, selector, el)) return selector;
+ parts[0] = nth(node, parts[0]);
+ if (unique(doc, parts.join(" > "), el)) return parts.join(" > ");
+ node = node.parentElement;
+ }
+ return parts.join(" > ");
+}
+
+// elementLabel is the short name shown on the composer chip.
+export function elementLabel(el) {
+ const tag = el.tagName.toLowerCase();
+ const id = el.id ? `#${el.id}` : "";
+ const cls = !id && meaningfulClasses(el)[0] ? `.${meaningfulClasses(el)[0]}` : "";
+ const text = clip(el.innerText ?? el.textContent, 16);
+ return `${tag}${id}${cls}${text ? ` 「${text}」` : ""}`;
+}
+
+function sourceOf(file, line, column) {
+ if (!file) return "";
+ return `${file}${line ? `:${line}` : ""}${line && column ? `:${column}` : ""}`;
+}
+
+function reactComponent(el) {
+ const key = Object.keys(el).find((k) => k.startsWith("__reactFiber$") || k.startsWith("__reactInternalInstance$"));
+ let fiber = key ? el[key] : null;
+ let source = fiber?._debugSource || null;
+ for (let i = 0; fiber && i < 40; i++, fiber = fiber.return) {
+ source = source || fiber._debugSource || null;
+ const type = fiber.type;
+ if (typeof type === "function" || (type && typeof type === "object" && (type.render || type.type))) {
+ const name = type.displayName || type.name || type.render?.displayName || type.render?.name || type.type?.name;
+ if (name)
+ return {
+ framework: "React",
+ component: name,
+ source: sourceOf(source?.fileName, source?.lineNumber, source?.columnNumber),
+ };
+ }
+ }
+ return null;
+}
+
+function vueComponent(el) {
+ const instance = el.__vueParentComponent;
+ if (instance?.type) {
+ const type = instance.type;
+ const file = type.__file || "";
+ const name = type.name || type.__name || (file ? file.split("/").pop().replace(/\.vue$/, "") : "");
+ if (name || file) return { framework: "Vue", component: name, source: file };
+ }
+ const vm = el.__vue__;
+ if (vm?.$options) {
+ const file = vm.$options.__file || "";
+ return { framework: "Vue", component: vm.$options.name || vm.$options._componentTag || "", source: file };
+ }
+ return null;
+}
+
+function svelteComponent(el) {
+ const loc = el.__svelte_meta?.loc;
+ if (!loc?.file) return null;
+ return {
+ framework: "Svelte",
+ component: loc.file.split("/").pop().replace(/\.svelte$/, ""),
+ source: sourceOf(loc.file, loc.line != null ? loc.line + 1 : 0, loc.column != null ? loc.column + 1 : 0),
+ };
+}
+
+// componentOf finds the nearest framework component a dev build exposes on
+// the element or its ancestors.
+export function componentOf(el) {
+ for (let node = el, depth = 0; node && node.nodeType === 1 && depth < 25; node = node.parentElement, depth++) {
+ const found = reactComponent(node) || vueComponent(node) || svelteComponent(node);
+ if (found) return found;
+ }
+ return null;
+}
+
+// describeElement collects what the Agent needs about el. url is the page
+// address as the sandbox sees it (http://localhost:PORT/… or file://…).
+export function describeElement(el, url) {
+ const win = el.ownerDocument.defaultView;
+ const rect = el.getBoundingClientRect();
+ const computed = win.getComputedStyle(el);
+ const styles = {};
+ const layout = /flex|grid/.test(computed.getPropertyValue("display"));
+ for (const key of styleKeys) {
+ const value = computed.getPropertyValue(key);
+ if (!layout && layoutKeys.has(key)) continue;
+ if (!value || boringStyle.test(value) || styleDefaults[key] === value) continue;
+ styles[key] = value;
+ }
+ // The Agent's page snapshots tag elements with data-agentflow-ref; they mean
+ // nothing outside that snapshot.
+ const html = (el.outerHTML || "").replace(agentRefAttr, "");
+ const component = componentOf(el);
+ return {
+ url,
+ title: clip(el.ownerDocument.title, 120),
+ selector: elementSelector(el),
+ tag: el.tagName.toLowerCase(),
+ text: clip(el.innerText ?? el.textContent, textLimit),
+ html: html.length > htmlLimit ? `${html.slice(0, htmlLimit)}…` : html,
+ rect: {
+ x: Math.round(rect.left),
+ y: Math.round(rect.top),
+ width: Math.round(rect.width),
+ height: Math.round(rect.height),
+ },
+ viewport: { width: win.innerWidth, height: win.innerHeight },
+ styles,
+ ...(component?.component ? { component: component.component, framework: component.framework } : {}),
+ ...(component?.source ? { source: component.source } : {}),
+ };
+}
+
+let sequence = 0;
+
+// elementReference wraps a description as a composer reference.
+export function elementReference(el, url) {
+ sequence += 1;
+ return {
+ type: "element",
+ id: `el-${Date.now().toString(36)}-${sequence}`,
+ name: elementLabel(el),
+ element: describeElement(el, url),
+ };
+}
+
+// inspectionReference wraps what the Agent browser reported for a picked
+// element (internal/runtime/browser_inspect.go) as a composer reference.
+export function inspectionReference(inspection) {
+ sequence += 1;
+ return {
+ type: "element",
+ id: `el-${Date.now().toString(36)}-${sequence}`,
+ name: inspection.label || inspection.element?.tag || "元素",
+ element: inspection.element,
+ };
+}
+
+// elementTitle is the hover text of an element chip.
+export function elementTitle(ref) {
+ const e = ref?.element || {};
+ return [
+ e.component ? `${e.component}${e.source ? ` · ${e.source}` : ""}` : e.source,
+ e.selector,
+ e.url,
+ ]
+ .filter(Boolean)
+ .join("\n");
+}
diff --git a/chatbot/src/styles.css b/chatbot/src/styles.css
index 085319f1..04f201b5 100644
--- a/chatbot/src/styles.css
+++ b/chatbot/src/styles.css
@@ -651,6 +651,17 @@ small {
font-size: 11px;
color: var(--muted);
}
+.message-references > span {
+ display: inline-flex;
+ align-items: center;
+ gap: 3px;
+ min-width: 0;
+}
+/* A page element picked in the native preview: the label is page text. */
+.composer-attachment-file.is-element {
+ max-width: 260px;
+ white-space: nowrap;
+}
/* The send time stays visible; the copy and info buttons appear on hover. */
.message-actions {
display: flex;
diff --git a/chatbot/tests/agent-browser.test.jsx b/chatbot/tests/agent-browser.test.jsx
index 2b241c3b..d12ccde7 100644
--- a/chatbot/tests/agent-browser.test.jsx
+++ b/chatbot/tests/agent-browser.test.jsx
@@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import AgentBrowser from "../src/AgentBrowser.jsx";
import ArtifactPanel from "../src/ArtifactPanel.jsx";
import { RichContent } from "../src/RichContent.jsx";
-import { AgentBrowserLinks, agentBrowserTarget } from "../src/agent-browser-links.js";
+import { AgentBrowserLinks, agentBrowserTarget, devServerAddress, devServerTarget } from "../src/agent-browser-links.js";
class FakeSocket {
static OPEN = 1;
@@ -56,6 +56,26 @@ beforeEach(() => {
});
afterEach(() => vi.unstubAllGlobals());
+describe("devServerTarget", () => {
+ test("maps plain-HTTP execution machine addresses to a port and path", () => {
+ expect(devServerTarget("http://localhost:5173/a/b?c=1#d")).toEqual({ port: 5173, path: "/a/b?c=1#d" });
+ expect(devServerTarget("http://127.0.0.1:3000")).toEqual({ port: 3000, path: "/" });
+ expect(devServerTarget("http://localhost/")).toEqual({ port: 80, path: "/" });
+ for (const value of ["https://localhost:5173/", "http://example.com:5173/", "localhost:5173", "file:///", "file://host/a.html"])
+ expect(devServerTarget(value)).toBeNull();
+ expect(devServerAddress("run-1", 5173, { pathname: "/devpreview/run-1/5173/x", search: "?q", hash: "" })).toBe("http://localhost:5173/x?q");
+ expect(devServerAddress("run-1", 5173, { pathname: "/x", search: "", hash: "" })).toBe("http://localhost:5173/x");
+ });
+
+ test("maps workspace files to the file channel", () => {
+ expect(devServerTarget("file:///workspace/demo.html#top")).toEqual({ port: "file", path: "/workspace/demo.html#top" });
+ expect(devServerTarget("file:///workspace/页面.html")).toEqual({ port: "file", path: "/workspace/%E9%A1%B5%E9%9D%A2.html" });
+ expect(devServerAddress("run-1", "file", { pathname: "/devpreview/run-1/file/workspace/site/", search: "", hash: "" })).toBe(
+ "file:///workspace/site/",
+ );
+ });
+});
+
describe("agentBrowserTarget", () => {
test("accepts only addresses on the execution machine", () => {
expect(agentBrowserTarget("http://localhost:5173/sidebar-demo-2.html")).toBe("http://localhost:5173/sidebar-demo-2.html");
@@ -83,7 +103,7 @@ describe("AgentBrowser", () => {
test("streams the workspace browser and opens requested links", async () => {
const handled = vi.fn();
const { rerender, container } = render(
- <AgentBrowser runId="run-1" request={{ id: "r1", url: "file:///workspace/demo.html" }} onRequestHandled={handled} />,
+ <AgentBrowser runId="run-1" request={{ id: "r1", url: "https://example.org/demo.html" }} onRequestHandled={handled} />,
);
const socket = lastSocket();
expect(socket.url).toBe(`ws://${location.host}/api/runs/run-1/browser/stream`);
@@ -91,7 +111,7 @@ describe("AgentBrowser", () => {
await waitFor(() => expect(fetchMock).toHaveBeenCalled());
const [path, init] = fetchMock.mock.calls[0];
expect(path).toBe("/api/runs/run-1/browser");
- expect(JSON.parse(init.body)).toMatchObject({ operation: "navigate", url: "file:///workspace/demo.html", ensure: true });
+ expect(JSON.parse(init.body)).toMatchObject({ operation: "navigate", url: "https://example.org/demo.html", ensure: true });
expect(handled).toHaveBeenCalledWith("r1");
act(() => {
@@ -117,12 +137,12 @@ describe("AgentBrowser", () => {
expect(socket.sent).toContainEqual({ operation: "activate", targetId: "t2" });
rerender(
- <AgentBrowser runId="run-1" request={{ id: "r2", url: "http://localhost:5173/" }} onRequestHandled={handled} />,
+ <AgentBrowser runId="run-1" request={{ id: "r2", url: "https://example.com/" }} onRequestHandled={handled} />,
);
- expect(socket.sent).toContainEqual({ operation: "navigate", url: "http://localhost:5173/" });
+ expect(socket.sent).toContainEqual({ operation: "navigate", url: "https://example.com/" });
expect(handled).toHaveBeenCalledWith("r2");
rerender(
- <AgentBrowser runId="run-1" request={{ id: "r2", url: "http://localhost:5173/" }} onRequestHandled={handled} />,
+ <AgentBrowser runId="run-1" request={{ id: "r2", url: "https://example.com/" }} onRequestHandled={handled} />,
);
expect(socket.sent.filter((m) => m.operation === "navigate")).toHaveLength(1);
@@ -133,6 +153,72 @@ describe("AgentBrowser", () => {
expect(fireEvent.keyDown(ime, { key: "Escape", code: "Escape", keyCode: 27 })).toBe(false);
});
+ test("opens dev server addresses natively and switches back to the screencast", async () => {
+ const handled = vi.fn();
+ const open = vi.spyOn(window, "open").mockImplementation(() => null);
+ const { container } = render(
+ <AgentBrowser runId="run-5" request={{ id: "n1", url: "http://localhost:5173/admin?tab=2" }} onRequestHandled={handled} />,
+ );
+ expect(handled).toHaveBeenCalledWith("n1");
+ const frame = container.querySelector("iframe.agent-browser-frame");
+ expect(frame).toHaveAttribute("src", "/devpreview/run-5/5173/admin?tab=2");
+ expect(screen.getByLabelText("网址")).toHaveValue("http://localhost:5173/admin?tab=2");
+ // No screencast while the page renders natively.
+ expect(FakeSocket.instances).toHaveLength(0);
+ fireEvent.click(screen.getByLabelText("在新窗口打开"));
+ expect(open).toHaveBeenCalledWith(expect.stringContaining("/devpreview/run-5/5173/"), "_blank", "noopener");
+ fireEvent.click(screen.getByLabelText("切换到 Agent 浏览器画面"));
+ expect(container.querySelector("iframe.agent-browser-frame")).toBeNull();
+ expect(lastSocket().url).toContain("/api/runs/run-5/browser/stream");
+ });
+
+ test("opens workspace files natively", async () => {
+ const handled = vi.fn();
+ const { container } = render(
+ <AgentBrowser runId="run-6" request={{ id: "f1", url: "file:///workspace/demo.html" }} onRequestHandled={handled} />,
+ );
+ expect(handled).toHaveBeenCalledWith("f1");
+ expect(container.querySelector("iframe.agent-browser-frame")).toHaveAttribute("src", "/devpreview/run-6/file/workspace/demo.html");
+ expect(screen.getByLabelText("网址")).toHaveValue("file:///workspace/demo.html");
+ expect(screen.getByText("原生预览 · 工作区文件")).toBeInTheDocument();
+ expect(FakeSocket.instances).toHaveLength(0);
+ });
+
+ test("the panel in use takes the shared viewport back and never slides the frame", () => {
+ const { container } = render(<AgentBrowser runId="run-5" />);
+ const socket = lastSocket();
+ const stage = container.querySelector(".agent-browser-stage");
+ const img = container.querySelector(".agent-browser-screen");
+ // A tall panel; another viewer last sized the browser to 1024×866.
+ stage.getBoundingClientRect = () => ({ left: 0, top: 0, width: 600, height: 1200, right: 600, bottom: 1200, x: 0, y: 0 });
+ img.getBoundingClientRect = () => ({ left: 0, top: 0, width: 600, height: 507, right: 600, bottom: 507, x: 0, y: 0 });
+ act(() => {
+ socket.open();
+ socket.state({ ...live, width: 1024, height: 866 });
+ });
+ const resizes = () => socket.sent.filter((m) => m.operation === "resize");
+ fireEvent.wheel(stage, { deltaY: 120 });
+ expect(resizes()).toEqual([expect.objectContaining({ width: 1024, height: 2000 })]);
+ expect(img.style.transform).toBe("");
+ // Not again while the resize is on its way.
+ fireEvent.wheel(stage, { deltaY: 120 });
+ expect(resizes()).toHaveLength(1);
+ });
+
+ test("a panel whose size already fits leaves the viewport alone", () => {
+ const { container } = render(<AgentBrowser runId="run-6" />);
+ const socket = lastSocket();
+ const stage = container.querySelector(".agent-browser-stage");
+ stage.getBoundingClientRect = () => ({ left: 0, top: 0, width: 1024, height: 866, right: 1024, bottom: 866, x: 0, y: 0 });
+ act(() => {
+ socket.open();
+ // Frames count the page's 15px scrollbar the resize left out.
+ socket.state({ ...live, width: 1039, height: 866 });
+ });
+ fireEvent.wheel(stage, { deltaY: 120 });
+ expect(socket.sent.filter((m) => m.operation === "resize")).toHaveLength(0);
+ });
+
test("project viewers watch the browser without driving it", async () => {
const { container } = render(<AgentBrowser runId="run-2" />);
const socket = lastSocket();
@@ -242,6 +328,44 @@ describe("side panel browser tab", () => {
expect(FakeSocket.instances).toHaveLength(0);
});
+ test("a compact panel goes back to the chat after adding an element", () => {
+ // A narrow window: the panel covers the conversation.
+ const measure = vi
+ .spyOn(HTMLElement.prototype, "getBoundingClientRect")
+ .mockReturnValue({ left: 0, top: 0, width: 600, height: 800, right: 600, bottom: 800, x: 0, y: 0 });
+ const onAddElement = vi.fn();
+ const onClose = vi.fn();
+ const props = {
+ view: "browser",
+ workspaceRunId: "run-9",
+ browserRequest: { id: "b1", url: "http://localhost:5173/" },
+ onBrowserRequestHandled: vi.fn(),
+ onAddElement,
+ onClose,
+ };
+ try {
+ const { container } = render(panel(props));
+ expect(screen.getByRole("button", { name: "返回聊天" })).toBeInTheDocument();
+ const frame = container.querySelector("iframe.agent-browser-frame");
+ const doc = frame.contentDocument;
+ doc.open();
+ doc.write(`<!doctype html><html><body><button>提交</button></body></html>`);
+ doc.close();
+ act(() => {
+ fireEvent.load(frame);
+ });
+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));
+ act(() => {
+ doc.querySelector("button").dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true }));
+ });
+ fireEvent.click(screen.getByText("加入对话框"));
+ expect(onAddElement).toHaveBeenCalledWith(expect.objectContaining({ type: "element" }));
+ expect(onClose).toHaveBeenCalled();
+ } finally {
+ measure.mockRestore();
+ }
+ });
+
test("shows the workspace browser", async () => {
const onViewChange = vi.fn();
const { rerender } = render(panel({ view: "artifacts", workspaceRunId: "run-9", onViewChange }));
diff --git a/chatbot/tests/devserver-cookie.test.js b/chatbot/tests/devserver-cookie.test.js
new file mode 100644
index 00000000..27fa2118
--- /dev/null
+++ b/chatbot/tests/devserver-cookie.test.js
@@ -0,0 +1,65 @@
+// The dev server intercept script (internal/preview/devserver_intercept.js)
+// runs in the user's browser under /devpreview/{run}/{port}/. Its cookie
+// setter must keep a page's cookies inside that preview: same port in another
+// task, and the platform itself, never see them.
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { afterEach, expect, test } from "vitest";
+
+const source = readFileSync(resolve(__dirname, "../../internal/preview/devserver_intercept.js"), "utf8");
+const raw = Object.getOwnPropertyDescriptor(Document.prototype, "cookie");
+const browserCookies = () => raw.get.call(document);
+
+function openPreview(run, port) {
+ history.replaceState(null, "", `/devpreview/${run}/${port}/app/`);
+ delete window.__AGENTFLOW_DEV_SERVER__;
+ delete document.cookie;
+ const prefix = `/devpreview/${run}/${port}`;
+ // As preview.DevServerCookiePrefix fills it in.
+ const cookiePrefix = `af_devpreview_${run}_${port}_`;
+ (0, eval)(
+ source
+ .replace("__AF_PREFIX__", JSON.stringify(prefix))
+ .replace("__AF_PORT__", String(port))
+ .replace("__AF_COOKIE__", JSON.stringify(cookiePrefix)),
+ );
+}
+
+afterEach(() => {
+ history.replaceState(null, "", "/");
+});
+
+test("a page's root-path cookie stays inside its own preview", () => {
+ openPreview("run-a", 3000);
+ document.cookie = "sid=secret; Path=/; Domain=localhost";
+ document.cookie = "theme=dark";
+ expect(document.cookie).toContain("sid=secret");
+ expect(document.cookie).toContain("theme=dark");
+ expect(browserCookies()).toContain("af_devpreview_run-a_3000_sid=secret");
+
+ // The platform root does not carry it…
+ history.replaceState(null, "", "/chatbot/");
+ expect(browserCookies()).not.toContain("sid=secret");
+ // …and neither does another task serving the same port.
+ openPreview("run-b", 3000);
+ expect(browserCookies()).not.toContain("sid=secret");
+ expect(document.cookie).not.toContain("sid");
+ document.cookie = "sid=other; Path=/";
+ expect(document.cookie).toBe("sid=other");
+
+ // Back in the first preview its own cookie is intact.
+ openPreview("run-a", 3000);
+ expect(document.cookie).toContain("sid=secret");
+ expect(document.cookie).not.toContain("sid=other");
+});
+
+test("paths already inside the preview are kept", () => {
+ openPreview("run-c", 4000);
+ document.cookie = "a=1; Path=/devpreview/run-c/4000/app";
+ document.cookie = "b=2; path=/api";
+ history.replaceState(null, "", "/devpreview/run-c/4000/api/x");
+ expect(document.cookie).toContain("b=2");
+ expect(document.cookie).not.toContain("a=1");
+ history.replaceState(null, "", "/devpreview/run-c/4000/app/x");
+ expect(document.cookie).toContain("a=1");
+});
diff --git a/chatbot/tests/element-picker.test.jsx b/chatbot/tests/element-picker.test.jsx
new file mode 100644
index 00000000..23c05346
--- /dev/null
+++ b/chatbot/tests/element-picker.test.jsx
@@ -0,0 +1,246 @@
+import React from "react";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import AgentBrowser from "../src/AgentBrowser.jsx";
+import { componentOf, describeElement, elementLabel, elementSelector, elementTitle } from "../src/element-context.js";
+
+function page(html) {
+ document.body.innerHTML = html;
+ return document;
+}
+
+describe("element context", () => {
+ test("builds selectors that match only the element", () => {
+ const doc = page(`
+ <main id="app">
+ <ul class="list"><li class="item">a</li><li class="item">b</li></ul>
+ <button data-testid="save" class="css-1x2y3z primary">保存</button>
+ <section><p>one</p><p>two</p></section>
+ </main>`);
+ const second = doc.querySelectorAll("li")[1];
+ const save = doc.querySelector("button");
+ const two = doc.querySelectorAll("p")[1];
+ for (const el of [second, save, two, doc.getElementById("app")]) {
+ const selector = elementSelector(el);
+ expect(doc.querySelectorAll(selector)).toHaveLength(1);
+ expect(doc.querySelector(selector)).toBe(el);
+ }
+ expect(elementSelector(save)).toBe('button[data-testid="save"]');
+ expect(elementSelector(save)).not.toContain("css-1x2y3z");
+ expect(elementLabel(save)).toBe("button.primary 「保存」");
+ });
+
+ test("finds the component a dev build exposes", () => {
+ const doc = page(`<div id="root"><span>hi</span></div><b>vue</b><i>svelte</i>`);
+ function SaveButton() {}
+ doc.querySelector("span")["__reactFiber$abc"] = {
+ type: "span",
+ _debugSource: { fileName: "src/Save.tsx", lineNumber: 12, columnNumber: 3 },
+ return: { type: SaveButton, return: null },
+ };
+ expect(componentOf(doc.querySelector("span"))).toEqual({
+ framework: "React",
+ component: "SaveButton",
+ source: "src/Save.tsx:12:3",
+ });
+ doc.querySelector("b").__vueParentComponent = { type: { __name: "Toolbar", __file: "src/Toolbar.vue" } };
+ expect(componentOf(doc.querySelector("b"))).toEqual({ framework: "Vue", component: "Toolbar", source: "src/Toolbar.vue" });
+ doc.querySelector("i").__svelte_meta = { loc: { file: "src/Card.svelte", line: 4, column: 0 } };
+ expect(componentOf(doc.querySelector("i"))).toEqual({ framework: "Svelte", component: "Card", source: "src/Card.svelte:5:1" });
+ expect(componentOf(doc.getElementById("root"))).toBeNull();
+ });
+
+ test("describes the element for the Agent", () => {
+ const doc = page(`<button id="go" style="color: red">${"长".repeat(400)}</button>`);
+ const info = describeElement(doc.getElementById("go"), "http://localhost:5173/x");
+ expect(info).toMatchObject({ url: "http://localhost:5173/x", selector: "#go", tag: "button" });
+ expect(info.text.length).toBeLessThanOrEqual(201);
+ expect(info.styles.color).toBe("rgb(255, 0, 0)");
+ expect(elementTitle({ element: { ...info, component: "Go", source: "src/Go.tsx:1" } })).toBe(
+ "Go · src/Go.tsx:1\n#go\nhttp://localhost:5173/x",
+ );
+ });
+});
+
+describe("element picking in the native preview", () => {
+ test("picks an element in the frame and adds it to the composer", async () => {
+ const add = vi.fn();
+ const { container } = render(
+ <AgentBrowser runId="run-7" request={{ id: "p1", url: "http://localhost:5173/" }} onAddElement={add} />,
+ );
+ const frame = container.querySelector("iframe.agent-browser-frame");
+ const doc = frame.contentDocument;
+ doc.open();
+ doc.write(`<!doctype html><html><head></head><body><main><button class="primary">提交</button><a href="#x">link</a></main></body></html>`);
+ doc.close();
+ act(() => {
+ fireEvent.load(frame);
+ });
+ // The button sits at the end of the address bar.
+ expect(screen.getByRole("button", { name: "标记元素" }).closest(".agent-browser-address")).not.toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));
+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "true");
+ expect(doc.querySelector("style[data-af-pick]")).not.toBeNull();
+ const button = doc.querySelector("button");
+ const pageClick = vi.fn();
+ button.addEventListener("click", pageClick);
+ act(() => {
+ button.dispatchEvent(new MouseEvent("mousemove", { bubbles: true }));
+ });
+ expect(container.querySelector(".agent-browser-pick-outline")).not.toBeNull();
+ act(() => {
+ button.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true }));
+ });
+ // The page never sees clicks made while picking.
+ expect(pageClick).not.toHaveBeenCalled();
+ expect(doc.querySelector("style[data-af-pick]")).toBeNull();
+ expect(screen.getByRole("dialog", { name: "选中的元素" })).toHaveTextContent("button.primary 「提交」");
+ fireEvent.click(screen.getByText("加入对话框"));
+ expect(add).toHaveBeenCalledTimes(1);
+ expect(add.mock.calls[0][0]).toMatchObject({
+ type: "element",
+ name: "button.primary 「提交」",
+ element: { tag: "button", selector: "button.primary", text: "提交" },
+ });
+ expect(screen.queryByRole("dialog", { name: "选中的元素" })).toBeNull();
+
+ // Escape leaves picking mode.
+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));
+ act(() => {
+ doc.dispatchEvent(new KeyboardEvent("keydown", { key: "Escape", bubbles: true }));
+ });
+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "false");
+ });
+
+ test("offers picking only when the composer can take elements", () => {
+ render(<AgentBrowser runId="run-8" request={{ id: "p2", url: "http://localhost:5173/" }} />);
+ expect(screen.queryByRole("button", { name: "标记元素" })).toBeNull();
+ });
+});
+
+class PickSocket {
+ static OPEN = 1;
+ static last = null;
+ constructor() {
+ this.readyState = 0;
+ this.sent = [];
+ PickSocket.last = this;
+ }
+ send(data) {
+ this.sent.push(JSON.parse(data));
+ }
+ close() {
+ this.readyState = 3;
+ }
+ open() {
+ this.readyState = PickSocket.OPEN;
+ this.onopen?.();
+ }
+ state(state) {
+ this.onmessage?.({ data: JSON.stringify({ type: "state", state }) });
+ }
+}
+
+const rect = (left, top, width, height) => () => ({ left, top, width, height, right: left + width, bottom: top + height, x: left, y: top });
+
+describe("element picking on the Agent browser screencast", () => {
+ afterEach(() => vi.unstubAllGlobals());
+
+ test("inspects the page over CDP and adds the picked element", async () => {
+ vi.stubGlobal("WebSocket", PickSocket);
+ // jsdom has no PointerEvent; without it pointer coordinates are dropped.
+ if (!window.PointerEvent) vi.stubGlobal("PointerEvent", class extends MouseEvent {});
+ const inspections = [];
+ const save = { label: "button#save 「保存」", rect: { x: 100, y: 50, width: 200, height: 40 }, viewport: { width: 1280, height: 800 } };
+ const fetchMock = vi.fn(async (_path, init) => {
+ const body = JSON.parse(init.body);
+ if (body.operation === "inspect") inspections.push(body);
+ const inspect = body.operation === "inspect" ? { ...save, ...(body.detail ? { element: { tag: "button", selector: "#save", url: "http://localhost:3000/" } } : {}) } : undefined;
+ return { ok: true, status: 200, json: async () => ({ available: true, inspect }) };
+ });
+ vi.stubGlobal("fetch", fetchMock);
+ const add = vi.fn();
+ const { container } = render(<AgentBrowser runId="run-9" onAddElement={add} />);
+ const socket = PickSocket.last;
+ act(() => {
+ socket.open();
+ socket.state({ available: true, url: "http://localhost:3000/", width: 1280, height: 800, tabs: [] });
+ });
+ const stage = container.querySelector(".agent-browser-stage");
+ const img = container.querySelector(".agent-browser-screen");
+ // The panel shows the 1280px page at half size, 10px below the stage top.
+ stage.getBoundingClientRect = rect(0, 0, 640, 500);
+ img.getBoundingClientRect = rect(0, 10, 640, 400);
+
+ expect(screen.getByRole("button", { name: "标记元素" }).closest(".agent-browser-address")).not.toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));
+ const capture = screen.getByTestId("agent-browser-pick-capture");
+ const before = socket.sent.length;
+ fireEvent.pointerMove(capture, { clientX: 160, clientY: 110 });
+ await waitFor(() => expect(container.querySelector(".agent-browser-pick-outline")).not.toBeNull());
+ expect(inspections[0]).toEqual({ operation: "inspect", point: { x: 0.25, y: 0.25 }, detail: false });
+ const outline = container.querySelector(".agent-browser-pick-outline");
+ expect(outline.style.left).toBe("50px");
+ expect(outline.style.top).toBe("35px");
+ expect(outline.style.width).toBe("100px");
+ expect(outline.querySelector(".agent-browser-pick-label")).toHaveTextContent("button#save 「保存」");
+ expect(outline.querySelector(".agent-browser-pick-size")).toHaveTextContent("100×20");
+
+ fireEvent.click(capture, { clientX: 160, clientY: 110 });
+ await waitFor(() => expect(screen.getByRole("button", { name: /加入对话框/ })).toBeEnabled());
+ expect(inspections.at(-1)).toMatchObject({ detail: true });
+ // Hovering and picking never reach the page as input.
+ expect(socket.sent.slice(before).filter((m) => m.operation === "input")).toHaveLength(0);
+ fireEvent.click(screen.getByRole("button", { name: /加入对话框/ }));
+ expect(add).toHaveBeenCalledWith(
+ expect.objectContaining({ type: "element", name: "button#save 「保存」", element: expect.objectContaining({ selector: "#save" }) }),
+ );
+ expect(screen.queryByRole("dialog", { name: "选中的元素" })).toBeNull();
+
+ // Escape leaves picking before the screencast forwards it to the page.
+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));
+ const sent = socket.sent.length;
+ fireEvent.keyDown(container.querySelector(".agent-browser-ime"), { key: "Escape", code: "Escape", keyCode: 27 });
+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "false");
+ expect(socket.sent).toHaveLength(sent);
+ });
+
+ test("the runtime inspect script describes the element at a point", () => {
+ const go = readFileSync(resolve(__dirname, "../../internal/runtime/browser_inspect.go"), "utf8");
+ const script = go.match(/const inspectScript = `([\s\S]*?)`/)[1];
+ document.body.innerHTML = `<main id="app"><button class="primary" data-testid="save" data-agentflow-ref="e14">保存<i data-agentflow-ref="e15"></i></button></main>`;
+ const button = document.querySelector("button");
+ button.getBoundingClientRect = rect(10, 20, 80, 30);
+ button["__reactFiber$x"] = { type: "button", _debugSource: { fileName: "src/Save.tsx", lineNumber: 4 }, return: { type: function SaveButton() {} } };
+ const hits = [];
+ document.elementFromPoint = (x, y) => {
+ hits.push([x, y]);
+ return button;
+ };
+ try {
+ const out = (0, eval)(script)(0.5, 0.25, true);
+ expect(hits[0]).toEqual([innerWidth * 0.5, innerHeight * 0.25]);
+ expect(out.label).toBe("button.primary 「保存」");
+ expect(out.rect).toEqual({ x: 10, y: 20, width: 80, height: 30 });
+ // Same description as the native preview builds in the panel.
+ const native = describeElement(button, location.href);
+ expect(out.element).toMatchObject({
+ selector: native.selector,
+ tag: "button",
+ text: "保存",
+ component: "SaveButton",
+ framework: "React",
+ source: "src/Save.tsx:4",
+ });
+ expect(out.element.selector).toBe('button[data-testid="save"]');
+ // The Agent's snapshot refs are dropped on both sides.
+ expect(out.element.html).toBe('<button class="primary" data-testid="save">保存<i></i></button>');
+ expect(native.html).toBe(out.element.html);
+ expect((0, eval)(script)(0.5, 0.25, false).element).toBeUndefined();
+ } finally {
+ delete document.elementFromPoint;
+ }
+ });
+});
diff --git a/internal/domain/assistant.go b/internal/domain/assistant.go
index 0756aebd..4bbaf7ba 100644
--- a/internal/domain/assistant.go
+++ b/internal/domain/assistant.go
@@ -76,6 +76,34 @@ type AssistantReference struct {
Name string `json:"name,omitempty"`
ProjectID string `json:"projectId,omitempty"`
ProjectName string `json:"projectName,omitempty"`
+ // Element describes a page element the user picked in the native
+ // preview (Type "element"); it is the user's own context, not an object
+ // the platform looks up.
+ Element *AssistantPageElement `json:"element,omitempty"`
+}
+
+// AssistantPageElement is a page element picked in the native preview.
+type AssistantPageElement struct {
+ URL string `json:"url,omitempty"`
+ Title string `json:"title,omitempty"`
+ Selector string `json:"selector,omitempty"`
+ Tag string `json:"tag,omitempty"`
+ Text string `json:"text,omitempty"`
+ HTML string `json:"html,omitempty"`
+ Rect *ElementRect `json:"rect,omitempty"`
+ Viewport *ElementRect `json:"viewport,omitempty"`
+ Styles map[string]string `json:"styles,omitempty"`
+ Component string `json:"component,omitempty"`
+ Framework string `json:"framework,omitempty"`
+ Source string `json:"source,omitempty"`
+}
+
+// ElementRect is a box in CSS pixels; a viewport only sets its size.
+type ElementRect struct {
+ X float64 `json:"x,omitempty"`
+ Y float64 `json:"y,omitempty"`
+ Width float64 `json:"width"`
+ Height float64 `json:"height"`
}
type AssistantAttachment struct {
diff --git a/internal/domain/runner_workspace.go b/internal/domain/runner_workspace.go
index 49651103..3fd99608 100644
--- a/internal/domain/runner_workspace.go
+++ b/internal/domain/runner_workspace.go
@@ -141,6 +141,50 @@ type BrowserRequest struct {
WithSnapshot bool `json:"withSnapshot,omitempty"`
// Ensure starts the run browser when none is running yet.
Ensure bool `json:"ensure,omitempty"`
+ // HTTP carries one request to a dev server port inside the run sandbox
+ // (operation "http"), so the user's own browser renders the page natively.
+ HTTP *BrowserHTTPRequest `json:"http,omitempty"`
+ // File is the absolute path of a workspace file as the run browser sees
+ // it (operation "file"), served so the user's browser renders file://
+ // pages natively. The response comes back in BrowserState.HTTP.
+ File string `json:"file,omitempty"`
+ // Point is where the panel inspects the page (operation "inspect"), as a
+ // fraction of the viewport like input events; Detail asks for the full
+ // element description rather than just its box and label.
+ Point *BrowserPoint `json:"point,omitempty"`
+ Detail bool `json:"detail,omitempty"`
+}
+
+// BrowserPoint is a position as a fraction of the viewport (0..1).
+type BrowserPoint struct {
+ X float64 `json:"x"`
+ Y float64 `json:"y"`
+}
+
+// BrowserInspection is the element under a point of the Agent browser page.
+// Rect is in CSS pixels of the top-level viewport.
+type BrowserInspection struct {
+ Label string `json:"label"`
+ Rect ElementRect `json:"rect"`
+ Viewport ElementRect `json:"viewport"`
+ Element *AssistantPageElement `json:"element,omitempty"`
+}
+
+// BrowserHTTPRequest is one proxied request to http://localhost:Port in the
+// run's network namespace. Path keeps the query string.
+type BrowserHTTPRequest struct {
+ Port int `json:"port"`
+ Method string `json:"method"`
+ Path string `json:"path"`
+ Header map[string][]string `json:"header,omitempty"`
+ Body []byte `json:"body,omitempty"`
+}
+
+// BrowserHTTPResponse is the dev server's answer to a BrowserHTTPRequest.
+type BrowserHTTPResponse struct {
+ Status int `json:"status"`
+ Header map[string][]string `json:"header,omitempty"`
+ Body []byte `json:"body,omitempty"`
}
// BrowserInputEvent coordinates are normalized to the frame (0..1).
@@ -169,19 +213,29 @@ type BrowserTab struct {
}
type BrowserState struct {
- Available bool `json:"available"`
- Reason string `json:"reason,omitempty"`
- Kind string `json:"kind,omitempty"`
- Seq int64 `json:"seq"`
- Frame string `json:"frame,omitempty"`
- Width float64 `json:"width,omitempty"`
- Height float64 `json:"height,omitempty"`
- URL string `json:"url,omitempty"`
- Title string `json:"title,omitempty"`
- CanGoBack bool `json:"canGoBack,omitempty"`
- CanGoForward bool `json:"canGoForward,omitempty"`
- Tabs []BrowserTab `json:"tabs,omitempty"`
- Snapshot *BrowserSnapshot `json:"snapshot,omitempty"`
+ Available bool `json:"available"`
+ Reason string `json:"reason,omitempty"`
+ Kind string `json:"kind,omitempty"`
+ Seq int64 `json:"seq"`
+ Frame string `json:"frame,omitempty"`
+ Width float64 `json:"width,omitempty"`
+ Height float64 `json:"height,omitempty"`
+ URL string `json:"url,omitempty"`
+ Title string `json:"title,omitempty"`
+ // ScrollX/Y and ContentWidth/Height (CSS px) bound how far the panel may
+ // slide a frame ahead of the page when it predicts a scroll.
+ ScrollX float64 `json:"scrollX,omitempty"`
+ ScrollY float64 `json:"scrollY,omitempty"`
+ ContentWidth float64 `json:"contentWidth,omitempty"`
+ ContentHeight float64 `json:"contentHeight,omitempty"`
+ CanGoBack bool `json:"canGoBack,omitempty"`
+ CanGoForward bool `json:"canGoForward,omitempty"`
+ Tabs []BrowserTab `json:"tabs,omitempty"`
+ Snapshot *BrowserSnapshot `json:"snapshot,omitempty"`
+ // HTTP answers operation "http"; Reason explains a missing answer.
+ HTTP *BrowserHTTPResponse `json:"http,omitempty"`
+ // Inspect answers operation "inspect"; nil when nothing is at the point.
+ Inspect *BrowserInspection `json:"inspect,omitempty"`
}
// BrowserSnapshot is a text view of the active page for language models.
diff --git a/internal/httpapi/assistant_browser.go b/internal/httpapi/assistant_browser.go
index 035d7180..6614454f 100644
--- a/internal/httpapi/assistant_browser.go
+++ b/internal/httpapi/assistant_browser.go
@@ -45,7 +45,7 @@ func mainBotBrowserTools(s *AgentToolSession) []agentToolDescriptor {
{"click", "用真实鼠标点击快照中的元素。返回点击后的新快照(ref 会刷新)。", `{"type":"object","properties":{"ref":{"type":"string"}},"required":["ref"],"additionalProperties":false}`},
{"type", "清空并在快照中的输入元素里输入文本;submit=true 时随后按回车。返回新快照。", `{"type":"object","properties":{"ref":{"type":"string"},"text":{"type":"string"},"submit":{"type":"boolean"}},"required":["ref","text"],"additionalProperties":false}`},
{"press", "在当前焦点按键:Enter、Tab、Escape、Backspace、Delete、Space、ArrowUp/Down/Left/Right、PageUp、PageDown、Home、End。返回新快照。", `{"type":"object","properties":{"key":{"type":"string"}},"required":["key"],"additionalProperties":false}`},
- {"screenshot", "截取当前标签页画面(图像),用于确认布局、样式或快照无法表达的内容。截图会自动附在本轮回复里给用户看(显示在正文下方,每轮保留最近 4 张)。用户还能在对话侧栏「浏览器」页签实时查看和操作这个浏览器:回复里写出页面地址(file:///workspace/… 或 http://localhost:端口/…),用户点击即在该页签打开。", `{"type":"object","properties":{},"additionalProperties":false}`},
+ {"screenshot", "截取当前标签页画面(图像),用于确认布局、样式或快照无法表达的内容。截图会自动附在本轮回复里给用户看(显示在正文下方,每轮保留最近 4 张)。用户还能在对话侧栏「浏览器」页签实时查看和操作这个浏览器:回复里写出页面地址(file:///workspace/… 或 http://localhost:端口/…),用户点击即在该页签打开;工作区文件和 http://localhost:端口 的开发服务器都由平台转发、在用户自己的浏览器里原生渲染(可在新窗口打开;开发服务器暂不支持热更新,改动后让用户刷新)。", `{"type":"object","properties":{},"additionalProperties":false}`},
{"tabs", "管理标签页:list 列出,switch 切换到 tabId,new 新建,close 关闭 tabId(缺省当前页)。", `{"type":"object","properties":{"action":{"type":"string","enum":["list","switch","new","close"]},"tabId":{"type":"string"}},"required":["action"],"additionalProperties":false}`},
}
out := make([]agentToolDescriptor, 0, len(descriptors))
diff --git a/internal/httpapi/assistant_page_element.go b/internal/httpapi/assistant_page_element.go
new file mode 100644
index 00000000..1f2be568
--- /dev/null
+++ b/internal/httpapi/assistant_page_element.go
@@ -0,0 +1,66 @@
+package httpapi
+
+import (
+ "encoding/json"
+ "fmt"
+ "regexp"
+ "sort"
+ "strings"
+
+ "agentflow/internal/domain"
+)
+
+// Page elements the user picks in the native preview ride along as
+// references of type "element". Unlike other references they are not looked
+// up: the browser describes the element, so the description is bounded and
+// framed as the user's context here.
+
+const pageElementReference = "element"
+
+var pageElementIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)
+
+// pageElementStyleLimit bounds the computed styles kept per element.
+const pageElementStyleLimit = 32
+
+func pageElementContext(ref domain.AssistantReference) (domain.AssistantReference, string, error) {
+ if !pageElementIDPattern.MatchString(ref.ID) || ref.Element == nil {
+ return ref, "", fmt.Errorf("页面元素引用无效")
+ }
+ in := ref.Element
+ e := &domain.AssistantPageElement{
+ URL: truncateRunes(strings.TrimSpace(in.URL), 2000),
+ Title: truncateRunes(in.Title, 200),
+ Selector: truncateRunes(in.Selector, 1000),
+ Tag: truncateRunes(in.Tag, 40),
+ Text: truncateRunes(in.Text, 500),
+ HTML: truncateRunes(in.HTML, 4000),
+ Rect: in.Rect,
+ Viewport: in.Viewport,
+ Component: truncateRunes(in.Component, 200),
+ Framework: truncateRunes(in.Framework, 40),
+ Source: truncateRunes(in.Source, 500),
+ }
+ if len(in.Styles) > 0 {
+ keys := make([]string, 0, len(in.Styles))
+ for k := range in.Styles {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ e.Styles = map[string]string{}
+ for _, k := range keys {
+ if len(e.Styles) == pageElementStyleLimit {
+ break
+ }
+ e.Styles[truncateRunes(k, 60)] = truncateRunes(in.Styles[k], 200)
+ }
+ }
+ name := strings.TrimSpace(truncateRunes(ref.Name, 80))
+ if name == "" {
+ name = e.Tag
+ }
+ canonical := domain.AssistantReference{Type: pageElementReference, ID: ref.ID, Name: name, Element: e}
+ raw, _ := json.Marshal(e)
+ text := "用户在对话侧栏的原生预览里选中的页面元素「" + name + "」,说明这一轮要改的位置。" +
+ "有 source/component 时先按它找代码;需要核对时用 workbench_browser 打开 url 并按 selector 查看。元素信息:" + string(raw)
+ return canonical, text, nil
+}
diff --git a/internal/httpapi/assistant_page_element_test.go b/internal/httpapi/assistant_page_element_test.go
new file mode 100644
index 00000000..68253f96
--- /dev/null
+++ b/internal/httpapi/assistant_page_element_test.go
@@ -0,0 +1,53 @@
+package httpapi
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+
+ "agentflow/internal/domain"
+)
+
+func TestPickedPageElementsBecomeTurnContext(t *testing.T) {
+ h, u := workbenchFixture(t)
+ h.repo.PutRun(domain.Run{ID: "task", Status: "waiting", ExecutionContext: domain.TelemetryContext{ProjectID: "workbench"}, Nodes: map[string]*domain.NodeRun{}})
+ element := func() *domain.AssistantPageElement {
+ return &domain.AssistantPageElement{
+ URL: "http://localhost:5173/settings",
+ Selector: "#save",
+ Tag: "button",
+ Text: "保存",
+ HTML: strings.Repeat("<b>x</b>", 2000),
+ Component: "SaveButton",
+ Source: "src/Settings.tsx:42",
+ Styles: map[string]string{"color": "rgb(0, 0, 0)"},
+ }
+ }
+ c, _ := workbenchContext(u, "POST", "/", bytes.NewBuffer(nil))
+ // Task conversations accept them too: they are not task objects.
+ req := assistantRequest{ProjectID: "workbench", RunID: "task", Messages: []assistantMessage{{Role: "user", Content: "改成次要按钮", References: []domain.AssistantReference{
+ {Type: "element", ID: "el-1", Name: "button#save 「保存」", ProjectID: "forged", Element: element()},
+ }}}}
+ if err := h.prepareWorkbenchContext(c, &req, u); err != nil {
+ t.Fatal(err)
+ }
+ ref := req.Messages[0].References[0]
+ if ref.ProjectID != "" || ref.Element == nil || len([]rune(ref.Element.HTML)) > 4001 {
+ t.Fatalf("canonical %+v", ref)
+ }
+ for _, want := range []string{"原生预览", "button#save 「保存」", "src/Settings.tsx:42", "SaveButton", "http://localhost:5173/settings", `"selector":"#save"`} {
+ if !strings.Contains(req.contextText, want) {
+ t.Fatalf("context misses %q: %s", want, req.contextText)
+ }
+ }
+
+ for _, bad := range []domain.AssistantReference{
+ {Type: "element", ID: "el-2"},
+ {Type: "element", ID: "../x", Element: element()},
+ } {
+ req := assistantRequest{ProjectID: "workbench", Messages: []assistantMessage{{Role: "user", Content: "x", References: []domain.AssistantReference{bad}}}}
+ if err := h.prepareWorkbenchContext(c, &req, u); err == nil {
+ t.Fatalf("accepted %+v", bad)
+ }
+ }
+}
diff --git a/internal/httpapi/assistant_workbench.go b/internal/httpapi/assistant_workbench.go
index c2197dfd..bf109443 100644
--- a/internal/httpapi/assistant_workbench.go
+++ b/internal/httpapi/assistant_workbench.go
@@ -278,6 +278,15 @@ func (h *assistantHandlers) prepareWorkbenchContext(c *gin.Context, req *assista
}
continue
}
+ if ref.Type == pageElementReference {
+ canonical, text, err := pageElementContext(ref)
+ if err != nil {
+ return err
+ }
+ m.References[j] = canonical
+ context.WriteString("引用(数据):" + text + "\n")
+ continue
+ }
if req.RunID != "" {
r, ok := h.repo.RunState(req.RunID)
if !ok || !h.taskReferenceAllowed(r, ref.Type, ref.ID) {
diff --git a/internal/httpapi/browser.go b/internal/httpapi/browser.go
index a574c934..b9c5a8d7 100644
--- a/internal/httpapi/browser.go
+++ b/internal/httpapi/browser.go
@@ -66,6 +66,10 @@ func (h *browserHandlers) browser(c *gin.Context) {
// executor is reported as an unavailable state rather than an error.
func (h *browserHandlers) dispatch(ctx context.Context, run domain.Run, req domain.BrowserRequest) (json.RawMessage, string) {
timeout := 15 * time.Second
+ if req.Operation == "http" || req.Operation == "file" {
+ // A dev server compiles a route on its first request.
+ timeout = devServerTimeout
+ }
if req.Ensure {
// The first browser of a host may still need its sidecar image built.
timeout = 4 * time.Minute
diff --git a/internal/httpapi/browser_devserver.go b/internal/httpapi/browser_devserver.go
new file mode 100644
index 00000000..2d50310f
--- /dev/null
+++ b/internal/httpapi/browser_devserver.go
@@ -0,0 +1,501 @@
+package httpapi
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "html"
+ "io"
+ "net/http"
+ "net/url"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "agentflow/internal/domain"
+ "agentflow/internal/preview"
+
+ "github.com/gin-gonic/gin"
+ "github.com/gorilla/websocket"
+)
+
+// Dev server preview: the user's own browser renders a dev server running in
+// the run sandbox, under /devpreview/{runID}/{port}/ on the platform origin,
+// so the page is native (input, scrolling, devtools) instead of a screencast.
+// Like build previews it lives under a path prefix: markup, module imports and
+// stylesheets are rewritten under it, the HTML gets the history shim and an
+// intercept script, and requests a page still sends to root paths are
+// redirected back under the prefix by Referer. Each request travels the
+// browser operation channel ("http") to the executor hosting the sandbox.
+// Workspace files the Agent opens as file:// are served the same way under
+// /devpreview/{runID}/file/ (operation "file"), as they are, unrewritten.
+
+const (
+ devServerPathPrefix = "/devpreview/"
+ devServerTimeout = 90 * time.Second
+ devServerRequestLimit = 8 << 20
+)
+
+var devServerRunIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
+
+func devServerPrefix(runID string, port int) string {
+ return devServerPathPrefix + runID + "/" + strconv.Itoa(port)
+}
+
+// parseDevServerPath splits /devpreview/{runID}/{port}/{rest}. rest keeps its
+// escaping; slash reports whether anything, even "/", follows the port.
+func parseDevServerPath(escaped string) (runID string, port int, rest string, slash, ok bool) {
+ tail, found := strings.CutPrefix(escaped, devServerPathPrefix)
+ if !found {
+ return "", 0, "", false, false
+ }
+ runID, tail, found = strings.Cut(tail, "/")
+ if !found || !devServerRunIDPattern.MatchString(runID) {
+ return "", 0, "", false, false
+ }
+ rawPort, rest, slash := strings.Cut(tail, "/")
+ port, err := strconv.Atoi(rawPort)
+ if err != nil || port <= 0 || port > 65535 || strconv.Itoa(port) != rawPort {
+ return "", 0, "", false, false
+ }
+ return runID, port, rest, slash, true
+}
+
+// devServerHost runs before routing, like the build preview host: it serves
+// /devpreview/ and sends a dev page's root-path requests back under its prefix.
+func (h *browserHandlers) devServerHost() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ r := c.Request
+ if runID, file, ok := parseDevFilePath(r.URL.Path); ok {
+ h.serveDevFile(c, runID, file)
+ c.Abort()
+ return
+ }
+ runID, port, rest, slash, ok := parseDevServerPath(r.URL.EscapedPath())
+ if !ok {
+ if target, stray := devServerStrayTarget(r); stray {
+ c.Redirect(http.StatusTemporaryRedirect, target)
+ c.Abort()
+ return
+ }
+ c.Next()
+ return
+ }
+ if !slash {
+ target := devServerPrefix(runID, port) + "/"
+ if r.URL.RawQuery != "" {
+ target += "?" + r.URL.RawQuery
+ }
+ c.Redirect(http.StatusFound, target)
+ c.Abort()
+ return
+ }
+ h.proxyDevServer(c, runID, port, "/"+rest)
+ c.Abort()
+ }
+}
+
+// devServerStrayTarget maps a request a dev page sent to a root path (module
+// imports such as /src/App.tsx, assets, links, forms) under the page's
+// prefix. Dev pages are served with Referrer-Policy: same-origin, so their
+// requests name the page they come from.
+func devServerStrayTarget(r *http.Request) (string, bool) {
+ if strings.HasPrefix(r.URL.Path, devServerPathPrefix) || websocket.IsWebSocketUpgrade(r) {
+ return "", false
+ }
+ ref, err := url.Parse(r.Referer())
+ if err != nil || ref.Host == "" {
+ return "", false
+ }
+ runID, port, _, _, ok := parseDevServerPath(ref.EscapedPath())
+ if !ok {
+ return "", false
+ }
+ switch r.Header.Get("Sec-Fetch-Site") {
+ case "same-origin":
+ case "":
+ // Plain-HTTP deployments get no fetch metadata; compare hosts.
+ if !strings.EqualFold(ref.Host, r.Host) {
+ return "", false
+ }
+ default:
+ return "", false
+ }
+ target := devServerPrefix(runID, port) + r.URL.EscapedPath()
+ if r.URL.RawQuery != "" {
+ target += "?" + r.URL.RawQuery
+ }
+ return target, true
+}
+
+func (h *browserHandlers) proxyDevServer(c *gin.Context, runID string, port int, path string) {
+ r := c.Request
+ if websocket.IsWebSocketUpgrade(r) {
+ devServerFail(c, http.StatusNotImplemented, "开发服务器的实时连接(如热更新)暂不支持,修改代码后请刷新页面", false)
+ return
+ }
+ run, ok := h.devServerRun(c, runID)
+ if !ok {
+ return
+ }
+ body, err := io.ReadAll(http.MaxBytesReader(c.Writer, r.Body, devServerRequestLimit))
+ if err != nil {
+ devServerFail(c, http.StatusRequestEntityTooLarge, "请求体过大", false)
+ return
+ }
+ prefix := devServerPrefix(runID, port)
+ target := path
+ if r.URL.RawQuery != "" {
+ target += "?" + r.URL.RawQuery
+ }
+ req := domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{
+ Port: port, Method: r.Method, Path: target, Header: devServerRequestHeader(r, prefix, port), Body: body,
+ }}
+ res, ok := h.devServerFetch(c, run, req, "开发服务器暂不可用")
+ if !ok {
+ return
+ }
+ writeDevServerResponse(c, res, prefix, port)
+}
+
+// devServerRun resolves the run of a preview request for a user who may
+// drive its workspace; otherwise it answers the request itself.
+func (h *browserHandlers) devServerRun(c *gin.Context, runID string) (domain.Run, bool) {
+ r := c.Request
+ run, ok := h.repo.RunState(runID)
+ if !ok {
+ devServerFail(c, http.StatusNotFound, "运行不存在", false)
+ return domain.Run{}, false
+ }
+ user, ok := h.requestUser(r, false)
+ if !ok {
+ devServerFail(c, http.StatusUnauthorized, "请先登录平台", false)
+ return domain.Run{}, false
+ }
+ if !h.canControl(r, run, user.ID) {
+ devServerFail(c, http.StatusForbidden, "无权访问该任务的开发服务器", false)
+ return domain.Run{}, false
+ }
+ h.touchWorkspace(run.ID)
+ return run, true
+}
+
+// devServerFetch runs a preview operation on the executor, starting the run
+// browser when it is not up: dev server ports live in its network namespace
+// and file:// paths are the ones it sees. Failures are answered here.
+func (h *browserHandlers) devServerFetch(c *gin.Context, run domain.Run, req domain.BrowserRequest, unavailable string) (*domain.BrowserHTTPResponse, bool) {
+ r := c.Request
+ state, opErr := h.devServerCall(r.Context(), run, req)
+ if state.HTTP == nil && !state.Available && opErr == "" {
+ req.Ensure = true
+ state, opErr = h.devServerCall(r.Context(), run, req)
+ }
+ if opErr != "" {
+ if strings.Contains(opErr, "不支持的浏览器操作") {
+ opErr = "执行机版本过旧,暂不支持原生预览,请升级执行机"
+ }
+ devServerFail(c, http.StatusBadGateway, opErr, false)
+ return nil, false
+ }
+ if state.HTTP == nil {
+ devServerFail(c, http.StatusBadGateway, firstNonEmpty(state.Reason, unavailable), devServerNavigation(r))
+ return nil, false
+ }
+ return state.HTTP, true
+}
+
+func (h *browserHandlers) devServerCall(ctx context.Context, run domain.Run, req domain.BrowserRequest) (domain.BrowserState, string) {
+ payload, opErr := h.dispatch(ctx, run, req)
+ var state domain.BrowserState
+ _ = json.Unmarshal(payload, &state)
+ return state, opErr
+}
+
+func devServerNavigation(r *http.Request) bool {
+ if mode := r.Header.Get("Sec-Fetch-Mode"); mode != "" {
+ return mode == "navigate"
+ }
+ return r.Method == http.MethodGet && strings.Contains(r.Header.Get("Accept"), "text/html")
+}
+
+// Request headers that carry the platform's identity or the proxy chain stay
+// on this side; the sandbox only sees what a browser talking to the dev
+// server directly would send.
+var devServerRequestHeaders = map[string]bool{
+ "Accept": true, "Accept-Language": true, "Cache-Control": true, "Pragma": true,
+ "Content-Type": true, "If-None-Match": true, "If-Modified-Since": true, "If-Match": true,
+ "If-Unmodified-Since": true, "Range": true, "If-Range": true, "User-Agent": true,
+ "Sec-Fetch-Dest": true, "Sec-Fetch-Mode": true, "Sec-Fetch-Site": true, "Sec-Fetch-User": true,
+ "Sec-Ch-Ua": true, "Sec-Ch-Ua-Mobile": true, "Sec-Ch-Ua-Platform": true,
+ "Upgrade-Insecure-Requests": true, "Purpose": true, "Sec-Purpose": true,
+}
+
+var devServerBlockedXHeaders = []string{"X-Forwarded-", "X-Real-", "X-Client-", "X-Rio-", "X-Tai-", "X-Ext-", "X-Original-"}
+
+func devServerRequestHeader(r *http.Request, prefix string, port int) map[string][]string {
+ out := map[string][]string{}
+ for name, values := range r.Header {
+ key := http.CanonicalHeaderKey(name)
+ switch {
+ case devServerRequestHeaders[key]:
+ case key == "Authorization":
+ // Browsers replay cached Basic credentials of the platform itself.
+ if v := r.Header.Get(key); v == "" || strings.HasPrefix(strings.ToLower(v), "basic ") {
+ continue
+ }
+ case strings.HasPrefix(key, "X-") && !devServerBlockedXHeader(key):
+ default:
+ continue
+ }
+ out[key] = values
+ }
+ local := "http://localhost:" + strconv.Itoa(port)
+ if r.Header.Get("Origin") != "" {
+ out["Origin"] = []string{local}
+ }
+ if ref, err := url.Parse(r.Referer()); err == nil {
+ if refRun, refPort, rest, _, ok := parseDevServerPath(ref.EscapedPath()); ok && devServerPrefix(refRun, refPort) == prefix {
+ referer := local + "/" + rest
+ if ref.RawQuery != "" {
+ referer += "?" + ref.RawQuery
+ }
+ out["Referer"] = []string{referer}
+ }
+ }
+ if tags := devServerUpstreamETags(r.Header.Values("If-None-Match")); tags != "" {
+ out["If-None-Match"] = []string{tags}
+ } else {
+ delete(out, "If-None-Match")
+ }
+ cookiePrefix := preview.DevServerCookiePrefix(prefix)
+ var cookies []string
+ for _, cookie := range r.Cookies() {
+ if name, ok := strings.CutPrefix(cookie.Name, cookiePrefix); ok && name != "" {
+ cookies = append(cookies, name+"="+cookie.Value)
+ }
+ }
+ if len(cookies) > 0 {
+ out["Cookie"] = []string{strings.Join(cookies, "; ")}
+ }
+ return out
+}
+
+// devServerRewriteVersion tags the ETags of rewritten responses: a cached
+// copy rewritten by an older gateway must not be revalidated as current.
+const devServerRewriteVersion = "-af1"
+
+func devServerETag(etag string) string {
+ if strings.HasSuffix(etag, `"`) {
+ return strings.TrimSuffix(etag, `"`) + devServerRewriteVersion + `"`
+ }
+ return etag + devServerRewriteVersion
+}
+
+// devServerUpstreamETags keeps the If-None-Match tags this gateway version
+// issued, restored to the dev server's own form.
+func devServerUpstreamETags(values []string) string {
+ var tags []string
+ for _, value := range values {
+ for _, tag := range strings.Split(value, ",") {
+ tag = strings.TrimSpace(tag)
+ quoted := strings.HasSuffix(tag, `"`)
+ bare := strings.TrimSuffix(tag, `"`)
+ if !strings.HasSuffix(bare, devServerRewriteVersion) {
+ continue
+ }
+ bare = strings.TrimSuffix(bare, devServerRewriteVersion)
+ if quoted {
+ bare += `"`
+ }
+ tags = append(tags, bare)
+ }
+ }
+ return strings.Join(tags, ", ")
+}
+
+func devServerBlockedXHeader(key string) bool {
+ for _, prefix := range devServerBlockedXHeaders {
+ if strings.HasPrefix(key, prefix) {
+ return true
+ }
+ }
+ return false
+}
+
+// Response headers that would reach beyond the preview: framing and CSP that
+// block the side panel, HSTS and Clear-Site-Data for the platform origin, and
+// Service-Worker-Allowed, which would let a worker control the platform.
+var devServerDroppedResponseHeaders = map[string]bool{
+ "Content-Length": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true,
+ "X-Frame-Options": true, "Strict-Transport-Security": true, "Clear-Site-Data": true,
+ "Service-Worker-Allowed": true, "Referrer-Policy": true, "Set-Cookie": true, "Location": true,
+ "Alt-Svc": true, "Access-Control-Allow-Origin": true, "Access-Control-Allow-Credentials": true,
+}
+
+func writeDevServerResponse(c *gin.Context, res *domain.BrowserHTTPResponse, prefix string, port int) {
+ header := c.Writer.Header()
+ for name, values := range res.Header {
+ key := http.CanonicalHeaderKey(name)
+ if devServerDroppedResponseHeaders[key] {
+ continue
+ }
+ for _, v := range values {
+ header.Add(key, v)
+ }
+ }
+ for name, values := range res.Header {
+ switch http.CanonicalHeaderKey(name) {
+ case "Location":
+ if len(values) > 0 {
+ header.Set("Location", preview.RewriteDevServerLocation(values[0], prefix, port))
+ }
+ case "Set-Cookie":
+ for _, v := range values {
+ header.Add("Set-Cookie", rewriteDevServerCookie(v, prefix))
+ }
+ }
+ }
+ if etag := header.Get("Etag"); etag != "" {
+ header.Set("Etag", devServerETag(etag))
+ }
+ // Root-path requests find their way back by Referer.
+ header.Set("Referrer-Policy", "same-origin")
+ header.Set("X-Content-Type-Options", "nosniff")
+ body := res.Body
+ if header.Get("Content-Encoding") == "" {
+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(header.Get("Content-Type"), ";")[0]))
+ switch mediaType {
+ case "text/html", "application/xhtml+xml":
+ body = preview.RewriteDevServerHTML(body, prefix, port)
+ case "text/javascript", "application/javascript", "application/x-javascript", "text/ecmascript", "application/ecmascript":
+ body = preview.RewriteDevServerScript(body, prefix)
+ case "text/css":
+ body = preview.RewriteDevServerCSS(body, prefix)
+ }
+ }
+ header.Set("Content-Length", strconv.Itoa(len(body)))
+ status := res.Status
+ if status < 100 || status > 999 {
+ status = http.StatusBadGateway
+ }
+ c.Status(status)
+ if c.Request.Method != http.MethodHead && status != http.StatusNoContent && status != http.StatusNotModified {
+ _, _ = c.Writer.Write(body)
+ }
+}
+
+// rewriteDevServerCookie namespaces a dev server cookie by run and port and keeps it
+// on the platform host under the preview prefix.
+func rewriteDevServerCookie(raw, prefix string) string {
+ parts := strings.Split(raw, ";")
+ out := []string{preview.DevServerCookiePrefix(prefix) + strings.TrimSpace(parts[0])}
+ for _, attr := range parts[1:] {
+ attr = strings.TrimSpace(attr)
+ name, value, _ := strings.Cut(attr, "=")
+ switch strings.ToLower(strings.TrimSpace(name)) {
+ case "":
+ continue
+ case "domain":
+ continue
+ case "path":
+ value = strings.TrimSpace(value)
+ if !strings.HasPrefix(value, "/") {
+ value = "/"
+ }
+ if value != prefix && !strings.HasPrefix(value, prefix+"/") {
+ value = prefix + value
+ }
+ attr = "Path=" + value
+ }
+ out = append(out, attr)
+ }
+ return strings.Join(out, "; ")
+}
+
+func devServerFail(c *gin.Context, status int, message string, retry bool) {
+ if !devServerNavigation(c.Request) {
+ c.Data(status, "text/plain; charset=utf-8", []byte(message))
+ return
+ }
+ refresh := ""
+ hint := ""
+ if retry {
+ refresh = `<meta http-equiv="refresh" content="3">`
+ hint = `<p class="hint">页面每 3 秒自动重试;让 Agent 启动开发服务器后这里会自动打开。</p>`
+ }
+ page := fmt.Sprintf(`<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">%s<title>开发服务器预览</title><style>body{margin:0;min-height:100vh;display:grid;place-items:center;font:14px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#3b3f45;background:#f7f8fa}@media(prefers-color-scheme:dark){body{color:#d7dae0;background:#1d1f23}}main{max-width:420px;padding:24px;text-align:center}.hint{opacity:.65;font-size:13px}</style></head><body><main><p>%s</p>%s</main></body></html>`,
+ refresh, html.EscapeString(message), hint)
+ c.Header("Cache-Control", "no-store")
+ c.Data(status, "text/html; charset=utf-8", []byte(page))
+}
+
+// devServerFileSegment marks workspace files in place of a port:
+// /devpreview/{runID}/file/{absolute path} serves what the run browser opens
+// as file://{absolute path}, so relative links resolve as they do there.
+const devServerFileSegment = "file"
+
+// parseDevFilePath splits /devpreview/{runID}/file/{path} of an unescaped
+// request path; file is absolute and cleaned, keeping a trailing slash.
+func parseDevFilePath(p string) (runID, file string, ok bool) {
+ tail, found := strings.CutPrefix(p, devServerPathPrefix)
+ if !found {
+ return "", "", false
+ }
+ runID, tail, found = strings.Cut(tail, "/")
+ if !found || !devServerRunIDPattern.MatchString(runID) {
+ return "", "", false
+ }
+ rest, found := strings.CutPrefix(tail, devServerFileSegment+"/")
+ if !found || rest == "" {
+ return "", "", false
+ }
+ file = path.Clean("/" + rest)
+ if strings.HasSuffix(rest, "/") && file != "/" {
+ file += "/"
+ }
+ return runID, file, true
+}
+
+func (h *browserHandlers) serveDevFile(c *gin.Context, runID, file string) {
+ r := c.Request
+ if r.Method != http.MethodGet && r.Method != http.MethodHead {
+ devServerFail(c, http.StatusMethodNotAllowed, "工作区文件只支持读取", false)
+ return
+ }
+ run, ok := h.devServerRun(c, runID)
+ if !ok {
+ return
+ }
+ res, ok := h.devServerFetch(c, run, domain.BrowserRequest{Operation: "file", File: file}, "工作区文件暂不可用")
+ if !ok {
+ return
+ }
+ header := c.Writer.Header()
+ if res.Status == http.StatusMovedPermanently {
+ // A directory: serve it with a slash so its relative links resolve.
+ target := r.URL.EscapedPath() + "/"
+ if r.URL.RawQuery != "" {
+ target += "?" + r.URL.RawQuery
+ }
+ c.Redirect(http.StatusMovedPermanently, target)
+ return
+ }
+ if v := res.Header["Content-Type"]; len(v) > 0 {
+ header.Set("Content-Type", v[0])
+ }
+ // The Agent keeps editing these files; always revalidate.
+ header.Set("Cache-Control", "no-cache")
+ header.Set("Referrer-Policy", "same-origin")
+ header.Set("X-Content-Type-Options", "nosniff")
+ header.Set("Content-Length", strconv.Itoa(len(res.Body)))
+ status := res.Status
+ if status < 100 || status > 999 {
+ status = http.StatusBadGateway
+ }
+ c.Status(status)
+ if r.Method != http.MethodHead {
+ _, _ = c.Writer.Write(res.Body)
+ }
+}
diff --git a/internal/httpapi/browser_devserver_sandbox_live_test.go b/internal/httpapi/browser_devserver_sandbox_live_test.go
new file mode 100644
index 00000000..ea1c6609
--- /dev/null
+++ b/internal/httpapi/browser_devserver_sandbox_live_test.go
@@ -0,0 +1,198 @@
+//go:build sandboxlive
+
+package httpapi
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "agentflow/internal/domain"
+ "agentflow/internal/engine"
+ "agentflow/internal/runtime"
+
+ "github.com/gin-gonic/gin"
+)
+
+// The dev server binds 127.0.0.1 inside the sandbox and the port is not
+// published. The preview reaches it through the browser sidecar, which
+// shares that network namespace, the same way a run does.
+func TestDevServerPreviewReachesDockerSandbox(t *testing.T) {
+ if _, err := exec.LookPath("docker"); err != nil {
+ t.Skip("docker not installed")
+ }
+ if err := exec.Command("docker", "info").Run(); err != nil {
+ t.Skip("docker daemon not running")
+ }
+ const (
+ runID = "live-devpreview"
+ port = "4173"
+ image = "agentflow-browser-c34051af5914b90c"
+ sandbox = "lf-devpreview-sandbox"
+ sidecar = "lf-devpreview-browser"
+ pageToken = "sandbox-live-7f3a"
+ fileToken = "workspace-live-marker"
+ )
+ if err := exec.Command("docker", "image", "inspect", image).Run(); err != nil {
+ t.Fatalf("browser sidecar image %s is not present", image)
+ }
+
+ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)
+ run, ok := a.repo.RunState("shared-run")
+ if !ok {
+ t.Fatal("fixture run missing")
+ }
+ run.ID = runID
+ run.ExecutionContext.ExecutionMode = domain.ExecutionModeCloud
+ if err := a.repo.PutRun(run); err != nil {
+ t.Fatal(err)
+ }
+
+ // The Docker VM shares the home directory. macOS temp dirs and /tmp are
+ // not visible inside containers, so the server script and workspace have
+ // to live under the home directory.
+ home, err := os.UserHomeDir()
+ if err != nil {
+ t.Fatal(err)
+ }
+ root := filepath.Join(home, "tmp", "lf-devpreview-live")
+ _ = os.RemoveAll(root)
+ ws := filepath.Join(root, "workspace")
+ if err := os.MkdirAll(ws, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.RemoveAll(root) })
+ if err := os.WriteFile(filepath.Join(ws, "index.html"), []byte("<p>"+fileToken+"</p>"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ serverJS := `const http=require("http");
+const token=` + "`" + pageToken + "`" + `;
+http.createServer((req,res)=>{
+ if(req.url==="/headers"){
+ const names=["host","cookie","x-forwarded-for","x-real-ip","x-app-trace","authorization"];
+ const out={}; for(const n of names) out[n]=req.headers[n]||"";
+ res.setHeader("Content-Type","application/json"); res.end(JSON.stringify(out)); return;
+ }
+ if(req.url==="/src/main.ts"){
+ res.setHeader("Content-Type","text/javascript"); res.end('export default "from-sandbox"'); return;
+ }
+ res.setHeader("Content-Type","text/html; charset=utf-8");
+ res.setHeader("Set-Cookie","sid=live; Path=/");
+ res.end('<!doctype html><html><head></head><body><p>'+token+'</p><script type="module" src="/src/main.ts"></script></body></html>');
+}).listen(4173,"127.0.0.1");`
+ script := filepath.Join(root, "server.js")
+ if err := os.WriteFile(script, []byte(serverJS), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ rm := func() {
+ _ = exec.Command("docker", "rm", "-f", sidecar, sandbox).Run()
+ }
+ rm()
+ t.Cleanup(rm)
+ if out, err := exec.Command("docker", "run", "-d", "--name", sandbox,
+ "-v", script+":/server.js:ro", "--entrypoint", "node", "node:22-bookworm-slim", "/server.js").CombinedOutput(); err != nil {
+ t.Fatalf("start sandbox: %v: %s", err, out)
+ }
+ time.Sleep(300 * time.Millisecond)
+ if state, _ := exec.Command("docker", "inspect", "-f", "{{.State.Status}}", sandbox).Output(); strings.TrimSpace(string(state)) != "running" {
+ logs, _ := exec.Command("docker", "logs", sandbox).CombinedOutput()
+ t.Fatalf("sandbox %s: %s", strings.TrimSpace(string(state)), logs)
+ }
+ if out, err := exec.Command("docker", "run", "-d", "--name", sidecar, "--init",
+ "--label", "agentflow.browser.run="+runID,
+ "--label", "agentflow.browser.owner="+sandbox,
+ "--network", "container:"+sandbox,
+ "--cap-drop=ALL", "--security-opt", "no-new-privileges",
+ "--entrypoint", "sleep",
+ "-v", ws+":/workspace",
+ image, "600").CombinedOutput(); err != nil {
+ t.Fatalf("start sidecar: %v: %s", err, out)
+ }
+
+ h := &browserHandlers{deps: a.deps}
+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))
+ current := owner
+ g := gin.New()
+ g.Use(func(c *gin.Context) {
+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(current)))
+ })
+ g.Use(h.devServerHost())
+ prefix := "/devpreview/" + runID + "/" + port
+ serve := func(method, path string, header map[string]string) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(method, "http://platform.test"+path, nil)
+ for k, v := range header {
+ req.Header.Set(k, v)
+ }
+ rec := httptest.NewRecorder()
+ g.ServeHTTP(rec, req)
+ return rec
+ }
+
+ conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, 300*time.Millisecond)
+ if err == nil {
+ conn.Close()
+ t.Fatal("sandbox port is reachable on the host; the preview would not be proving the network namespace")
+ }
+ if published, err := exec.Command("docker", "port", sandbox).Output(); err != nil || strings.TrimSpace(string(published)) != "" {
+ t.Fatalf("sandbox published ports: %q err=%v", published, err)
+ }
+
+ rec := serve(http.MethodGet, prefix+"/", map[string]string{"Accept": "text/html", "Sec-Fetch-Mode": "navigate"})
+ body := rec.Body.String()
+ if rec.Code != 200 || !strings.Contains(body, pageToken) || !strings.Contains(body, `src="`+prefix+`/src/main.ts"`) || !strings.Contains(body, "data-af-dev-server") {
+ t.Fatalf("page: %d %s", rec.Code, body)
+ }
+ if cookie := rec.Header().Get("Set-Cookie"); cookie != "af_devpreview_"+runID+"_"+port+"_sid=live; Path="+prefix+"/" {
+ t.Fatalf("cookie %q", cookie)
+ }
+
+ rec = serve(http.MethodGet, prefix+"/src/main.ts", nil)
+ if rec.Code != 200 || !strings.Contains(rec.Body.String(), `export default "from-sandbox"`) {
+ t.Fatalf("module: %d %s", rec.Code, rec.Body.String())
+ }
+
+ rec = serve(http.MethodGet, prefix+"/headers", map[string]string{
+ "Cookie": "af_devpreview_" + runID + "_" + port + "_sid=live; platform_session=secret",
+ "X-Forwarded-For": "203.0.113.8",
+ "X-Real-IP": "203.0.113.9",
+ "X-App-Trace": "trace-from-browser",
+ "Authorization": "Basic cGxhdGZvcm06cGFzcw==",
+ "Origin": "http://platform.test",
+ "Referer": "http://platform.test" + prefix + "/admin",
+ })
+ if rec.Code != 200 {
+ t.Fatalf("headers: %d %s", rec.Code, rec.Body.String())
+ }
+ var seen map[string]string
+ if err := json.Unmarshal(rec.Body.Bytes(), &seen); err != nil {
+ t.Fatal(err)
+ }
+ if seen["host"] != "localhost:"+port || seen["cookie"] != "sid=live" || seen["x-forwarded-for"] != "" || seen["x-real-ip"] != "" || seen["authorization"] != "" || seen["x-app-trace"] != "trace-from-browser" {
+ t.Fatalf("upstream headers %+v", seen)
+ }
+
+ rec = serve(http.MethodGet, "/devpreview/"+runID+"/file/workspace/index.html", nil)
+ if rec.Code != 200 || !strings.Contains(rec.Body.String(), fileToken) {
+ t.Fatalf("workspace file: %d %s", rec.Code, rec.Body.String())
+ }
+
+ rec = serve(http.MethodGet, "/devpreview/"+runID+"/4174/", map[string]string{"Sec-Fetch-Mode": "navigate"})
+ if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "没有正在监听") {
+ t.Fatalf("idle port: %d %s", rec.Code, rec.Body.String())
+ }
+
+ current = outsider
+ if rec = serve(http.MethodGet, prefix+"/", nil); rec.Code != http.StatusForbidden {
+ t.Fatalf("outsider: %d %s", rec.Code, rec.Body.String())
+ }
+}
diff --git a/internal/httpapi/browser_devserver_test.go b/internal/httpapi/browser_devserver_test.go
new file mode 100644
index 00000000..ae6edf24
--- /dev/null
+++ b/internal/httpapi/browser_devserver_test.go
@@ -0,0 +1,316 @@
+package httpapi
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+
+ "agentflow/internal/domain"
+ "agentflow/internal/engine"
+ "agentflow/internal/runtime"
+
+ "github.com/gin-gonic/gin"
+)
+
+// fakeHostBrowser registers a host browser for runID the way a host driver
+// does, so dev server requests are served from this machine's loopback.
+func fakeHostBrowser(t *testing.T, runID string) {
+ t.Helper()
+ fakeHostBrowserIn(t, runID, "")
+}
+
+// fakeHostBrowserIn also records the workspace the browser serves file:// from.
+func fakeHostBrowserIn(t *testing.T, runID, workspace string) {
+ t.Helper()
+ devtools := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }))
+ t.Cleanup(devtools.Close)
+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(devtools.URL, "http://"))
+ port, _ := strconv.Atoi(rawPort)
+ dir := filepath.Join(os.TempDir(), "agentflow-browser", runID)
+ _ = os.RemoveAll(dir)
+ t.Cleanup(func() { _ = os.RemoveAll(dir) })
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ record, _ := json.Marshal(map[string]any{"pid": os.Getpid(), "owner": os.Getpid(), "webSocket": "ws://127.0.0.1:" + rawPort, "port": port, "workspace": workspace})
+ if err := os.WriteFile(filepath.Join(dir, "endpoint.json"), record, 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestDevServerPreviewProxiesTheSandboxPort(t *testing.T) {
+ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)
+ h := &browserHandlers{deps: a.deps}
+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))
+ fakeHostBrowser(t, "shared-run")
+
+ var seen *http.Request
+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ seen = r
+ switch r.URL.Path {
+ case "/":
+ http.SetCookie(w, &http.Cookie{Name: "sid", Value: "1", Path: "/", Domain: "localhost"})
+ w.Header().Set("X-Frame-Options", "DENY")
+ w.Header().Set("Content-Type", "text/html")
+ _, _ = w.Write([]byte(`<!doctype html><html><head><title>app</title></head><body><script type="module" src="/src/main.ts"></script></body></html>`))
+ case "/login":
+ http.Redirect(w, r, "http://localhost:"+r.Host[strings.LastIndex(r.Host, ":")+1:]+"/home?x=1", http.StatusFound)
+ default:
+ w.Header().Set("Content-Type", "text/javascript")
+ _, _ = w.Write([]byte(`export default 1`))
+ if r.URL.Path == "/src/app.ts" {
+ _, _ = w.Write([]byte(`;import "/src/b.ts"`))
+ }
+ }
+ }))
+ defer dev.Close()
+ _, devPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))
+ prefix := "/devpreview/shared-run/" + devPort
+
+ g := gin.New()
+ user := owner
+ g.Use(func(c *gin.Context) {
+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(user)))
+ })
+ g.Use(h.devServerHost())
+ g.NoRoute(func(c *gin.Context) { c.String(404, "platform") })
+ serve := func(method, path string, header map[string]string) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(method, "http://platform.test"+path, nil)
+ for k, v := range header {
+ req.Header.Set(k, v)
+ }
+ rec := httptest.NewRecorder()
+ g.ServeHTTP(rec, req)
+ return rec
+ }
+
+ if rec := serve("GET", prefix, nil); rec.Code != 302 || rec.Header().Get("Location") != prefix+"/" {
+ t.Fatalf("bare prefix: %d %q", rec.Code, rec.Header().Get("Location"))
+ }
+ rec := serve("GET", prefix+"/", map[string]string{"Accept": "text/html"})
+ body := rec.Body.String()
+ if rec.Code != 200 || !strings.Contains(body, "data-af-dev-server") || !strings.Contains(body, "data-af-preview-history") {
+ t.Fatalf("document: %d %s", rec.Code, body)
+ }
+ if !strings.Contains(body, `src="`+prefix+`/src/main.ts"`) || strings.Contains(body, "<base") {
+ t.Fatalf("module references must move under the prefix: %s", body)
+ }
+ if rec.Header().Get("X-Frame-Options") != "" || rec.Header().Get("Referrer-Policy") != "same-origin" {
+ t.Fatalf("headers: %v", rec.Header())
+ }
+ if cookie := rec.Header().Get("Set-Cookie"); cookie != "af_devpreview_shared-run_"+devPort+"_sid=1; Path="+prefix+"/" {
+ t.Fatalf("cookie: %q", cookie)
+ }
+
+ // A module import from the page comes back under the prefix by Referer.
+ rec = serve("GET", "/src/main.ts?v=2", map[string]string{"Referer": "http://platform.test" + prefix + "/", "Sec-Fetch-Site": "same-origin"})
+ if rec.Code != 307 || rec.Header().Get("Location") != prefix+"/src/main.ts?v=2" {
+ t.Fatalf("stray import: %d %q", rec.Code, rec.Header().Get("Location"))
+ }
+ if rec := serve("GET", "/src/main.ts", map[string]string{"Referer": "http://platform.test/chatbot/", "Sec-Fetch-Site": "same-origin"}); rec.Code != 404 || rec.Body.String() != "platform" {
+ t.Fatalf("platform request rerouted: %d", rec.Code)
+ }
+ if rec := serve("GET", "/src/main.ts", map[string]string{"Referer": "http://evil.test" + prefix + "/", "Sec-Fetch-Site": "cross-site"}); rec.Code != 404 {
+ t.Fatalf("cross-site request rerouted: %d", rec.Code)
+ }
+
+ if rec := serve("GET", prefix+"/src/app.ts", nil); !strings.Contains(rec.Body.String(), `import "`+prefix+`/src/b.ts"`) {
+ t.Fatalf("module imports: %s", rec.Body.String())
+ }
+
+ rec = serve("GET", prefix+"/login", nil)
+ if rec.Code != 302 || rec.Header().Get("Location") != prefix+"/home?x=1" {
+ t.Fatalf("redirect: %d %q", rec.Code, rec.Header().Get("Location"))
+ }
+
+ rec = serve("GET", prefix+"/src/main.ts", map[string]string{
+ // Only this run's cookies reach its dev server: not the platform's,
+ // not another run's on the same port, not the old port-only names.
+ "Cookie": "af_devpreview_shared-run_" + devPort + "_sid=1; " + authCookieName + "=secret; " +
+ "af_devpreview_other-run_" + devPort + "_leak=2; afdev" + devPort + "_old=3",
+ "Origin": "http://platform.test",
+ "Referer": "http://platform.test" + prefix + "/admin?a=1",
+ "Staffname": "bob",
+ "X-Forwarded-For": "10.0.0.1",
+ "X-Requested-With": "XMLHttpRequest",
+ "Authorization": "Basic cGxhdGZvcm06cGFzcw==",
+ })
+ if rec.Code != 200 || rec.Body.String() != "export default 1" {
+ t.Fatalf("module: %d %s", rec.Code, rec.Body.String())
+ }
+ if got := seen.Header.Get("Cookie"); got != "sid=1" {
+ t.Fatalf("upstream cookie %q", got)
+ }
+ if seen.Host != "localhost:"+devPort || seen.Header.Get("Origin") != "http://localhost:"+devPort || seen.Header.Get("Referer") != "http://localhost:"+devPort+"/admin?a=1" {
+ t.Fatalf("upstream host/origin/referer: %q %q %q", seen.Host, seen.Header.Get("Origin"), seen.Header.Get("Referer"))
+ }
+ for _, name := range []string{"Staffname", "X-Forwarded-For", "Authorization"} {
+ if seen.Header.Get(name) != "" {
+ t.Fatalf("upstream got %s", name)
+ }
+ }
+ if seen.Header.Get("X-Requested-With") != "XMLHttpRequest" {
+ t.Fatal("app headers must pass")
+ }
+
+ if rec := serve("GET", prefix+"/", map[string]string{"Upgrade": "websocket", "Connection": "Upgrade", "Sec-Websocket-Version": "13"}); rec.Code != 501 {
+ t.Fatalf("websocket: %d", rec.Code)
+ }
+
+ closed, _ := net.Listen("tcp", "127.0.0.1:0")
+ _, idlePort, _ := net.SplitHostPort(closed.Addr().String())
+ closed.Close()
+ rec = serve("GET", "/devpreview/shared-run/"+idlePort+"/", map[string]string{"Sec-Fetch-Mode": "navigate"})
+ if rec.Code != 502 || !strings.Contains(rec.Body.String(), "没有正在监听") || !strings.Contains(rec.Body.String(), `http-equiv="refresh"`) {
+ t.Fatalf("idle port: %d %s", rec.Code, rec.Body.String())
+ }
+
+ user = outsider
+ if rec := serve("GET", prefix+"/", nil); rec.Code != 403 {
+ t.Fatalf("outsider: %d", rec.Code)
+ }
+}
+
+func TestParseDevServerPath(t *testing.T) {
+ for _, tc := range []struct {
+ path string
+ run string
+ port int
+ rest string
+ slash bool
+ ok bool
+ }{
+ {"/devpreview/run-1/5173/src/a.ts", "run-1", 5173, "src/a.ts", true, true},
+ {"/devpreview/run-1/5173/", "run-1", 5173, "", true, true},
+ {"/devpreview/run-1/5173", "run-1", 5173, "", false, true},
+ {"/devpreview/run-1/05173/", "", 0, "", false, false},
+ {"/devpreview/run-1/70000/", "", 0, "", false, false},
+ {"/devpreview/run 1/80/", "", 0, "", false, false},
+ {"/devpreview/run-1", "", 0, "", false, false},
+ {"/preview/run-1/80/", "", 0, "", false, false},
+ } {
+ run, port, rest, slash, ok := parseDevServerPath(tc.path)
+ if run != tc.run || port != tc.port || rest != tc.rest || slash != tc.slash || ok != tc.ok {
+ t.Fatalf("%s: %q %d %q %v %v", tc.path, run, port, rest, slash, ok)
+ }
+ }
+}
+
+func TestDevServerETagsCarryTheRewriteVersion(t *testing.T) {
+ if got := devServerETag(`W/"12-abc"`); got != `W/"12-abc-af1"` {
+ t.Fatal(got)
+ }
+ if got := devServerUpstreamETags([]string{`W/"12-abc-af1", "old"`, `"x-af1"`}); got != `W/"12-abc", "x"` {
+ t.Fatal(got)
+ }
+ if got := devServerUpstreamETags([]string{`"old"`}); got != "" {
+ t.Fatal(got)
+ }
+}
+
+func TestRewriteDevServerCookie(t *testing.T) {
+ prefix := "/devpreview/r/3000"
+ for raw, want := range map[string]string{
+ "a=1; Path=/; Domain=localhost; HttpOnly": "af_devpreview_r_3000_a=1; Path=/devpreview/r/3000/; HttpOnly",
+ "a=1; path=/api; SameSite=Lax": "af_devpreview_r_3000_a=1; Path=/devpreview/r/3000/api; SameSite=Lax",
+ "a=1": "af_devpreview_r_3000_a=1",
+ } {
+ if got := rewriteDevServerCookie(raw, prefix); got != want {
+ t.Fatalf("%q: %q", raw, got)
+ }
+ }
+ _ = domain.BrowserRequest{}
+}
+
+func TestDevServerPreviewServesWorkspaceFiles(t *testing.T) {
+ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)
+ h := &browserHandlers{deps: a.deps}
+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))
+ ws := t.TempDir()
+ for name, body := range map[string]string{"demo.html": `<link href="style.css"><h1>demo</h1>`, "style.css": "h1{}", "site/index.html": "<p>site</p>"} {
+ _ = os.MkdirAll(filepath.Dir(filepath.Join(ws, name)), 0o755)
+ if err := os.WriteFile(filepath.Join(ws, name), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ fakeHostBrowserIn(t, "shared-run", ws)
+
+ g := gin.New()
+ user := owner
+ g.Use(func(c *gin.Context) {
+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(user)))
+ })
+ g.Use(h.devServerHost())
+ g.NoRoute(func(c *gin.Context) { c.String(404, "platform") })
+ serve := func(method, path string) *httptest.ResponseRecorder {
+ t.Helper()
+ rec := httptest.NewRecorder()
+ g.ServeHTTP(rec, httptest.NewRequest(method, "http://platform.test"+path, nil))
+ return rec
+ }
+ prefix := "/devpreview/shared-run/file" + ws
+
+ rec := serve("GET", prefix+"/demo.html")
+ if rec.Code != 200 || rec.Body.String() != `<link href="style.css"><h1>demo</h1>` || rec.Header().Get("Content-Type") != "text/html; charset=utf-8" {
+ t.Fatalf("page: %d %q %v", rec.Code, rec.Body.String(), rec.Header())
+ }
+ if rec.Header().Get("Cache-Control") != "no-cache" || rec.Header().Get("Referrer-Policy") != "same-origin" {
+ t.Fatalf("headers: %v", rec.Header())
+ }
+ if rec := serve("GET", prefix+"/style.css"); rec.Code != 200 || rec.Header().Get("Content-Type") != "text/css; charset=utf-8" {
+ t.Fatalf("stylesheet: %d %v", rec.Code, rec.Header())
+ }
+ if rec := serve("GET", prefix+"/site"); rec.Code != 301 || rec.Header().Get("Location") != prefix+"/site/" {
+ t.Fatalf("directory: %d %q", rec.Code, rec.Header().Get("Location"))
+ }
+ if rec := serve("GET", prefix+"/site/"); rec.Code != 200 || rec.Body.String() != "<p>site</p>" {
+ t.Fatalf("index: %d %q", rec.Code, rec.Body.String())
+ }
+ if rec := serve("HEAD", prefix+"/site/"); rec.Code != 200 || rec.Body.Len() != 0 {
+ t.Fatalf("head: %d %q", rec.Code, rec.Body.String())
+ }
+ if rec := serve("GET", prefix+"/missing.html"); rec.Code != 404 {
+ t.Fatalf("missing: %d", rec.Code)
+ }
+ if err := os.WriteFile(filepath.Join(filepath.Dir(ws), "secret.txt"), []byte("secret"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if rec := serve("GET", prefix+"/../secret.txt"); rec.Code != 403 {
+ t.Fatalf("outside: %d %q", rec.Code, rec.Body.String())
+ }
+ if rec := serve("POST", prefix+"/demo.html"); rec.Code != 405 {
+ t.Fatalf("post: %d", rec.Code)
+ }
+ user = outsider
+ if rec := serve("GET", prefix+"/demo.html"); rec.Code != 403 {
+ t.Fatalf("outsider: %d", rec.Code)
+ }
+}
+
+func TestParseDevFilePath(t *testing.T) {
+ for _, tc := range []struct {
+ path, run, file string
+ ok bool
+ }{
+ {"/devpreview/run-1/file/workspace/demo.html", "run-1", "/workspace/demo.html", true},
+ {"/devpreview/run-1/file/workspace/site/", "run-1", "/workspace/site/", true},
+ {"/devpreview/run-1/file/workspace/a/../b.html", "run-1", "/workspace/b.html", true},
+ {"/devpreview/run-1/file/", "", "", false},
+ {"/devpreview/run-1/file", "", "", false},
+ {"/devpreview/run-1/5173/file/a", "", "", false},
+ {"/devpreview/run 1/file/a", "", "", false},
+ } {
+ run, file, ok := parseDevFilePath(tc.path)
+ if run != tc.run || file != tc.file || ok != tc.ok {
+ t.Fatalf("%s: %q %q %v", tc.path, run, file, ok)
+ }
+ }
+}
diff --git a/internal/httpapi/browser_stream.go b/internal/httpapi/browser_stream.go
index 8fbddef0..62256dce 100644
--- a/internal/httpapi/browser_stream.go
+++ b/internal/httpapi/browser_stream.go
@@ -29,9 +29,10 @@ const (
browserStreamPing = 20 * time.Second
browserStreamRunRefresh = 10 * time.Second
browserStreamRetry = 4 * time.Second
- // An animated page screencasts at display rate; 25 fps is enough to
- // follow the agent and keeps a busy page from saturating the link.
- browserStreamFrameInterval = 40 * time.Millisecond
+ // Frame long-polls kept waiting at the browser. Three cover a round trip
+ // of a few frame intervals at the rates a sandbox browser renders.
+ browserStreamFramePulls = 3
+ browserStreamPullGap = 15 * time.Millisecond
)
type browserStreamMessage struct {
@@ -169,37 +170,104 @@ func (s *browserStream) execute(ctx context.Context, cancel context.CancelFunc,
}
func (s *browserStream) frames(ctx context.Context, cancel context.CancelFunc) {
- var last time.Time
- for ctx.Err() == nil {
- if wait := browserStreamFrameInterval - time.Since(last); wait > 0 {
- select {
- case <-ctx.Done():
- return
- case <-time.After(wait):
- }
- }
- last = time.Now()
- s.mu.Lock()
- after := s.seq
- s.mu.Unlock()
+ pull := func(ctx context.Context, after int64) json.RawMessage {
payload, _ := s.h.dispatch(ctx, s.currentRun(false), domain.BrowserRequest{Operation: "frame", After: after, WaitMs: 3000})
- if ctx.Err() != nil {
- return
+ return payload
+ }
+ err := pipelineFrames(ctx, browserStreamFramePulls, pull, s.publish, func() int64 {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.seq
+ }, func() bool {
+ select {
+ case <-ctx.Done():
+ return false
+ case <-time.After(browserStreamRetry):
}
- available, err := s.publish(payload)
- if err != nil {
- cancel()
- return
+ s.currentRun(true)
+ return true
+ })
+ if err != nil {
+ cancel()
+ }
+}
+
+// pipelineFrames keeps pulls frame long-polls waiting at the browser, each
+// for a later frame than the latest shown (latest, latest+1, …). A frame
+// then leaves the moment it is captured instead of one round trip after the
+// previous one arrived: a single poll delivered about half the frames a
+// scrolling page produced.
+//
+// Thresholds always count from the latest frame shown, never from earlier
+// polls: a poll that times out on an idle page is sent again for the same
+// frame. Counting up per poll ran the thresholds ahead of the browser while
+// nothing moved, and the runtime only returns an image newer than the
+// threshold, so the next frames were never shown.
+//
+// publish reports whether the browser is available; while it is not, the
+// polls drain and retry waits before the next round.
+func pipelineFrames(
+ ctx context.Context,
+ pulls int,
+ pull func(context.Context, int64) json.RawMessage,
+ publish func(json.RawMessage) (bool, error),
+ latest func() int64,
+ retry func() bool,
+) error {
+ type answer struct {
+ after int64
+ payload json.RawMessage
+ }
+ results := make(chan answer, pulls)
+ var issued time.Time
+ waiting := map[int64]bool{}
+ down := false
+ for {
+ for !down && len(waiting) < pulls && ctx.Err() == nil {
+ // A spaced start keeps a runtime that answers at once (no frame
+ // wait) from turning this into a hot loop.
+ if wait := browserStreamPullGap - time.Since(issued); wait > 0 {
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-time.After(wait):
+ }
+ }
+ issued = time.Now()
+ after := latest()
+ for waiting[after] {
+ after++
+ }
+ waiting[after] = true
+ go func() {
+ payload := pull(ctx, after)
+ select {
+ case results <- answer{after, payload}:
+ case <-ctx.Done():
+ }
+ }()
}
- if available {
+ if len(waiting) == 0 {
+ if ctx.Err() != nil || !retry() {
+ return nil
+ }
+ down = false
continue
}
+ var got answer
select {
case <-ctx.Done():
- return
- case <-time.After(browserStreamRetry):
+ return nil
+ case got = <-results:
+ }
+ delete(waiting, got.after)
+ available, err := publish(got.payload)
+ if err != nil {
+ return err
+ }
+ if !available {
+ down = true
}
- s.currentRun(true)
}
}
diff --git a/internal/httpapi/browser_stream_pipeline_test.go b/internal/httpapi/browser_stream_pipeline_test.go
new file mode 100644
index 00000000..c1af21d5
--- /dev/null
+++ b/internal/httpapi/browser_stream_pipeline_test.go
@@ -0,0 +1,162 @@
+package httpapi
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// fakeFrames is a browser behind a link with the given one-way delay. It
+// captures a frame every interval, or, with interval 0, only when frame is
+// raised. Like browserView.state it sends the image only when it is newer
+// than the poll's threshold, and a poll gives up after timeout.
+type fakeFrames struct {
+ start time.Time
+ interval time.Duration
+ delay time.Duration
+ timeout time.Duration
+ frame atomic.Int64
+ down atomic.Bool
+ pulls atomic.Int64
+}
+
+func (f *fakeFrames) seq() int64 {
+ if f.interval == 0 {
+ return f.frame.Load()
+ }
+ return int64(time.Since(f.start) / f.interval)
+}
+
+func (f *fakeFrames) pull(ctx context.Context, after int64) json.RawMessage {
+ f.pulls.Add(1)
+ time.Sleep(f.delay)
+ if f.down.Load() {
+ return json.RawMessage(`{"available":false,"reason":"gone"}`)
+ }
+ timeout := f.timeout
+ if timeout == 0 {
+ timeout = time.Second
+ }
+ deadline := time.Now().Add(timeout)
+ for f.seq() <= after && time.Now().Before(deadline) && ctx.Err() == nil {
+ time.Sleep(time.Millisecond)
+ }
+ seq, frame := f.seq(), ""
+ if seq > after {
+ frame = "x"
+ }
+ payload := json.RawMessage(fmt.Sprintf(`{"available":true,"seq":%d,"frame":%q}`, seq, frame))
+ time.Sleep(f.delay)
+ return payload
+}
+
+// panelFrames stands in for browserStream.publish: it shows a frame only when
+// it carries an image newer than the last one shown.
+type panelFrames struct {
+ mu sync.Mutex
+ last int64
+ shown int
+}
+
+func (p *panelFrames) publish(payload json.RawMessage) (bool, error) {
+ var state struct {
+ Available bool `json:"available"`
+ Seq int64 `json:"seq"`
+ Frame string `json:"frame"`
+ }
+ _ = json.Unmarshal(payload, &state)
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if state.Frame != "" && state.Seq > p.last {
+ p.last = state.Seq
+ p.shown++
+ }
+ return state.Available, nil
+}
+
+func (p *panelFrames) latest() int64 {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.last
+}
+
+// deliver runs the pipeline for d and counts frames shown.
+func deliver(t *testing.T, pulls int, f *fakeFrames, d time.Duration) int {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), d)
+ defer cancel()
+ panel := &panelFrames{}
+ if err := pipelineFrames(ctx, pulls, f.pull, panel.publish, panel.latest, func() bool { return false }); err != nil {
+ t.Fatal(err)
+ }
+ return panel.shown
+}
+
+func TestPipelinedFramePollsKeepUpWithTheBrowser(t *testing.T) {
+ const run = 1500 * time.Millisecond
+ single := deliver(t, 1, &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond, delay: 60 * time.Millisecond}, run)
+ f := &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond, delay: 60 * time.Millisecond}
+ piped := deliver(t, 3, f, run)
+ produced := int(run / f.interval)
+ // One poll waits a round trip (120ms) per frame; three waiting ahead
+ // deliver up to three per round trip of the 50 captured meanwhile.
+ if single > 15 || piped*2 < single*3 || piped*3 < produced {
+ t.Fatalf("single poll showed %d, pipelined %d of %d produced", single, piped, produced)
+ }
+ // A jump in sequence makes the polls waiting below it stale; each comes
+ // back once and is not re-sent, so polls stay within twice the frames.
+ if pulls := f.pulls.Load(); pulls > int64(2*piped)+5 {
+ t.Fatalf("%d polls for %d frames", pulls, piped)
+ }
+}
+
+func TestPipelinedFramePollsRetryWhileTheBrowserIsDown(t *testing.T) {
+ f := &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond}
+ f.down.Store(true)
+ retries := 0
+ published := 0
+ err := pipelineFrames(context.Background(), 3, f.pull, func(json.RawMessage) (bool, error) {
+ published++
+ return false, nil
+ }, func() int64 { return 0 }, func() bool {
+ retries++
+ return retries < 2
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Each round sends the polls once and drains them before waiting.
+ if retries != 2 || published != 6 || f.pulls.Load() != 6 {
+ t.Fatalf("retries=%d published=%d pulls=%d", retries, published, f.pulls.Load())
+ }
+}
+
+func TestPipelinedFramePollsShowTheFirstFrameAfterAnIdlePage(t *testing.T) {
+ f := &fakeFrames{timeout: 20 * time.Millisecond}
+ f.frame.Store(1)
+ panel := &panelFrames{}
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() { _ = pipelineFrames(ctx, 3, f.pull, panel.publish, panel.latest, func() bool { return false }) }()
+ waitFor := func(seq int64) {
+ t.Helper()
+ for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(2 * time.Millisecond) {
+ if panel.latest() >= seq {
+ return
+ }
+ }
+ t.Fatalf("panel stuck at frame %d, browser at %d", panel.latest(), seq)
+ }
+ waitFor(1)
+ // A still page: every poll times out many times over.
+ time.Sleep(400 * time.Millisecond)
+ f.frame.Store(2)
+ waitFor(2)
+ time.Sleep(200 * time.Millisecond)
+ f.frame.Store(3)
+ waitFor(3)
+}
diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go
index c777dcba..f0c30a7e 100644
--- a/internal/httpapi/server.go
+++ b/internal/httpapi/server.go
@@ -202,6 +202,7 @@ func newWithNotifications(s *repo.Store, e *engine.Engine, syncSvc *tracker.Serv
// 405/404;gin 默认会 301 重定向到无斜杠路径,行为不同,关闭之。
g.RedirectTrailingSlash = false
g.Use(api.previews.previewHost())
+ g.Use(api.browser.devServerHost())
// gin 只有在显式注册 NoMethod 后才会对"路径匹配但方法不符"返回 405,
// 否则落入 404;这里同时把 405 响应统一成 JSON 形态。
g.NoMethod(func(c *gin.Context) {
diff --git a/internal/preview/devserver.go b/internal/preview/devserver.go
new file mode 100644
index 00000000..dd680f3b
--- /dev/null
+++ b/internal/preview/devserver.go
@@ -0,0 +1,184 @@
+package preview
+
+import (
+ _ "embed"
+ "encoding/json"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+//go:embed devserver_intercept.js
+var devServerInterceptJS string
+
+// devServerScriptMax bounds rewritten scripts; pre-bundled dependencies of a
+// dev server are larger than built HTML.
+const devServerScriptMax = 16 << 20
+
+// Root-absolute ES module specifiers: `from "/x"`, `import "/x"`, `import("/x")`.
+var devServerImportRe = regexp.MustCompile(`(\bfrom\s*|\bimport\s*\(?\s*)(["'])(/[^/"'\s][^"'\s]*)(["'])`)
+var devServerScriptRe = regexp.MustCompile(`(?is)(<script\b[^>]*>)(.*?)(</script>)`)
+
+// RewriteDevServerHTML prepares a dev server document for its path prefix,
+// the way build previews are served: root references in markup move under
+// the prefix, the history shim hides the prefix from client routers, and the
+// intercept script keeps runtime requests, sockets and cookies inside it. No
+// <base> is injected: pages resolve relative URLs against their own address.
+//
+// Module specifiers must be rewritten, not just redirected: a module loaded
+// through a redirect keys the module map by the root URL while import.meta.url
+// is the prefixed one, and anything importing import.meta.url (React Refresh
+// does) would evaluate the module a second time. Requests that still go to
+// root paths are redirected under the prefix by the gateway.
+func RewriteDevServerHTML(body []byte, prefix string, port int) []byte {
+ if len(body) > maxRewrittenHTML {
+ return body
+ }
+ prefix = strings.TrimRight(prefix, "/")
+ html := rewriteReferrerMeta(string(body))
+ html = devServerScriptRe.ReplaceAllStringFunc(html, func(m string) string {
+ parts := devServerScriptRe.FindStringSubmatch(m)
+ if len(parts) != 4 || strings.Contains(parts[1], "data-af-") {
+ return m
+ }
+ return parts[1] + rewriteDevServerImports(parts[2], prefix) + parts[3]
+ })
+ html = rewriteRootRelativeRefs(html, prefix)
+ html = rewriteCSSURLs(html, prefix)
+ html = injectDevServerIntercept(html, prefix, port)
+ html = injectPreviewHistory(html, prefix, true)
+ return []byte(html)
+}
+
+// RewriteDevServerScript moves root-absolute module specifiers of a dev
+// server script under the prefix.
+func RewriteDevServerScript(body []byte, prefix string) []byte {
+ if len(body) > devServerScriptMax {
+ return body
+ }
+ prefix = strings.TrimRight(prefix, "/")
+ src := rewriteDevServerImports(string(body), prefix)
+ if strings.Contains(src, "__vite__css") {
+ // Vite serves imported stylesheets as modules that inject a <style>.
+ src = rewriteCSSURLs(src, prefix)
+ }
+ return []byte(src)
+}
+
+// RewriteDevServerCSS moves root url() references under the prefix.
+func RewriteDevServerCSS(body []byte, prefix string) []byte {
+ if len(body) > devServerScriptMax {
+ return body
+ }
+ return []byte(rewriteCSSURLs(string(body), strings.TrimRight(prefix, "/")))
+}
+
+func rewriteDevServerImports(src, prefix string) string {
+ if prefix == "" || !strings.Contains(src, "/") {
+ return src
+ }
+ return devServerImportRe.ReplaceAllStringFunc(src, func(m string) string {
+ parts := devServerImportRe.FindStringSubmatch(m)
+ if len(parts) != 5 || parts[2] != parts[4] {
+ return m
+ }
+ ref := parts[3]
+ if ref == prefix || strings.HasPrefix(ref, prefix+"/") {
+ return m
+ }
+ return parts[1] + parts[2] + prefix + ref + parts[4]
+ })
+}
+
+// DevServerCookiePrefix namespaces the cookies of one dev server preview
+// (its /devpreview/{run}/{port} prefix): the gateway forwards only these to
+// the dev server, and the intercept script shows only these to the page.
+// With the run in the name, two tasks serving the same port never share a
+// cookie, whatever path it was set for.
+func DevServerCookiePrefix(prefix string) string {
+ var b strings.Builder
+ b.WriteString("af_")
+ for _, r := range strings.Trim(prefix, "/") {
+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' {
+ b.WriteRune(r)
+ } else {
+ b.WriteByte('_')
+ }
+ }
+ b.WriteByte('_')
+ return b.String()
+}
+
+func injectDevServerIntercept(html, prefix string, port int) string {
+ if strings.Contains(html, `data-af-dev-server`) || strings.Contains(devServerInterceptJS, "</script>") {
+ return html
+ }
+ prefixJSON, err := json.Marshal(prefix)
+ if err != nil {
+ return html
+ }
+ cookieJSON, _ := json.Marshal(DevServerCookiePrefix(prefix))
+ script := strings.NewReplacer("__AF_PREFIX__", string(prefixJSON), "__AF_PORT__", strconv.Itoa(port),
+ "__AF_COOKIE__", string(cookieJSON)).Replace(devServerInterceptJS)
+ tag := `<script data-af-dev-server>` + script + `</script>`
+ // First thing in the document: module scripts and inline boot code must
+ // already see the patched fetch/WebSocket.
+ lower := strings.ToLower(html)
+ if i := strings.Index(lower, "<head>"); i >= 0 {
+ at := i + len("<head>")
+ return html[:at] + tag + html[at:]
+ }
+ if i := strings.Index(lower, "<head "); i >= 0 {
+ if end := strings.Index(html[i:], ">"); end >= 0 {
+ at := i + end + 1
+ return html[:at] + tag + html[at:]
+ }
+ }
+ if i := strings.Index(lower, "<html"); i >= 0 {
+ if end := strings.Index(html[i:], ">"); end >= 0 {
+ at := i + end + 1
+ return html[:at] + tag + html[at:]
+ }
+ }
+ return tag + html
+}
+
+// RewriteDevServerLocation maps a redirect target of the dev server back
+// under the prefix: root paths and http://localhost:PORT addresses move,
+// other origins stay as they are.
+func RewriteDevServerLocation(raw, prefix string, port int) string {
+ raw = strings.TrimSpace(raw)
+ prefix = strings.TrimRight(prefix, "/")
+ if raw == "" {
+ return raw
+ }
+ u, err := url.Parse(raw)
+ if err != nil {
+ return raw
+ }
+ if u.IsAbs() || strings.HasPrefix(raw, "//") {
+ host := strings.ToLower(u.Hostname())
+ local := host == "localhost" || host == "127.0.0.1" || host == "0.0.0.0" || host == "::1"
+ p := u.Port()
+ if p == "" {
+ p = map[string]string{"https": "443"}[u.Scheme]
+ if p == "" {
+ p = "80"
+ }
+ }
+ if !local || p != strconv.Itoa(port) {
+ return raw
+ }
+ u.Scheme, u.Host, u.User = "", "", nil
+ } else if !strings.HasPrefix(u.Path, "/") {
+ return raw
+ }
+ if u.Path != prefix && !strings.HasPrefix(u.Path, prefix+"/") {
+ u.Path = prefix + u.Path
+ if u.RawPath != "" {
+ u.RawPath = prefix + u.RawPath
+ }
+ }
+ return u.String()
+}
diff --git a/internal/preview/devserver_intercept.js b/internal/preview/devserver_intercept.js
new file mode 100644
index 00000000..b3e3be0d
--- /dev/null
+++ b/internal/preview/devserver_intercept.js
@@ -0,0 +1,229 @@
+(function () {
+ // Dev server pages run under a path prefix on the platform origin. This
+ // keeps their requests, sockets and cookies inside that prefix: root paths
+ // and http://localhost:PORT addresses are moved under it, and cookies are
+ // namespaced so the page neither sees nor sends platform cookies.
+ var root = __AF_PREFIX__;
+ var port = String(__AF_PORT__);
+ var loc = window.location;
+ if (window.__AGENTFLOW_DEV_SERVER__) return;
+ window.__AGENTFLOW_DEV_SERVER__ = { prefix: root, port: Number(port) };
+ var localHosts = { localhost: 1, "127.0.0.1": 1, "0.0.0.0": 1, "[::1]": 1 };
+ function inside(path) {
+ return path === root || path.indexOf(root + "/") === 0;
+ }
+ function fix(href) {
+ if (href == null || href === "") return href;
+ var u;
+ try {
+ u = new URL(String(href), loc.href);
+ } catch (e) {
+ return href;
+ }
+ var socket = u.protocol === "ws:" || u.protocol === "wss:";
+ if (!socket && u.protocol !== "http:" && u.protocol !== "https:") return href;
+ var changed = false;
+ if (localHosts[u.hostname] && (u.port || (socket || u.protocol === "http:" ? "80" : "443")) === port) {
+ u = new URL(u.pathname + u.search + u.hash, loc.href);
+ if (socket) u.protocol = loc.protocol === "https:" ? "wss:" : "ws:";
+ changed = true;
+ } else if (socket && u.host === loc.host) {
+ changed = false;
+ } else if (u.origin !== loc.origin) {
+ return href;
+ }
+ if (!inside(u.pathname)) {
+ u.pathname = root + u.pathname;
+ changed = true;
+ }
+ return changed ? u.href : href;
+ }
+ var ofetch = window.fetch;
+ if (ofetch) {
+ window.fetch = function (input, init) {
+ try {
+ if (typeof Request !== "undefined" && input instanceof Request) {
+ var next = fix(input.url);
+ if (next !== input.url) input = new Request(next, input);
+ } else if (input != null) {
+ input = fix(input instanceof URL ? input.href : input);
+ }
+ } catch (e) {}
+ return ofetch.call(this, input, init);
+ };
+ }
+ var oopen = XMLHttpRequest.prototype.open;
+ XMLHttpRequest.prototype.open = function (m, u) {
+ if (arguments.length > 1 && u != null) {
+ try {
+ arguments[1] = fix(u instanceof URL ? u.href : u);
+ } catch (e) {}
+ }
+ return oopen.apply(this, arguments);
+ };
+ if (navigator.sendBeacon) {
+ var obeacon = navigator.sendBeacon.bind(navigator);
+ navigator.sendBeacon = function (u, d) {
+ return obeacon(fix(u), d);
+ };
+ }
+ if (window.EventSource) {
+ var ES = window.EventSource;
+ var PatchedES = function (u, o) {
+ return new ES(fix(u), o);
+ };
+ PatchedES.prototype = ES.prototype;
+ PatchedES.CONNECTING = ES.CONNECTING;
+ PatchedES.OPEN = ES.OPEN;
+ PatchedES.CLOSED = ES.CLOSED;
+ window.EventSource = PatchedES;
+ }
+ // Live reload sockets cannot cross the platform yet. A socket that fails
+ // makes Vite poll the (reachable) dev server and reload the page forever,
+ // so HMR clients get one that simply never connects; the page works and
+ // shows edits on refresh.
+ function hmrSocket(u, protocols) {
+ var list = protocols == null ? [] : [].concat(protocols);
+ for (var i = 0; i < list.length; i++) if (/^vite-(hmr|ping)$/.test(String(list[i]))) return true;
+ try {
+ return /\/_next\/webpack-hmr$/.test(new URL(u).pathname);
+ } catch (e) {
+ return false;
+ }
+ }
+ if (window.WebSocket) {
+ var WS = window.WebSocket;
+ var Idle = function (u) {
+ var t = typeof EventTarget === "function" ? new EventTarget() : document.createDocumentFragment();
+ t.url = String(u);
+ t.readyState = 0;
+ t.protocol = "";
+ t.extensions = "";
+ t.bufferedAmount = 0;
+ t.binaryType = "blob";
+ t.onopen = t.onmessage = t.onerror = t.onclose = null;
+ t.send = function () {};
+ t.close = function () {
+ t.readyState = 3;
+ };
+ t.CONNECTING = 0;
+ t.OPEN = 1;
+ t.CLOSING = 2;
+ t.CLOSED = 3;
+ return t;
+ };
+ var PatchedWS = function (u, p) {
+ var next = fix(u instanceof URL ? u.href : u);
+ if (hmrSocket(next, p)) return Idle(next);
+ return p === undefined ? new WS(next) : new WS(next, p);
+ };
+ PatchedWS.prototype = WS.prototype;
+ PatchedWS.CONNECTING = WS.CONNECTING;
+ PatchedWS.OPEN = WS.OPEN;
+ PatchedWS.CLOSING = WS.CLOSING;
+ PatchedWS.CLOSED = WS.CLOSED;
+ window.WebSocket = PatchedWS;
+ }
+ // Cookies are namespaced by run and port and kept under the preview
+ // prefix, like the gateway does for Set-Cookie: a page writing
+ // "sid=1; Path=/" must not leave it at the platform root, where every
+ // other preview (and the platform) would see it.
+ var cookiePrefix = __AF_COOKIE__;
+ function scopeCookie(v) {
+ var parts = String(v).split(";");
+ var out = [cookiePrefix + parts[0].replace(/^\s+/, "")];
+ for (var i = 1; i < parts.length; i++) {
+ var attr = parts[i].replace(/^\s+|\s+$/g, "");
+ var eq = attr.indexOf("=");
+ var name = (eq < 0 ? attr : attr.slice(0, eq)).toLowerCase();
+ if (!name || name === "domain") continue;
+ if (name === "path") {
+ var path = eq < 0 ? "" : attr.slice(eq + 1).replace(/^\s+/, "");
+ if (path.charAt(0) !== "/") path = "/";
+ if (!inside(path)) path = root + path;
+ attr = "Path=" + path;
+ }
+ out.push(attr);
+ }
+ return out.join("; ");
+ }
+ var cookie = Object.getOwnPropertyDescriptor(Document.prototype, "cookie");
+ if (cookie && cookie.configurable !== false && cookie.get && cookie.set) {
+ Object.defineProperty(document, "cookie", {
+ configurable: true,
+ get: function () {
+ return String(cookie.get.call(document))
+ .split(/;\s*/)
+ .filter(function (c) {
+ return c.indexOf(cookiePrefix) === 0;
+ })
+ .map(function (c) {
+ return c.slice(cookiePrefix.length);
+ })
+ .join("; ");
+ },
+ set: function (v) {
+ cookie.set.call(document, scopeCookie(v));
+ },
+ });
+ }
+ // While the history shim hides the prefix from a booting router, root
+ // paths resolve against the bare origin and requests carry no usable
+ // Referer, so resource URLs set from script are moved here. Anchors are
+ // left alone: routers read them, and the shim routes their clicks.
+ var resourceAttrs = { src: 1, href: 1, poster: 1, data: 1 };
+ function resource(el, name) {
+ var tag = el && el.tagName;
+ if (!tag || !resourceAttrs[name]) return false;
+ if (name === "href") return tag === "LINK" || tag === "use" || tag === "image";
+ return tag !== "A" && tag !== "AREA" && tag !== "FORM" && tag !== "BASE";
+ }
+ var osetAttribute = Element.prototype.setAttribute;
+ Element.prototype.setAttribute = function (name, value) {
+ try {
+ var lower = String(name).toLowerCase();
+ if (resource(this, lower) && typeof value === "string" && value.charAt(0) === "/") value = fix(value);
+ } catch (e) {}
+ return osetAttribute.call(this, name, value);
+ };
+ function patchProperty(Ctor, prop) {
+ var proto = Ctor && Ctor.prototype;
+ var desc = proto && Object.getOwnPropertyDescriptor(proto, prop);
+ if (!desc || !desc.set || desc.configurable === false) return;
+ Object.defineProperty(proto, prop, {
+ configurable: true,
+ enumerable: desc.enumerable,
+ get: desc.get,
+ set: function (value) {
+ try {
+ if (typeof value === "string" || value instanceof URL) value = fix(String(value));
+ } catch (e) {}
+ return desc.set.call(this, value);
+ },
+ });
+ }
+ [
+ [window.HTMLImageElement, "src"],
+ [window.HTMLScriptElement, "src"],
+ [window.HTMLLinkElement, "href"],
+ [window.HTMLSourceElement, "src"],
+ [window.HTMLMediaElement, "src"],
+ [window.HTMLVideoElement, "poster"],
+ [window.HTMLIFrameElement, "src"],
+ [window.HTMLTrackElement, "src"],
+ [window.HTMLEmbedElement, "src"],
+ [window.HTMLObjectElement, "data"],
+ [window.HTMLInputElement, "src"],
+ ].forEach(function (pair) {
+ patchProperty(pair[0], pair[1]);
+ });
+ var oopenWindow = window.open;
+ window.open = function (u) {
+ if (arguments.length > 0 && u != null && u !== "") {
+ try {
+ arguments[0] = fix(u instanceof URL ? u.href : u);
+ } catch (e) {}
+ }
+ return oopenWindow.apply(this, arguments);
+ };
+})();
diff --git a/internal/preview/devserver_test.go b/internal/preview/devserver_test.go
new file mode 100644
index 00000000..0cf38e05
--- /dev/null
+++ b/internal/preview/devserver_test.go
@@ -0,0 +1,98 @@
+package preview
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestRewriteDevServerHTML(t *testing.T) {
+ in := `<!doctype html><html><head><meta name="referrer" content="no-referrer"><script type="module" src="/@vite/client"></script></head><body><img src="/logo.svg"></body></html>`
+ out := string(RewriteDevServerHTML([]byte(in), "/devpreview/r/5173/", 5173))
+ intercept := strings.Index(out, "<script data-af-dev-server>")
+ if intercept < 0 || intercept > strings.Index(out, `src="/devpreview/r/5173/@vite/client"`) {
+ t.Fatalf("intercept must run before page scripts: %s", out)
+ }
+ if !strings.Contains(out, `var root = "/devpreview/r/5173"`) || !strings.Contains(out, `String(5173)`) {
+ t.Fatalf("intercept placeholders: %s", out)
+ }
+ if !strings.Contains(out, "data-af-preview-history") || !strings.Contains(out, `content="same-origin"`) {
+ t.Fatalf("history shim / referrer: %s", out)
+ }
+ if !strings.Contains(out, `src="/devpreview/r/5173/logo.svg"`) || !strings.Contains(out, `src="/devpreview/r/5173/@vite/client"`) || strings.Contains(out, "<base") {
+ t.Fatalf("markup references: %s", out)
+ }
+ if again := string(RewriteDevServerHTML([]byte(out), "/devpreview/r/5173", 5173)); strings.Count(again, "data-af-dev-server") != 1 {
+ t.Fatal("rewrite must be idempotent")
+ }
+}
+
+func TestRewriteDevServerModuleSpecifiers(t *testing.T) {
+ prefix := "/devpreview/r/5173"
+ html := string(RewriteDevServerHTML([]byte(`<head><script type="module">import { injectIntoGlobalHook } from "/@react-refresh";</script></head>`), prefix, 5173))
+ if !strings.Contains(html, `from "/devpreview/r/5173/@react-refresh"`) {
+ t.Fatalf("inline module: %s", html)
+ }
+ js := `import React from "/node_modules/.vite/deps/react.js?v=1";import "/src/a.css";import{x}from'/src/x.ts';` +
+ `const m = import ( "/src/lazy.tsx" );export * from "/src/re.ts";` +
+ `Array.from("/not/a/module");fetch("/api");importScripts("/sw.js");import "//cdn.example.com/a.js";import "./rel.js";` +
+ `import "/devpreview/r/5173/done.js";`
+ out := string(RewriteDevServerScript([]byte(js), prefix))
+ for _, want := range []string{
+ `from "/devpreview/r/5173/node_modules/.vite/deps/react.js?v=1"`,
+ `import "/devpreview/r/5173/src/a.css"`,
+ `from'/devpreview/r/5173/src/x.ts'`,
+ `import ( "/devpreview/r/5173/src/lazy.tsx" )`,
+ `export * from "/devpreview/r/5173/src/re.ts"`,
+ `Array.from("/not/a/module")`,
+ `fetch("/api")`,
+ `importScripts("/sw.js")`,
+ `import "//cdn.example.com/a.js"`,
+ `import "./rel.js"`,
+ `import "/devpreview/r/5173/done.js"`,
+ } {
+ if !strings.Contains(out, want) {
+ t.Fatalf("missing %s in %s", want, out)
+ }
+ }
+ viteCSS := string(RewriteDevServerScript([]byte(`const __vite__css = ".card{background:url(/bg.svg)}"`), prefix))
+ if !strings.Contains(viteCSS, `url(/devpreview/r/5173/bg.svg)`) {
+ t.Fatalf("vite css module: %s", viteCSS)
+ }
+ if plain := string(RewriteDevServerScript([]byte(`const s = "url(/x)"`), prefix)); plain != `const s = "url(/x)"` {
+ t.Fatalf("plain script urls must stay: %s", plain)
+ }
+ if css := string(RewriteDevServerCSS([]byte(`.a{background:url(/bg.svg)}`), prefix)); css != `.a{background:url(/devpreview/r/5173/bg.svg)}` {
+ t.Fatalf("css: %s", css)
+ }
+}
+
+func TestRewriteDevServerLocation(t *testing.T) {
+ prefix := "/devpreview/r/5173"
+ for raw, want := range map[string]string{
+ "/login?next=/a": prefix + "/login?next=/a",
+ "http://localhost:5173/home#x": prefix + "/home#x",
+ "http://127.0.0.1:5173": prefix,
+ prefix + "/already": prefix + "/already",
+ "http://localhost:3000/other": "http://localhost:3000/other",
+ "https://sso.example.com/login": "https://sso.example.com/login",
+ "relative/path": "relative/path",
+ } {
+ if got := RewriteDevServerLocation(raw, prefix, 5173); got != want {
+ t.Fatalf("%q: got %q want %q", raw, got, want)
+ }
+ }
+}
+
+func TestDevServerCookiePrefixSeparatesRunsOnOnePort(t *testing.T) {
+ a, b := DevServerCookiePrefix("/devpreview/run-a/3000"), DevServerCookiePrefix("/devpreview/run-b/3000/")
+ if a != "af_devpreview_run-a_3000_" || b != "af_devpreview_run-b_3000_" {
+ t.Fatalf("prefixes %q %q", a, b)
+ }
+ if got := DevServerCookiePrefix("/devpreview/r.x;y/file"); got != "af_devpreview_r_x_y_file_" {
+ t.Fatalf("cookie names allow only token characters: %q", got)
+ }
+ html := string(RewriteDevServerHTML([]byte("<html><head></head><body></body></html>"), "/devpreview/run-a/3000", 3000))
+ if !strings.Contains(html, `var cookiePrefix = "af_devpreview_run-a_3000_"`) || strings.Contains(html, "__AF_") {
+ t.Fatalf("intercept script must carry the run's cookie namespace")
+ }
+}
diff --git a/internal/runtime/browser.go b/internal/runtime/browser.go
index 090fef08..a1eb90e0 100644
--- a/internal/runtime/browser.go
+++ b/internal/runtime/browser.go
@@ -63,6 +63,8 @@ type browserEndpoint struct {
Kind string
Container string
WebSocket string
+ // Workspace is the host directory a local browser serves file:// from.
+ Workspace string
}
func (e browserEndpoint) dial(ctx context.Context) (cdpTransport, error) {
@@ -130,6 +132,18 @@ func browserUsageHint(servers []domain.MCPServerDefinition) string {
return ""
}
+// BrowserDeviceScale is the device scale factor run browsers render at
+// (AGENTFLOW_BROWSER_SCALE, 1 to 3, default 2). The task panel shows their
+// screencast on high-DPI screens, where 1x frames are upscaled and blurred;
+// screencast frames follow the browser's real scale, not an emulated one.
+func BrowserDeviceScale() string {
+ raw := strings.TrimSpace(os.Getenv("AGENTFLOW_BROWSER_SCALE"))
+ if value, err := strconv.ParseFloat(raw, 64); err == nil && value >= 1 && value <= 3 {
+ return strconv.FormatFloat(value, 'f', -1, 64)
+ }
+ return "2"
+}
+
func dockerBrowserContainerName(base string) string { return base + "-browser" }
var browserImageMu sync.Mutex
@@ -234,7 +248,7 @@ func (s *dockerSandbox) ensureBrowser(ctx context.Context, runID string, sink Ev
"--label", browserRunLabel + "=" + runID, "--label", browserOwnerLabel + "=" + s.name,
"--network", "container:" + s.name, "--cap-drop=ALL", "--security-opt", "no-new-privileges",
"--shm-size", "512m", "-v", s.workspace + ":/workspace", "-w", "/workspace",
- "-e", "AGENTFLOW_BROWSER_OUTPUT=/workspace/.agentflow/browser", image}
+ "-e", "AGENTFLOW_BROWSER_OUTPUT=/workspace/.agentflow/browser", "-e", "AGENTFLOW_BROWSER_SCALE=" + BrowserDeviceScale(), image}
if out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput(); err != nil {
return domain.MCPServerDefinition{}, fmt.Errorf("启动浏览器容器失败: %w: %s", err, tailOutput(out))
}
@@ -323,7 +337,7 @@ func ensureHostBrowser(ctx context.Context, workspace, runID string) (domain.MCP
_ = os.Remove(filepath.Join(profile, "DevToolsActivePort"))
args := []string{"--headless=new", "--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0",
"--user-data-dir=" + profile, "--no-first-run", "--no-default-browser-check", "--disable-background-networking",
- "--window-size=1280,800", "--lang=zh-CN"}
+ "--window-size=1280,800", "--lang=zh-CN", "--force-device-scale-factor=" + BrowserDeviceScale()}
if goruntime.GOOS == "linux" && os.Geteuid() == 0 {
args = append(args, "--no-sandbox", "--disable-dev-shm-usage")
}
@@ -508,7 +522,7 @@ func lookupBrowserEndpoint(ctx context.Context, runID string) (browserEndpoint,
if raw, err := os.ReadFile(filepath.Join(hostBrowserDir(runID), "endpoint.json")); err == nil {
var record hostBrowserRecord
if json.Unmarshal(raw, &record) == nil && record.PID > 0 && syscall.Kill(record.PID, 0) == nil && hostDevToolsAlive(ctx, record.Port) {
- return browserEndpoint{Kind: "local", WebSocket: record.WebSocket}, true
+ return browserEndpoint{Kind: "local", WebSocket: record.WebSocket, Workspace: record.Workspace}, true
}
}
if !dockerAvailable() {
diff --git a/internal/runtime/browser_agent.go b/internal/runtime/browser_agent.go
index b6b6e7f6..caada619 100644
--- a/internal/runtime/browser_agent.go
+++ b/internal/runtime/browser_agent.go
@@ -211,10 +211,33 @@ func (v *browserView) screenshot(ctx context.Context) (string, error) {
if session == "" {
return "", errors.New("没有可操作的标签页")
}
+ // The browser renders at a high device scale for the panel; the model
+ // gets CSS-pixel screenshots, which carry the same content in a quarter
+ // of the image.
+ params := map[string]any{"format": "jpeg", "quality": 75}
+ var metrics struct {
+ CSSVisualViewport struct {
+ PageX float64 `json:"pageX"`
+ PageY float64 `json:"pageY"`
+ ClientWidth float64 `json:"clientWidth"`
+ ClientHeight float64 `json:"clientHeight"`
+ } `json:"cssVisualViewport"`
+ }
+ var ratio struct {
+ Result struct {
+ Value float64 `json:"value"`
+ } `json:"result"`
+ }
+ if v.client.call(ctx, session, "Page.getLayoutMetrics", nil, &metrics) == nil &&
+ v.client.call(ctx, session, "Runtime.evaluate", map[string]any{"expression": "window.devicePixelRatio", "returnByValue": true}, &ratio) == nil &&
+ ratio.Result.Value > 1 && metrics.CSSVisualViewport.ClientWidth > 0 {
+ vp := metrics.CSSVisualViewport
+ params["clip"] = map[string]any{"x": vp.PageX, "y": vp.PageY, "width": vp.ClientWidth, "height": vp.ClientHeight, "scale": 1 / ratio.Result.Value}
+ }
var shot struct {
Data string `json:"data"`
}
- if err := v.client.call(ctx, session, "Page.captureScreenshot", map[string]any{"format": "jpeg", "quality": 75}, &shot); err != nil {
+ if err := v.client.call(ctx, session, "Page.captureScreenshot", params, &shot); err != nil {
return "", err
}
return shot.Data, nil
diff --git a/internal/runtime/browser_assets/entrypoint.sh b/internal/runtime/browser_assets/entrypoint.sh
index 0992ff5a..6bf2c12d 100644
--- a/internal/runtime/browser_assets/entrypoint.sh
+++ b/internal/runtime/browser_assets/entrypoint.sh
@@ -10,7 +10,8 @@ mkdir -p "$profile" "$output"
chromium --headless=new --no-sandbox --disable-dev-shm-usage --disable-gpu \
--no-first-run --no-default-browser-check --disable-background-networking \
--remote-debugging-address=127.0.0.1 --remote-debugging-port="${AGENTFLOW_BROWSER_CDP_PORT:-9222}" \
- --user-data-dir="$profile" --window-size=1280,800 --lang=zh-CN about:blank >/tmp/chromium.log 2>&1
+ --user-data-dir="$profile" --window-size=1280,800 --lang=zh-CN \
+ --force-device-scale-factor="${AGENTFLOW_BROWSER_SCALE:-2}" about:blank >/tmp/chromium.log 2>&1
sleep 1
done
) &
diff --git a/internal/runtime/browser_devserver.go b/internal/runtime/browser_devserver.go
new file mode 100644
index 00000000..6102903e
--- /dev/null
+++ b/internal/runtime/browser_devserver.go
@@ -0,0 +1,317 @@
+package runtime
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net"
+ "net/http"
+ "os/exec"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "agentflow/internal/domain"
+)
+
+// Dev server access: the user's own browser renders a dev server that runs
+// in the run sandbox, and this side forwards each HTTP request to
+// localhost:port in the sandbox's network namespace. Docker sandboxes are
+// reached through the browser sidecar (it shares that namespace and ships
+// node); host drivers reach the port directly.
+
+const (
+ // DevServerBodyLimit keeps a response, base64 encoded, inside the runner
+ // socket's 20 MiB message limit.
+ DevServerBodyLimit = 12 << 20
+ devServerEndpointTTL = 15 * time.Second
+ devServerClientIdle = 10 * time.Minute
+)
+
+// ErrDevServerUnreachable means nothing accepted a connection on the port.
+var ErrDevServerUnreachable = errors.New("dev server unreachable")
+
+type devServerClient struct {
+ endpoint browserEndpoint
+ checked time.Time
+ used time.Time
+ client *http.Client
+}
+
+var devServerClients = struct {
+ sync.Mutex
+ items map[string]*devServerClient
+}{items: map[string]*devServerClient{}}
+
+// devServerDial is swapped in tests.
+var devServerDial = func(ctx context.Context, endpoint browserEndpoint, port int) (net.Conn, error) {
+ if endpoint.Kind == "docker" {
+ return dialDockerPort(ctx, endpoint.Container, port)
+ }
+ var d net.Dialer
+ return d.DialContext(ctx, "tcp", net.JoinHostPort("localhost", strconv.Itoa(port)))
+}
+
+// devServerLookup is swapped in tests.
+var devServerLookup = lookupBrowserEndpoint
+
+func handleDevServerHTTP(ctx context.Context, runID string, req *domain.BrowserHTTPRequest) (domain.BrowserState, error) {
+ if req == nil || req.Port <= 0 || req.Port > 65535 || !strings.HasPrefix(req.Path, "/") {
+ return domain.BrowserState{}, errors.New("无效的开发服务器请求")
+ }
+ client, endpoint, ok := devServerClientFor(ctx, runID)
+ if !ok {
+ return domain.BrowserState{Available: false, Reason: "Agent 浏览器尚未启动:开发服务器预览需要它所在的沙箱网络"}, nil
+ }
+ method := strings.ToUpper(strings.TrimSpace(req.Method))
+ if method == "" {
+ method = http.MethodGet
+ }
+ target := "http://localhost:" + strconv.Itoa(req.Port) + req.Path
+ out, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(req.Body))
+ if err != nil {
+ return domain.BrowserState{}, err
+ }
+ for name, values := range req.Header {
+ if devServerHopHeader(name) || strings.EqualFold(name, "Host") {
+ continue
+ }
+ for _, v := range values {
+ out.Header.Add(name, v)
+ }
+ }
+ // Dev servers check Host against their allow list (Vite allowedHosts).
+ out.Host = "localhost:" + strconv.Itoa(req.Port)
+ res, err := client.Do(out)
+ if err != nil {
+ if errors.Is(err, ErrDevServerUnreachable) {
+ forgetDevServerEndpoint(runID, endpoint)
+ }
+ var unreachable devServerDialError
+ if errors.As(err, &unreachable) {
+ if devServerRefused(unreachable.err) {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("端口 %d 上没有正在监听的开发服务器", req.Port)}, nil
+ }
+ log.Printf("browser dev server dial run=%s kind=%s container=%s port=%d: %v", runID, endpoint.Kind, endpoint.Container, req.Port, unreachable.err)
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("连接端口 %d 失败:%v", req.Port, unreachable.err)}, nil
+ }
+ if ctx.Err() != nil {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("端口 %d 的开发服务器响应超时", req.Port)}, nil
+ }
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "访问开发服务器失败:" + err.Error()}, nil
+ }
+ defer res.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(res.Body, DevServerBodyLimit+1))
+ if err != nil {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "读取开发服务器响应失败:" + err.Error()}, nil
+ }
+ if len(body) > DevServerBodyLimit {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("响应超过 %d MB,预览暂不支持", DevServerBodyLimit>>20)}, nil
+ }
+ header := map[string][]string{}
+ for name, values := range res.Header {
+ if devServerHopHeader(name) || strings.EqualFold(name, "Content-Length") {
+ continue
+ }
+ header[name] = values
+ }
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{Status: res.StatusCode, Header: header, Body: body}}, nil
+}
+
+var devServerHop = map[string]bool{
+ "connection": true, "keep-alive": true, "proxy-connection": true, "proxy-authenticate": true,
+ "proxy-authorization": true, "te": true, "trailer": true, "transfer-encoding": true, "upgrade": true,
+}
+
+func devServerHopHeader(name string) bool { return devServerHop[strings.ToLower(name)] }
+
+func devServerClientFor(ctx context.Context, runID string) (*http.Client, browserEndpoint, bool) {
+ now := time.Now()
+ devServerClients.Lock()
+ for id, c := range devServerClients.items {
+ if now.Sub(c.used) > devServerClientIdle {
+ c.client.CloseIdleConnections()
+ delete(devServerClients.items, id)
+ }
+ }
+ cached := devServerClients.items[runID]
+ if cached != nil && now.Sub(cached.checked) < devServerEndpointTTL {
+ cached.used = now
+ devServerClients.Unlock()
+ return cached.client, cached.endpoint, true
+ }
+ devServerClients.Unlock()
+ endpoint, ok := devServerLookup(ctx, runID)
+ if !ok {
+ return nil, browserEndpoint{}, false
+ }
+ devServerClients.Lock()
+ defer devServerClients.Unlock()
+ cached = devServerClients.items[runID]
+ if cached != nil && cached.endpoint == endpoint {
+ cached.checked, cached.used = now, now
+ return cached.client, endpoint, true
+ }
+ if cached != nil {
+ cached.client.CloseIdleConnections()
+ }
+ transport := &http.Transport{
+ DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {
+ _, rawPort, err := net.SplitHostPort(addr)
+ if err != nil {
+ return nil, err
+ }
+ port, err := strconv.Atoi(rawPort)
+ if err != nil {
+ return nil, err
+ }
+ conn, err := devServerDial(ctx, endpoint, port)
+ if err != nil {
+ return nil, devServerDialError{err}
+ }
+ return conn, nil
+ },
+ // Each docker connection is a process; keep a few warm per run so a
+ // page load does not pay a docker exec per request.
+ MaxIdleConnsPerHost: 8,
+ IdleConnTimeout: 90 * time.Second,
+ ResponseHeaderTimeout: 2 * time.Minute,
+ ForceAttemptHTTP2: false,
+ }
+ client := &http.Client{
+ Transport: transport,
+ CheckRedirect: func(*http.Request, []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ }
+ devServerClients.items[runID] = &devServerClient{endpoint: endpoint, checked: now, used: now, client: client}
+ return client, endpoint, true
+}
+
+func forgetDevServerEndpoint(runID string, endpoint browserEndpoint) {
+ devServerClients.Lock()
+ defer devServerClients.Unlock()
+ if c := devServerClients.items[runID]; c != nil && c.endpoint == endpoint {
+ c.checked = time.Time{}
+ }
+}
+
+// devServerRefused tells a port nobody listens on from a relay that could
+// not reach the sandbox at all: only refusals (and a loopback family the
+// sandbox lacks) mean the dev server is not up.
+func devServerRefused(err error) bool {
+ if errors.Is(err, syscall.ECONNREFUSED) {
+ return true
+ }
+ refused := false
+ for _, code := range strings.Split(strings.TrimSpace(err.Error()), ",") {
+ switch strings.TrimSpace(code) {
+ case "ECONNREFUSED":
+ refused = true
+ case "EADDRNOTAVAIL", "ENETUNREACH", "EAFNOSUPPORT":
+ default:
+ return false
+ }
+ }
+ return refused
+}
+
+type devServerDialError struct{ err error }
+
+func (e devServerDialError) Error() string { return e.err.Error() }
+func (e devServerDialError) Unwrap() error { return ErrDevServerUnreachable }
+
+// devServerPipeScript connects stdio to localhost:port inside the sidecar.
+// It prints one 0x01 byte once connected, so a refused port fails the dial
+// instead of surfacing later as an empty response. Vite and friends may bind
+// either loopback family, so ::1 is tried after 127.0.0.1; a failure
+// reports each attempt's error code, comma separated.
+const devServerPipeScript = `const net=require("net");const port=+process.argv[1];const hosts=["127.0.0.1","::1"];const codes=[];` +
+ `(function go(){const s=net.connect({port,host:hosts.shift()});let up=false;` +
+ `s.once("connect",()=>{up=true;process.stdout.write("\x01");process.stdin.pipe(s);s.on("data",d=>process.stdout.write(d));s.on("close",()=>process.stdout.write("",()=>process.exit(0)));});` +
+ `s.on("error",e=>{if(up)return;codes.push(String(e.code||e));if(hosts.length)return go();process.stderr.write(codes.join(","));process.exit(2);});})();`
+
+func dialDockerPort(ctx context.Context, container string, port int) (net.Conn, error) {
+ return dialExecPort(ctx, "docker", "exec", "-i", container, "node", "-e", devServerPipeScript, strconv.Itoa(port))
+}
+
+// dialExecPort runs a command that relays its stdio to a TCP port.
+func dialExecPort(ctx context.Context, name string, args ...string) (net.Conn, error) {
+ cmd := exec.Command(name, args...)
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ return nil, err
+ }
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return nil, err
+ }
+ var stderr bytes.Buffer
+ cmd.Stderr = &stderr
+ if err := cmd.Start(); err != nil {
+ return nil, err
+ }
+ conn := &execConn{cmd: cmd, stdin: stdin, stdout: stdout}
+ ready := make(chan error, 1)
+ go func() {
+ var b [1]byte
+ _, err := io.ReadFull(stdout, b[:])
+ if err == nil && b[0] != 1 {
+ err = errors.New("unexpected handshake")
+ }
+ ready <- err
+ }()
+ select {
+ case err := <-ready:
+ if err != nil {
+ _ = conn.Close()
+ if msg := strings.TrimSpace(stderr.String()); msg != "" {
+ return nil, errors.New(msg)
+ }
+ return nil, err
+ }
+ return conn, nil
+ case <-ctx.Done():
+ _ = conn.Close()
+ return nil, ctx.Err()
+ case <-time.After(15 * time.Second):
+ _ = conn.Close()
+ return nil, errors.New("连接开发服务器超时")
+ }
+}
+
+// execConn is a net.Conn over a child process's stdio.
+type execConn struct {
+ cmd *exec.Cmd
+ stdin io.WriteCloser
+ stdout io.ReadCloser
+ once sync.Once
+}
+
+func (c *execConn) Read(p []byte) (int, error) { return c.stdout.Read(p) }
+func (c *execConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
+func (c *execConn) Close() error {
+ c.once.Do(func() {
+ _ = c.stdin.Close()
+ if c.cmd.Process != nil {
+ _ = c.cmd.Process.Kill()
+ }
+ go func() { _ = c.cmd.Wait() }()
+ })
+ return nil
+}
+func (c *execConn) LocalAddr() net.Addr { return execAddr{} }
+func (c *execConn) RemoteAddr() net.Addr { return execAddr{} }
+func (c *execConn) SetDeadline(time.Time) error { return nil }
+func (c *execConn) SetReadDeadline(time.Time) error { return nil }
+func (c *execConn) SetWriteDeadline(time.Time) error { return nil }
+
+type execAddr struct{}
+
+func (execAddr) Network() string { return "exec" }
+func (execAddr) String() string { return "docker-exec" }
diff --git a/internal/runtime/browser_devserver_test.go b/internal/runtime/browser_devserver_test.go
new file mode 100644
index 00000000..f8b99f07
--- /dev/null
+++ b/internal/runtime/browser_devserver_test.go
@@ -0,0 +1,122 @@
+package runtime
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os/exec"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+
+ "agentflow/internal/domain"
+)
+
+func TestDevServerPipeScriptRelaysHTTP(t *testing.T) {
+ if _, err := exec.LookPath("node"); err != nil {
+ t.Skip("node not installed")
+ }
+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.WriteString(w, "hello "+r.URL.Path)
+ }))
+ defer dev.Close()
+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))
+ conn, err := dialExecPort(context.Background(), "node", "-e", devServerPipeScript, rawPort)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+ for i := 0; i < 2; i++ { // keep-alive: two requests on one relay
+ req, _ := http.NewRequest("GET", "http://localhost:"+rawPort+"/p"+strconv.Itoa(i), nil)
+ if err := req.Write(conn); err != nil {
+ t.Fatal(err)
+ }
+ res, err := http.ReadResponse(bufio.NewReader(conn), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, _ := io.ReadAll(res.Body)
+ if string(body) != "hello /p"+strconv.Itoa(i) {
+ t.Fatalf("body %q", body)
+ }
+ }
+
+ closed, _ := net.Listen("tcp", "127.0.0.1:0")
+ _, idle, _ := net.SplitHostPort(closed.Addr().String())
+ closed.Close()
+ if _, err := dialExecPort(context.Background(), "node", "-e", devServerPipeScript, idle); err == nil || !strings.Contains(err.Error(), "ECONNREFUSED") {
+ t.Fatalf("refused port: %v", err)
+ }
+}
+
+func TestHandleDevServerHTTPThroughEndpoint(t *testing.T) {
+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain")
+ w.Header().Set("Connection", "keep-alive")
+ _, _ = io.WriteString(w, r.Method+" "+r.URL.RequestURI()+" "+r.Host)
+ }))
+ defer dev.Close()
+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))
+ port, _ := strconv.Atoi(rawPort)
+ lookup := devServerLookup
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{Kind: "local"}, true }
+ t.Cleanup(func() { devServerLookup = lookup })
+
+ state, err := HandleBrowser(context.Background(), "dev-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Method: "POST", Path: "/a?b=1"}})
+ if err != nil || state.HTTP == nil {
+ t.Fatalf("state=%+v err=%v", state, err)
+ }
+ if got := string(state.HTTP.Body); got != "POST /a?b=1 localhost:"+rawPort {
+ t.Fatalf("body %q", got)
+ }
+ if _, hop := state.HTTP.Header["Connection"]; hop {
+ t.Fatal("hop-by-hop header leaked")
+ }
+ if _, err := HandleBrowser(context.Background(), "dev-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Path: "no-slash"}}); err == nil {
+ t.Fatal("invalid path accepted")
+ }
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{}, false }
+ devServerClients.Lock()
+ delete(devServerClients.items, "other-run")
+ devServerClients.Unlock()
+ if state, _ := HandleBrowser(context.Background(), "other-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Path: "/"}}); state.Available || state.HTTP != nil {
+ t.Fatalf("missing browser: %+v", state)
+ }
+}
+
+func TestDevServerDialFailuresAreNotReportedAsIdlePorts(t *testing.T) {
+ for raw, want := range map[string]bool{
+ "ECONNREFUSED,ECONNREFUSED": true,
+ "ECONNREFUSED,EADDRNOTAVAIL": true,
+ "EADDRNOTAVAIL,ENETUNREACH": false,
+ "ETIMEDOUT,ECONNREFUSED": false,
+ "exit status 126": false,
+ "连接开发服务器超时": false,
+ } {
+ if got := devServerRefused(errors.New(raw)); got != want {
+ t.Fatalf("%q: %v", raw, got)
+ }
+ }
+ if !devServerRefused(fmt.Errorf("dial: %w", syscall.ECONNREFUSED)) {
+ t.Fatal("host refusal")
+ }
+
+ lookup, dial := devServerLookup, devServerDial
+ t.Cleanup(func() { devServerLookup, devServerDial = lookup, dial })
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {
+ return browserEndpoint{Kind: "docker", Container: "dial-sidecar"}, true
+ }
+ devServerDial = func(context.Context, browserEndpoint, int) (net.Conn, error) {
+ return nil, errors.New(`OCI runtime exec failed: exec: "node": executable file not found`)
+ }
+ state, err := HandleBrowser(context.Background(), "dial-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: 8000, Path: "/"}})
+ if err != nil || state.HTTP != nil || !strings.Contains(state.Reason, "连接端口 8000 失败") || !strings.Contains(state.Reason, "executable file not found") {
+ t.Fatalf("state=%+v err=%v", state, err)
+ }
+}
diff --git a/internal/runtime/browser_file.go b/internal/runtime/browser_file.go
new file mode 100644
index 00000000..6a70cecb
--- /dev/null
+++ b/internal/runtime/browser_file.go
@@ -0,0 +1,165 @@
+package runtime
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "mime"
+ "net/http"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "agentflow/internal/domain"
+)
+
+// Workspace file access: pages the Agent opens as file:// are served to the
+// user's own browser so they render natively, like dev servers. Paths are
+// the ones the run browser sees (/workspace/... in a docker sidecar, the host
+// workspace for a host browser) and must resolve inside that workspace.
+
+// dockerWorkspaceRoot is where the browser sidecar mounts the workspace.
+const dockerWorkspaceRoot = "/workspace"
+
+// Outcomes shared by the sidecar script (exit codes) and the host reader.
+const (
+ workspaceFileOK = 0
+ workspaceFileMissing = 44
+ workspaceFileOutside = 45
+ workspaceFileDirectory = 46 // a directory asked for without a trailing slash
+ workspaceFileTooLarge = 47
+ workspaceFileNoIndexHTML = 48
+)
+
+// workspaceFileScript prints a workspace file inside the sidecar:
+// node -e script ROOT PATH LIMIT. Symlinks are resolved before the
+// containment check, and a directory serves its index.html.
+const workspaceFileScript = `const fs=require("fs"),path=require("path");const[root0,p,limit]=process.argv.slice(1);` +
+ `const real=f=>{try{return fs.realpathSync(f)}catch(e){process.exit(44)}};const root=real(root0);` +
+ `const inside=r=>r===root||r.startsWith(root.endsWith("/")?root:root+"/");` +
+ `let r=real(p);if(!inside(r))process.exit(45);let st=fs.statSync(r);` +
+ `if(st.isDirectory()){if(!p.endsWith("/"))process.exit(46);const i=path.join(r,"index.html");if(!fs.existsSync(i))process.exit(48);` +
+ `r=real(i);if(!inside(r))process.exit(45);st=fs.statSync(r);}` +
+ `if(!st.isFile())process.exit(44);if(st.size>+limit)process.exit(47);process.stdout.write(fs.readFileSync(r),()=>process.exit(0));`
+
+// workspaceFileExec runs the script in a sidecar; swapped in tests.
+var workspaceFileExec = func(ctx context.Context, container, file string) ([]byte, int, error) {
+ out, err := exec.CommandContext(ctx, "docker", "exec", container, "node", "-e", workspaceFileScript,
+ dockerWorkspaceRoot, file, strconv.Itoa(DevServerBodyLimit)).Output()
+ var exit *exec.ExitError
+ if errors.As(err, &exit) && exit.ExitCode() >= workspaceFileMissing && exit.ExitCode() <= workspaceFileNoIndexHTML {
+ return nil, exit.ExitCode(), nil
+ }
+ if err != nil {
+ return nil, 0, err
+ }
+ return out, workspaceFileOK, nil
+}
+
+func handleWorkspaceFile(ctx context.Context, runID, file string) (domain.BrowserState, error) {
+ if !strings.HasPrefix(file, "/") || strings.ContainsRune(file, 0) {
+ return domain.BrowserState{}, errors.New("无效的工作区文件路径")
+ }
+ endpoint, ok := devServerLookup(ctx, runID)
+ if !ok {
+ return domain.BrowserState{Available: false, Reason: "Agent 浏览器尚未启动:工作区文件预览需要它所在的执行环境"}, nil
+ }
+ var (
+ body []byte
+ code int
+ err error
+ )
+ if endpoint.Kind == "docker" {
+ body, code, err = workspaceFileExec(ctx, endpoint.Container, file)
+ } else if endpoint.Workspace == "" {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "无法确定 Agent 浏览器的工作区目录"}, nil
+ } else {
+ body, code = serveWorkspaceFile(endpoint.Workspace, file, DevServerBodyLimit)
+ }
+ if err != nil {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "读取工作区文件失败:" + err.Error()}, nil
+ }
+ text := func(status int, message string) (domain.BrowserState, error) {
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{
+ Status: status, Header: map[string][]string{"Content-Type": {"text/plain; charset=utf-8"}}, Body: []byte(message),
+ }}, nil
+ }
+ switch code {
+ case workspaceFileOK:
+ case workspaceFileMissing:
+ return text(http.StatusNotFound, "文件不存在:"+file)
+ case workspaceFileOutside:
+ return text(http.StatusForbidden, "只能预览工作区内的文件")
+ case workspaceFileDirectory:
+ // The gateway adds the slash, so relative links resolve in the directory.
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{Status: http.StatusMovedPermanently}}, nil
+ case workspaceFileNoIndexHTML:
+ return text(http.StatusNotFound, "目录下没有 index.html:"+file)
+ case workspaceFileTooLarge:
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("文件超过 %d MB,预览暂不支持", DevServerBodyLimit>>20)}, nil
+ }
+ name := file
+ if strings.HasSuffix(name, "/") {
+ name += "index.html"
+ }
+ contentType := mime.TypeByExtension(path.Ext(name))
+ if contentType == "" {
+ contentType = http.DetectContentType(body)
+ }
+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{
+ Status: http.StatusOK, Header: map[string][]string{"Content-Type": {contentType}}, Body: body,
+ }}, nil
+}
+
+// serveWorkspaceFile is the host browser's counterpart of workspaceFileScript.
+func serveWorkspaceFile(root, file string, limit int64) ([]byte, int) {
+ realRoot, err := filepath.EvalSymlinks(root)
+ if err != nil {
+ return nil, workspaceFileMissing
+ }
+ inside := func(p string) bool {
+ rel, err := filepath.Rel(realRoot, p)
+ return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
+ }
+ real, err := filepath.EvalSymlinks(filepath.FromSlash(file))
+ if err != nil {
+ return nil, workspaceFileMissing
+ }
+ if !inside(real) {
+ return nil, workspaceFileOutside
+ }
+ info, err := os.Stat(real)
+ if err != nil {
+ return nil, workspaceFileMissing
+ }
+ if info.IsDir() {
+ if !strings.HasSuffix(file, "/") {
+ return nil, workspaceFileDirectory
+ }
+ index, err := filepath.EvalSymlinks(filepath.Join(real, "index.html"))
+ if err != nil {
+ return nil, workspaceFileNoIndexHTML
+ }
+ if !inside(index) {
+ return nil, workspaceFileOutside
+ }
+ if info, err = os.Stat(index); err != nil {
+ return nil, workspaceFileNoIndexHTML
+ }
+ real = index
+ }
+ if !info.Mode().IsRegular() {
+ return nil, workspaceFileMissing
+ }
+ if info.Size() > limit {
+ return nil, workspaceFileTooLarge
+ }
+ body, err := os.ReadFile(real)
+ if err != nil {
+ return nil, workspaceFileMissing
+ }
+ return body, workspaceFileOK
+}
diff --git a/internal/runtime/browser_file_test.go b/internal/runtime/browser_file_test.go
new file mode 100644
index 00000000..ea034199
--- /dev/null
+++ b/internal/runtime/browser_file_test.go
@@ -0,0 +1,151 @@
+package runtime
+
+import (
+ "context"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+
+ "agentflow/internal/domain"
+)
+
+// workspaceFileTree lays out a workspace plus a file outside it.
+func workspaceFileTree(t *testing.T) string {
+ t.Helper()
+ base := t.TempDir()
+ root := filepath.Join(base, "ws")
+ for name, body := range map[string]string{
+ "ws/demo.html": "<h1>demo</h1>",
+ "ws/site/index.html": "<p>site</p>",
+ "ws/site/app.js": "console.log(1)",
+ "ws/empty/.keep": "",
+ "ws/big.txt": strings.Repeat("x", 64),
+ "secret.txt": "secret",
+ } {
+ p := filepath.Join(base, filepath.FromSlash(name))
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.Symlink(filepath.Join(base, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil {
+ t.Fatal(err)
+ }
+ return root
+}
+
+var workspaceFileCases = []struct {
+ path string
+ code int
+ body string
+}{
+ {"/demo.html", workspaceFileOK, "<h1>demo</h1>"},
+ {"/site/", workspaceFileOK, "<p>site</p>"},
+ {"/site/app.js", workspaceFileOK, "console.log(1)"},
+ {"/site", workspaceFileDirectory, ""},
+ {"/empty/", workspaceFileNoIndexHTML, ""},
+ {"/missing.html", workspaceFileMissing, ""},
+ {"/leak.txt", workspaceFileOutside, ""},
+ {"/../secret.txt", workspaceFileOutside, ""},
+ {"/big.txt", workspaceFileTooLarge, ""},
+}
+
+func TestServeWorkspaceFileStaysInTheWorkspace(t *testing.T) {
+ root := workspaceFileTree(t)
+ for _, tc := range workspaceFileCases {
+ body, code := serveWorkspaceFile(root, root+tc.path, 32)
+ if code != tc.code || string(body) != tc.body {
+ t.Fatalf("%s: code %d body %q", tc.path, code, body)
+ }
+ }
+}
+
+// The sidecar script must agree with the host reader on every case.
+func TestWorkspaceFileScriptMatchesHostReader(t *testing.T) {
+ if _, err := exec.LookPath("node"); err != nil {
+ t.Skip("node not installed")
+ }
+ root := workspaceFileTree(t)
+ for _, tc := range workspaceFileCases {
+ out, err := exec.Command("node", "-e", workspaceFileScript, root, root+tc.path, "32").Output()
+ code := 0
+ var exit *exec.ExitError
+ if errors.As(err, &exit) {
+ code = exit.ExitCode()
+ } else if err != nil {
+ t.Fatal(err)
+ }
+ if code != tc.code || (code == 0 && string(out) != tc.body) {
+ t.Fatalf("%s: code %d body %q", tc.path, code, out)
+ }
+ }
+}
+
+func TestHandleWorkspaceFileResponses(t *testing.T) {
+ root := workspaceFileTree(t)
+ lookup, execFile := devServerLookup, workspaceFileExec
+ t.Cleanup(func() { devServerLookup, workspaceFileExec = lookup, execFile })
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {
+ return browserEndpoint{Kind: "docker", Container: "sidecar"}, true
+ }
+ var seen []string
+ workspaceFileExec = func(_ context.Context, container, file string) ([]byte, int, error) {
+ seen = append(seen, container+" "+file)
+ body, code := serveWorkspaceFile(root, root+strings.TrimPrefix(file, dockerWorkspaceRoot), 32)
+ return body, code, nil
+ }
+ get := func(file string) domain.BrowserState {
+ t.Helper()
+ state, err := HandleBrowser(context.Background(), "file-run", domain.BrowserRequest{Operation: "file", File: file})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return state
+ }
+ for _, tc := range []struct {
+ file, contentType string
+ status int
+ }{
+ {"/workspace/demo.html", "text/html; charset=utf-8", 200},
+ {"/workspace/site/", "text/html; charset=utf-8", 200},
+ {"/workspace/site/app.js", "text/javascript; charset=utf-8", 200},
+ {"/workspace/site", "", 301},
+ {"/workspace/missing.html", "text/plain; charset=utf-8", 404},
+ {"/workspace/leak.txt", "text/plain; charset=utf-8", 403},
+ } {
+ state := get(tc.file)
+ if state.HTTP == nil || state.HTTP.Status != tc.status {
+ t.Fatalf("%s: %+v", tc.file, state)
+ }
+ if got := strings.Join(state.HTTP.Header["Content-Type"], ""); got != tc.contentType {
+ t.Fatalf("%s: content type %q", tc.file, got)
+ }
+ }
+ if state := get("/workspace/big.txt"); state.HTTP != nil || !strings.Contains(state.Reason, strconv.Itoa(DevServerBodyLimit>>20)+" MB") {
+ t.Fatalf("too large: %+v", state)
+ }
+ if seen[0] != "sidecar /workspace/demo.html" {
+ t.Fatalf("exec: %v", seen)
+ }
+ if _, err := HandleBrowser(context.Background(), "file-run", domain.BrowserRequest{Operation: "file", File: "relative.html"}); err == nil {
+ t.Fatal("relative path accepted")
+ }
+
+ // A host browser reads its own workspace directly.
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {
+ return browserEndpoint{Kind: "local", Workspace: root}, true
+ }
+ if state := get(root + "/site/"); state.HTTP == nil || string(state.HTTP.Body) != "<p>site</p>" {
+ t.Fatalf("host: %+v", state)
+ }
+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{}, false }
+ if state := get("/workspace/demo.html"); state.Available || state.HTTP != nil {
+ t.Fatalf("no browser: %+v", state)
+ }
+}
diff --git a/internal/runtime/browser_inspect.go b/internal/runtime/browser_inspect.go
new file mode 100644
index 00000000..8c888355
--- /dev/null
+++ b/internal/runtime/browser_inspect.go
@@ -0,0 +1,161 @@
+package runtime
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "agentflow/internal/domain"
+)
+
+// Element inspection for the panel's pick mode on the Agent browser
+// screencast: the page reports the element under a point, its box for the
+// outline and, on a pick, the description the composer sends to the Agent.
+// The description follows chatbot/src/element-context.js, which does the same
+// for native previews in the user's own browser; keep the two in step.
+
+const inspectScript = `(function (fx, fy, detail) {
+ var doc = document, x = fx * innerWidth, y = fy * innerHeight, ox = 0, oy = 0, el = null;
+ for (var depth = 0; depth < 8; depth++) {
+ var hit = doc.elementFromPoint(x, y);
+ while (hit && hit.shadowRoot) {
+ var inner = hit.shadowRoot.elementFromPoint(x, y);
+ if (!inner || inner === hit) break;
+ hit = inner;
+ }
+ if (!hit) break;
+ el = hit;
+ if (hit.tagName !== "IFRAME" && hit.tagName !== "FRAME") break;
+ var child = null;
+ try { child = hit.contentDocument; } catch (e) {}
+ if (!child || !child.documentElement) break;
+ var box = hit.getBoundingClientRect();
+ var dx = box.left + hit.clientLeft, dy = box.top + hit.clientTop;
+ x -= dx; y -= dy; ox += dx; oy += dy; doc = child;
+ }
+ if (!el || el === document.documentElement) return null;
+ var clip = function (value, limit) {
+ var text = String(value || "").replace(/\s+/g, " ").trim();
+ return text.length > limit ? text.slice(0, limit) + "…" : text;
+ };
+ var esc = function (v) { return window.CSS && CSS.escape ? CSS.escape(v) : String(v).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); };
+ var classes = function (node) {
+ return Array.prototype.filter.call(node.classList || [], function (c) {
+ return !/^(css|sc|jsx|svelte|emotion)-|^[a-z]{1,3}[-_]?(?=[0-9a-z]*[0-9])[0-9a-z]{5,}$/i.test(c) && c.length <= 40;
+ }).slice(0, 2);
+ };
+ var label = function (node) {
+ var tag = node.tagName.toLowerCase(), id = node.id ? "#" + node.id : "";
+ var cls = !id && classes(node)[0] ? "." + classes(node)[0] : "";
+ var text = clip(node.innerText != null ? node.innerText : node.textContent, 16);
+ return tag + id + cls + (text ? " 「" + text + "」" : "");
+ };
+ var rect = el.getBoundingClientRect();
+ var out = {
+ label: label(el),
+ rect: { x: Math.round(rect.left + ox), y: Math.round(rect.top + oy), width: Math.round(rect.width), height: Math.round(rect.height) },
+ viewport: { width: innerWidth, height: innerHeight }
+ };
+ if (!detail) return out;
+ var owner = el.ownerDocument;
+ var unique = function (selector) {
+ try { var found = owner.querySelectorAll(selector); return found.length === 1 && found[0] === el; } catch (e) { return false; }
+ };
+ var step = function (node) {
+ var tag = node.tagName.toLowerCase();
+ var testId = node.getAttribute("data-testid") || node.getAttribute("data-test-id");
+ if (testId) return tag + '[data-testid="' + testId.replace(/"/g, '\\"') + '"]';
+ return tag + classes(node).map(function (c) { return "." + esc(c); }).join("");
+ };
+ var nth = function (node, base) {
+ var parent = node.parentElement;
+ if (!parent) return base;
+ var same = Array.prototype.filter.call(parent.children, function (c) { return c.tagName === node.tagName; });
+ return same.length > 1 ? base + ":nth-of-type(" + (same.indexOf(node) + 1) + ")" : base;
+ };
+ var selector = function () {
+ if (el.id && unique("#" + esc(el.id))) return "#" + esc(el.id);
+ var parts = [], node = el;
+ while (node && node.nodeType === 1 && node !== owner.documentElement) {
+ if (node !== el && node.id && owner.querySelectorAll("#" + esc(node.id)).length === 1) { parts.unshift("#" + esc(node.id)); break; }
+ parts.unshift(step(node));
+ if (unique(parts.join(" > "))) return parts.join(" > ");
+ parts[0] = nth(node, parts[0]);
+ if (unique(parts.join(" > "))) return parts.join(" > ");
+ node = node.parentElement;
+ }
+ return parts.join(" > ");
+ };
+ var source = function (file, line, column) {
+ return file ? file + (line ? ":" + line : "") + (line && column ? ":" + column : "") : "";
+ };
+ var component = function () {
+ for (var node = el, i = 0; node && node.nodeType === 1 && i < 25; node = node.parentElement, i++) {
+ var key = Object.keys(node).find(function (k) { return k.indexOf("__reactFiber$") === 0 || k.indexOf("__reactInternalInstance$") === 0; });
+ if (key) {
+ var fiber = node[key], src = fiber && fiber._debugSource;
+ for (var j = 0; fiber && j < 40; j++, fiber = fiber.return) {
+ src = src || fiber._debugSource;
+ var t = fiber.type;
+ if (typeof t === "function" || (t && typeof t === "object" && (t.render || t.type))) {
+ var name = t.displayName || t.name || (t.render && (t.render.displayName || t.render.name)) || (t.type && t.type.name);
+ if (name) return { framework: "React", component: name, source: src ? source(src.fileName, src.lineNumber, src.columnNumber) : "" };
+ }
+ }
+ }
+ var vue = node.__vueParentComponent;
+ if (vue && vue.type) {
+ var file = vue.type.__file || "";
+ return { framework: "Vue", component: vue.type.name || vue.type.__name || file.split("/").pop().replace(/\.vue$/, ""), source: file };
+ }
+ if (node.__vue__ && node.__vue__.$options) {
+ var o = node.__vue__.$options;
+ return { framework: "Vue", component: o.name || o._componentTag || "", source: o.__file || "" };
+ }
+ var loc = node.__svelte_meta && node.__svelte_meta.loc;
+ if (loc && loc.file) {
+ return { framework: "Svelte", component: loc.file.split("/").pop().replace(/\.svelte$/, ""), source: source(loc.file, loc.line != null ? loc.line + 1 : 0, loc.column != null ? loc.column + 1 : 0) };
+ }
+ }
+ return null;
+ };
+ var computed = owner.defaultView.getComputedStyle(el), styles = {};
+ var layout = /flex|grid/.test(computed.getPropertyValue("display"));
+ var defaults = { position: "static", opacity: "1", "font-weight": "400" };
+ ["display", "position", "width", "height", "margin", "padding", "gap", "flex-direction", "justify-content", "align-items",
+ "color", "background-color", "font-size", "font-weight", "line-height", "border", "border-radius", "box-shadow", "opacity"].forEach(function (k) {
+ var v = computed.getPropertyValue(k);
+ if (!layout && ["gap", "flex-direction", "justify-content", "align-items"].indexOf(k) >= 0) return;
+ if (!v || /^(none|normal|auto|0px|0px none.*|rgba\(0, 0, 0, 0\))$/.test(v) || defaults[k] === v) return;
+ styles[k] = v;
+ });
+ // Snapshot refs (data-agentflow-ref) mean nothing outside their snapshot.
+ var html = (el.outerHTML || "").replace(/ data-agentflow-ref="[^"]*"/g, ""), found = component();
+ out.element = {
+ url: location.href,
+ title: clip(document.title, 120),
+ selector: selector(),
+ tag: el.tagName.toLowerCase(),
+ text: clip(el.innerText != null ? el.innerText : el.textContent, 200),
+ html: html.length > 3000 ? html.slice(0, 3000) + "…" : html,
+ rect: out.rect,
+ viewport: out.viewport,
+ styles: styles
+ };
+ if (found && found.component) { out.element.component = found.component; out.element.framework = found.framework; }
+ if (found && found.source) out.element.source = found.source;
+ return out;
+})`
+
+// inspect reports the element at point (a fraction of the viewport).
+func (v *browserView) inspect(ctx context.Context, point *domain.BrowserPoint, detail bool) (*domain.BrowserInspection, error) {
+ if point == nil {
+ return nil, errors.New("缺少检查位置")
+ }
+ var out *domain.BrowserInspection
+ expression := fmt.Sprintf("%s(%g, %g, %t)", inspectScript, clampUnit(point.X), clampUnit(point.Y), detail)
+ if err := v.evaluate(ctx, expression, &out); err != nil {
+ return nil, err
+ }
+ return out, nil
+}
diff --git a/internal/runtime/browser_inspect_test.go b/internal/runtime/browser_inspect_test.go
new file mode 100644
index 00000000..98c09f4f
--- /dev/null
+++ b/internal/runtime/browser_inspect_test.go
@@ -0,0 +1,50 @@
+package runtime
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "agentflow/internal/domain"
+)
+
+func TestInspectReportsTheElementAtAPoint(t *testing.T) {
+ transport := newFakeCDPTransport()
+ transport.results = map[string]string{
+ "Runtime.evaluate": `{"result":{"value":{"label":"button.btn 「保存」","rect":{"x":10,"y":20,"width":55,"height":35},"viewport":{"width":1280,"height":800},
+ "element":{"url":"https://example.com/","selector":"#save","tag":"button","source":"src/Save.tsx:3"}}}}`,
+ }
+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}
+ view.client = newCDPClient(transport, view.onEvent)
+ defer view.client.close()
+
+ if got, err := view.inspect(context.Background(), nil, true); err == nil || got != nil {
+ t.Fatal("missing point accepted")
+ }
+ got, err := view.inspect(context.Background(), &domain.BrowserPoint{X: 0.25, Y: 1.7}, true)
+ if err != nil || got == nil {
+ t.Fatalf("got=%+v err=%v", got, err)
+ }
+ if got.Label != "button.btn 「保存」" || got.Rect.Width != 55 || got.Viewport.Width != 1280 || got.Element == nil || got.Element.Source != "src/Save.tsx:3" {
+ t.Fatalf("inspection %+v %+v", got, got.Element)
+ }
+ transport.mu.Lock()
+ last := transport.sent[len(transport.sent)-1]
+ transport.mu.Unlock()
+ var params struct {
+ Expression string `json:"expression"`
+ }
+ _ = json.Unmarshal(last.Params, &params)
+ // Points are clamped to the viewport like input events.
+ if !strings.HasSuffix(params.Expression, "})(0.25, 1, true)") {
+ t.Fatalf("expression ends %q", params.Expression[len(params.Expression)-40:])
+ }
+
+ transport.mu.Lock()
+ transport.results["Runtime.evaluate"] = `{"result":{"value":null}}`
+ transport.mu.Unlock()
+ if got, err := view.inspect(context.Background(), &domain.BrowserPoint{X: 0.5, Y: 0.5}, false); err != nil || got != nil {
+ t.Fatalf("empty point: %+v %v", got, err)
+ }
+}
diff --git a/internal/runtime/browser_motion_test.go b/internal/runtime/browser_motion_test.go
new file mode 100644
index 00000000..3b0f13ed
--- /dev/null
+++ b/internal/runtime/browser_motion_test.go
@@ -0,0 +1,95 @@
+package runtime
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "fmt"
+ "image/jpeg"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "agentflow/internal/domain"
+)
+
+// Chromium re-sends the current page when the screencast restarts, so a
+// static page still ends on a sharp frame after a scroll.
+func TestBrowserScrollEndsOnASharpFrame(t *testing.T) {
+ if testing.Short() {
+ t.Skip("short mode")
+ }
+ if _, err := FindChromium(); err != nil {
+ t.Skip("chromium not installed")
+ }
+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ fmt.Fprint(w, `<title>静态</title><h1>不会动的页面</h1>`)
+ }))
+ defer page.Close()
+ workspace := t.TempDir()
+ runID := "browser-motion-" + fmt.Sprint(time.Now().UnixNano())
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+ if _, err := ensureHostBrowser(ctx, workspace, runID); err != nil {
+ t.Fatal(err)
+ }
+ defer stopHostBrowsers(workspace)
+ if _, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "resize", FrameWidth: 800}); err != nil {
+ t.Fatal(err)
+ }
+ state, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "navigate", URL: page.URL})
+ if err != nil {
+ t.Fatal(err)
+ }
+ // nextWidth waits for a frame after seq and returns its pixel width.
+ nextWidth := func(after int64) (int64, int) {
+ t.Helper()
+ for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); {
+ st, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "frame", After: after, WaitMs: 2000})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if st.Seq > after && st.Frame != "" {
+ raw, _ := base64.StdEncoding.DecodeString(st.Frame)
+ config, err := jpeg.DecodeConfig(bytes.NewReader(raw))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return st.Seq, config.Width
+ }
+ }
+ t.Fatal("no frame")
+ return 0, 0
+ }
+ seq, width := nextWidth(state.Seq)
+ // Let the page finish painting at full width.
+ for width < 700 {
+ seq, width = nextWidth(seq)
+ }
+ if _, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "input", Events: []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}}}); err != nil {
+ t.Fatal(err)
+ }
+ // A full-width frame already in flight can arrive before the screencast
+ // restarts at the lighter size. Keep reading until a light frame shows up.
+ sawLight := false
+ for deadline := time.Now().Add(8 * time.Second); time.Now().Before(deadline); {
+ seq, width = nextWidth(seq)
+ if width <= 400 {
+ sawLight = true
+ break
+ }
+ }
+ if !sawLight {
+ t.Fatalf("no light frame while scrolling (last %dpx)", width)
+ }
+ // More light frames may follow while the scroll animates; once input
+ // settles a full-width frame arrives although nothing changed.
+ for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
+ if seq, width = nextWidth(seq); width >= 700 {
+ return
+ }
+ }
+ t.Fatalf("no sharp frame after the scroll settled (last %dpx)", width)
+}
diff --git a/internal/runtime/browser_test.go b/internal/runtime/browser_test.go
index e74a8c38..71b2676a 100644
--- a/internal/runtime/browser_test.go
+++ b/internal/runtime/browser_test.go
@@ -10,6 +10,7 @@ import (
"strings"
"sync"
"testing"
+ "time"
"agentflow/internal/domain"
)
@@ -84,6 +85,8 @@ type fakeCDPTransport struct {
replies chan []byte
closed chan struct{}
once sync.Once
+ // results answers methods with a canned result instead of {}.
+ results map[string]string
}
func newFakeCDPTransport() *fakeCDPTransport {
@@ -107,7 +110,11 @@ func (f *fakeCDPTransport) write(body []byte) error {
f.mu.Lock()
f.sent = append(f.sent, msg)
f.mu.Unlock()
- reply, _ := json.Marshal(cdpMessage{ID: msg.ID, Result: json.RawMessage(`{}`)})
+ result := `{}`
+ if r := f.results[msg.Method]; r != "" {
+ result = r
+ }
+ reply, _ := json.Marshal(cdpMessage{ID: msg.ID, Result: json.RawMessage(result)})
f.replies <- reply
return nil
}
@@ -134,20 +141,28 @@ func TestBrowserViewInputMapsToCDP(t *testing.T) {
}
transport.mu.Lock()
defer transport.mu.Unlock()
- if len(transport.sent) != 5 {
- t.Fatalf("sent %d messages: %+v", len(transport.sent), transport.sent)
+ // The wheel also switches the screencast to light frames; only input
+ // events are checked here.
+ var sent []cdpMessage
+ for _, msg := range transport.sent {
+ if strings.HasPrefix(msg.Method, "Input.") {
+ sent = append(sent, msg)
+ }
+ }
+ if len(sent) != 5 {
+ t.Fatalf("sent %d input messages: %+v", len(sent), transport.sent)
}
params := func(i int) map[string]any {
var out map[string]any
- _ = json.Unmarshal(transport.sent[i].Params, &out)
+ _ = json.Unmarshal(sent[i].Params, &out)
return out
}
- for _, msg := range transport.sent {
+ for _, msg := range sent {
if msg.SessionID != "page-1" {
t.Fatalf("input must target the attached page session: %+v", msg)
}
}
- if p := params(0); transport.sent[0].Method != "Input.dispatchMouseEvent" || p["type"] != "mousePressed" || p["x"] != 500.0 || p["y"] != 125.0 {
+ if p := params(0); sent[0].Method != "Input.dispatchMouseEvent" || p["type"] != "mousePressed" || p["x"] != 500.0 || p["y"] != 125.0 {
t.Fatalf("mouse: %s %v", transport.sent[0].Method, p)
}
if p := params(1); p["type"] != "mouseWheel" || p["x"] != 1000.0 || p["y"] != 0.0 || p["deltaY"] != 120.0 {
@@ -159,7 +174,139 @@ func TestBrowserViewInputMapsToCDP(t *testing.T) {
if p := params(3); p["type"] != "rawKeyDown" || p["text"] != nil {
t.Fatalf("control key: %v", p)
}
- if p := params(4); transport.sent[4].Method != "Input.insertText" || p["text"] != "你好" {
- t.Fatalf("text: %s %v", transport.sent[4].Method, p)
+ if p := params(4); sent[4].Method != "Input.insertText" || p["text"] != "你好" {
+ t.Fatalf("text: %s %v", sent[4].Method, p)
+ }
+}
+
+func TestBrowserViewReportsScrollAndContentSize(t *testing.T) {
+ transport := newFakeCDPTransport()
+ transport.results = map[string]string{"Page.getLayoutMetrics": `{"cssContentSize":{"width":1000,"height":3200}}`}
+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}
+ view.client = newCDPClient(transport, view.onEvent)
+ defer view.client.close()
+ frame, _ := json.Marshal(map[string]any{"data": "AAAA", "sessionId": 7, "metadata": map[string]any{"deviceWidth": 1000, "deviceHeight": 800, "scrollOffsetX": 0, "scrollOffsetY": 640}})
+ view.onEvent(cdpMessage{Method: "Page.screencastFrame", SessionID: "page-1", Params: frame})
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ state := view.state(0)
+ if state.ContentHeight == 3200 {
+ if state.ScrollY != 640 || state.Height != 800 || state.ContentWidth != 1000 {
+ t.Fatalf("state %+v", state)
+ }
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("content size never arrived: %+v", state)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ // A second frame within the refresh interval does not read metrics again.
+ view.onEvent(cdpMessage{Method: "Page.screencastFrame", SessionID: "page-1", Params: frame})
+ time.Sleep(50 * time.Millisecond)
+ transport.mu.Lock()
+ defer transport.mu.Unlock()
+ reads := 0
+ for _, msg := range transport.sent {
+ if msg.Method == "Page.getLayoutMetrics" {
+ reads++
+ }
+ }
+ if reads != 1 {
+ t.Fatalf("layout metric reads: %d", reads)
+ }
+}
+
+func TestBrowserDeviceScale(t *testing.T) {
+ for raw, want := range map[string]string{"": "2", "1.5": "1.5", "3": "3", "0.5": "2", "x": "2", "4": "2"} {
+ t.Setenv("AGENTFLOW_BROWSER_SCALE", raw)
+ if got := BrowserDeviceScale(); got != want {
+ t.Fatalf("%q: %s", raw, got)
+ }
+ }
+}
+
+func TestAgentScreenshotsStayInCSSPixels(t *testing.T) {
+ transport := newFakeCDPTransport()
+ transport.results = map[string]string{
+ "Page.getLayoutMetrics": `{"cssVisualViewport":{"pageX":0,"pageY":300,"clientWidth":1280,"clientHeight":800}}`,
+ "Runtime.evaluate": `{"result":{"value":2}}`,
+ "Page.captureScreenshot": `{"data":"AAAA"}`,
+ }
+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}
+ view.client = newCDPClient(transport, view.onEvent)
+ defer view.client.close()
+ if data, err := view.screenshot(context.Background()); err != nil || data != "AAAA" {
+ t.Fatalf("data=%q err=%v", data, err)
+ }
+ transport.mu.Lock()
+ defer transport.mu.Unlock()
+ last := transport.sent[len(transport.sent)-1]
+ var params struct {
+ Clip map[string]float64 `json:"clip"`
+ }
+ _ = json.Unmarshal(last.Params, &params)
+ if last.Method != "Page.captureScreenshot" || params.Clip["scale"] != 0.5 || params.Clip["y"] != 300 || params.Clip["width"] != 1280 {
+ t.Fatalf("%s %s", last.Method, last.Params)
+ }
+}
+
+func TestBrowserViewCapturesLighterFramesWhileScrolling(t *testing.T) {
+ settle := browserMotionSettle
+ browserMotionSettle = 60 * time.Millisecond
+ defer func() { browserMotionSettle = settle }()
+ transport := newFakeCDPTransport()
+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1", width: 1000, height: 500, frameWidth: 1600}
+ view.client = newCDPClient(transport, view.onEvent)
+ defer view.client.close()
+ starts := func() []map[string]any {
+ transport.mu.Lock()
+ defer transport.mu.Unlock()
+ var out []map[string]any
+ for _, msg := range transport.sent {
+ if msg.Method == "Page.startScreencast" {
+ var p map[string]any
+ _ = json.Unmarshal(msg.Params, &p)
+ out = append(out, p)
+ }
+ }
+ return out
+ }
+ wheel := []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}}
+ ctx := context.Background()
+
+ // Hovering and clicking keep sharp frames.
+ if err := view.input(ctx, []domain.BrowserInputEvent{{Type: "mouse", Action: "move", X: 0.5, Y: 0.5}, {Type: "mouse", Action: "down", X: 0.5, Y: 0.5, Buttons: 1}}); err != nil {
+ t.Fatal(err)
+ }
+ if got := starts(); len(got) != 0 {
+ t.Fatalf("hover restarted the screencast: %v", got)
+ }
+ // A scroll switches to light frames once, however many follow.
+ for range 3 {
+ if err := view.input(ctx, wheel); err != nil {
+ t.Fatal(err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ got := starts()
+ if len(got) != 1 || got[0]["maxWidth"] != 800.0 || got[0]["quality"] != 60.0 {
+ t.Fatalf("scroll frames: %v", got)
+ }
+ // Quiet input returns to sharp frames at the panel width.
+ deadline := time.Now().Add(2 * time.Second)
+ for len(starts()) < 2 && time.Now().Before(deadline) {
+ time.Sleep(10 * time.Millisecond)
+ }
+ got = starts()
+ if len(got) != 2 || got[1]["maxWidth"] != 1600.0 || got[1]["quality"] != 80.0 {
+ t.Fatalf("settled frames: %v", got)
+ }
+ // Dragging counts as motion too.
+ if err := view.input(ctx, []domain.BrowserInputEvent{{Type: "mouse", Action: "move", X: 0.4, Y: 0.5, Buttons: 1}}); err != nil {
+ t.Fatal(err)
+ }
+ if got := starts(); len(got) != 3 || got[2]["quality"] != 60.0 {
+ t.Fatalf("drag frames: %v", got)
}
}
diff --git a/internal/runtime/browser_view.go b/internal/runtime/browser_view.go
index 2b11bc9d..82896957 100644
--- a/internal/runtime/browser_view.go
+++ b/internal/runtime/browser_view.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "math"
"net/url"
"strings"
"sync"
@@ -28,19 +29,29 @@ type browserView struct {
endpoint browserEndpoint
client *cdpClient
- mu sync.Mutex
- targets map[string]*browserTarget
- active string
- session string
- frame string
- seq int64
- width float64
- height float64
- lastUsed time.Time
- changed chan struct{}
- follow bool
+ mu sync.Mutex
+ targets map[string]*browserTarget
+ active string
+ session string
+ frame string
+ seq int64
+ width float64
+ height float64
+ // Document scroll position from frame metadata and content size from
+ // layout metrics, refreshed at most every browserContentRefresh.
+ scrollX, scrollY float64
+ contentWidth, contentHeight float64
+ contentAt time.Time
+ contentBusy bool
+ lastUsed time.Time
+ changed chan struct{}
+ follow bool
// frameWidth is the panel's device pixel width; 0 keeps the default cap.
frameWidth int
+ // motion is set while the panel scrolls or drags: frames are captured
+ // lighter until the input settles (see markMotion).
+ motion bool
+ motionGen int64
// History flags are cached per active page URL: re-reading them on every
// frame would add a CDP round trip to each one.
historyKey string
@@ -57,6 +68,13 @@ var browserViewReads = map[string]bool{"": true, "status": true, "frame": true}
// HandleBrowser serves one task panel browser request for a run whose browser
// lives on this host (docker sidecar or host Chromium).
func HandleBrowser(ctx context.Context, runID string, req domain.BrowserRequest) (domain.BrowserState, error) {
+ if req.Operation == "http" {
+ // Dev server preview needs the sandbox network, not the CDP view.
+ return handleDevServerHTTP(ctx, runID, req.HTTP)
+ }
+ if req.Operation == "file" {
+ return handleWorkspaceFile(ctx, runID, req.File)
+ }
view, err := openBrowserView(ctx, runID)
if err != nil {
return domain.BrowserState{Available: false, Reason: err.Error()}, nil
@@ -115,6 +133,12 @@ func HandleBrowser(ctx context.Context, runID string, req domain.BrowserRequest)
if err := view.client.call(ctx, "", "Target.closeTarget", map[string]any{"targetId": target}, nil); err != nil {
return view.state(req.After), err
}
+ case "inspect":
+ inspection, err := view.inspect(ctx, req.Point, req.Detail)
+ // Inspection never needs a frame: the panel already shows one.
+ state := view.state(math.MaxInt64)
+ state.Inspect = inspection
+ return state, err
case "snapshot", "screenshot", "click", "type", "press":
snap, frame, err := view.agentOperation(ctx, req)
state := view.state(req.After)
@@ -307,8 +331,11 @@ func (v *browserView) onEvent(msg cdpMessage) {
Data string `json:"data"`
SessionID int64 `json:"sessionId"`
Metadata struct {
- DeviceWidth float64 `json:"deviceWidth"`
- DeviceHeight float64 `json:"deviceHeight"`
+ DeviceWidth float64 `json:"deviceWidth"`
+ DeviceHeight float64 `json:"deviceHeight"`
+ ScrollOffsetX float64 `json:"scrollOffsetX"`
+ ScrollOffsetY float64 `json:"scrollOffsetY"`
+ PageScaleFactor float64 `json:"pageScaleFactor"`
} `json:"metadata"`
}
if json.Unmarshal(msg.Params, &ev) != nil {
@@ -316,12 +343,20 @@ func (v *browserView) onEvent(msg cdpMessage) {
}
v.mu.Lock()
current := msg.SessionID == v.session
+ refresh := false
if current {
v.frame, v.width, v.height = ev.Data, ev.Metadata.DeviceWidth, ev.Metadata.DeviceHeight
+ v.scrollX, v.scrollY = ev.Metadata.ScrollOffsetX, ev.Metadata.ScrollOffsetY
v.seq++
v.notify()
+ if !v.contentBusy && time.Since(v.contentAt) > browserContentRefresh {
+ v.contentBusy, refresh = true, true
+ }
}
v.mu.Unlock()
+ if refresh {
+ go v.refreshContentSize(msg.SessionID)
+ }
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@@ -330,6 +365,29 @@ func (v *browserView) onEvent(msg cdpMessage) {
}
}
+// browserContentRefresh bounds layout metric reads: content size changes
+// rarely, and one read per frame would add a CDP round trip to each.
+const browserContentRefresh = time.Second
+
+func (v *browserView) refreshContentSize(session string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ var metrics struct {
+ CSSContentSize struct {
+ Width float64 `json:"width"`
+ Height float64 `json:"height"`
+ } `json:"cssContentSize"`
+ }
+ err := v.client.call(ctx, session, "Page.getLayoutMetrics", nil, &metrics)
+ v.mu.Lock()
+ defer v.mu.Unlock()
+ v.contentBusy = false
+ v.contentAt = time.Now()
+ if err == nil && session == v.session {
+ v.contentWidth, v.contentHeight = metrics.CSSContentSize.Width, metrics.CSSContentSize.Height
+ }
+}
+
func (v *browserView) latestTarget() string {
v.mu.Lock()
defer v.mu.Unlock()
@@ -401,13 +459,64 @@ func (v *browserView) attach(ctx context.Context, targetID string) error {
func (v *browserView) screencastParams() map[string]any {
v.mu.Lock()
- width := v.frameWidth
+ width, motion := v.frameWidth, v.motion
v.mu.Unlock()
- if width <= 0 || width > 1600 {
- width = 1600
+ if width <= 0 || width > 2560 {
+ width = 2560
+ }
+ // Text needs a cleaner JPEG than photos; 80 keeps edges sharp at about
+ // twice the bytes of 60.
+ quality := 80
+ if motion {
+ // Moving content is not read: half the pixels at quality 60 cut a
+ // text page frame to about a fifth, so the sandbox renders and the
+ // link carries several times as many while scrolling.
+ width, quality = width/2, browserMotionQuality
}
width = max(width, 320)
- return map[string]any{"format": "jpeg", "quality": 60, "maxWidth": width, "maxHeight": min(2*width, 2000), "everyNthFrame": 1}
+ return map[string]any{"format": "jpeg", "quality": quality, "maxWidth": width, "maxHeight": min(2*width, 4000), "everyNthFrame": 1}
+}
+
+const browserMotionQuality = 60
+
+// browserMotionSettle is how long after the last scroll or drag the
+// screencast returns to sharp frames.
+var browserMotionSettle = 400 * time.Millisecond
+
+// markMotion switches the screencast to light frames for scrolling and
+// dragging, and back once input has been quiet for browserMotionSettle.
+// Restarting the screencast makes Chromium send the current page again, so
+// the view ends on a sharp frame even when nothing moves afterwards.
+func (v *browserView) markMotion(ctx context.Context) {
+ v.mu.Lock()
+ v.motionGen++
+ gen, start, session := v.motionGen, !v.motion, v.session
+ v.motion = true
+ v.mu.Unlock()
+ if start && session != "" {
+ v.restartScreencast(ctx, session)
+ }
+ time.AfterFunc(browserMotionSettle, func() {
+ v.mu.Lock()
+ if gen != v.motionGen || !v.motion {
+ v.mu.Unlock()
+ return
+ }
+ v.motion = false
+ session := v.session
+ v.mu.Unlock()
+ if session == "" {
+ return
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ v.restartScreencast(ctx, session)
+ })
+}
+
+func (v *browserView) restartScreencast(ctx context.Context, session string) {
+ _ = v.client.call(ctx, session, "Page.stopScreencast", nil, nil)
+ _ = v.client.call(ctx, session, "Page.startScreencast", v.screencastParams(), nil)
}
// setFrameWidth restarts the screencast when the panel's pixel width changes.
@@ -469,7 +578,8 @@ func (v *browserView) waitFrame(ctx context.Context, after int64, waitMs int) {
func (v *browserView) state(after int64) domain.BrowserState {
v.mu.Lock()
defer v.mu.Unlock()
- state := domain.BrowserState{Available: !v.client.closed(), Kind: v.endpoint.Kind, Seq: v.seq, Width: v.width, Height: v.height}
+ state := domain.BrowserState{Available: !v.client.closed(), Kind: v.endpoint.Kind, Seq: v.seq, Width: v.width, Height: v.height,
+ ScrollX: v.scrollX, ScrollY: v.scrollY, ContentWidth: v.contentWidth, ContentHeight: v.contentHeight}
if !state.Available {
state.Reason = "浏览器连接已断开"
}
@@ -620,6 +730,12 @@ func (v *browserView) input(ctx context.Context, events []domain.BrowserInputEve
if width <= 0 || height <= 0 {
width, height = 1280, 800
}
+ for _, ev := range events {
+ if ev.Type == "wheel" || ev.Type == "mouse" && ev.Action == "move" && ev.Buttons != 0 {
+ v.markMotion(ctx)
+ break
+ }
+ }
for _, ev := range events {
x, y := clampUnit(ev.X)*width, clampUnit(ev.Y)*height
var err error
diff --git a/internal/runtime/core_cases_test.go b/internal/runtime/core_cases_test.go
index 84d21929..94df90ce 100644
--- a/internal/runtime/core_cases_test.go
+++ b/internal/runtime/core_cases_test.go
@@ -125,7 +125,7 @@ func TestBrowserViewOperationsWithFakeCDP(t *testing.T) {
if err := view.pageCall(ctx, "Page.reload", nil); err != nil {
t.Fatal(err)
}
- if got := view.screencastParams(); got["maxWidth"] != 1600 {
+ if got := view.screencastParams(); got["maxWidth"] != 2560 {
t.Fatalf("params=%v", got)
}
view.frameWidth = 100