Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
feat: render sandbox dev servers natively in the side browser (#162)
crispinhu2741e6a07aa554
diff --git a/chatbot/src/AgentBrowser.jsx b/chatbot/src/AgentBrowser.jsxindex 1be66026..dc3205af 100644--- a/chatbot/src/AgentBrowser.jsx+++ b/chatbot/src/AgentBrowser.jsx@@ -1,6 +1,8 @@-import React, { useEffect, useRef, useState } from "react";+import React, { useCallback, useEffect, useRef, useState } from "react";import HarnessIcon from "./HarnessIcon";import { api } from "./api";+import { devServerAddress, devServerLabel, devServerOrigin, devServerTarget, devServerURL } from "./agent-browser-links.js";+import { elementLabel, elementReference, inspectionReference } from "./element-context.js";import "./agent-browser.css";const modifiers = (e) =>@@ -10,12 +12,595 @@ const buttonName = (b) => (b === 1 ? "middle" : b === 2 ? "right" : "left");const deniedError = "无权操作";const startingText = "正在启动 Agent 浏览器…";-// AgentBrowser shows the Agent browser of the conversation's workspace: the-// browser Main Bot drives with workbench_browser_*, running on the execution-// machine next to the workspace. The user watches it live and can take over.-// Protocol and input handling follow the task panel in web/collaboration.js:-// one WebSocket carries frames and input, plain polling is the fallback.-export default function AgentBrowser({ runId, request = null, onRequestHandled = () => {} }) {+// AgentBrowser is the side panel's 浏览器 tab. Dev server addresses+// (http://localhost:PORT) and workspace files (file:///workspace/…) open+// natively: the platform serves them and the user's own browser renders the+// page. Everything else shows the Agent's+// browser on the execution machine as a live screencast.+// onAddElement, when given, lets the user pick an element of a native page+// and add it to the composer as context for the Agent.+export default function AgentBrowser({ runId, request = null, onRequestHandled = () => {}, onAddElement = null }) {+ // Decided before the first render, so a native page never starts a screencast.+ const [native, setNative] = useState(() => {+ const target = request?.url ? devServerTarget(request.url) : null;+ return target ? { ...target, key: request.id } : null;+ });+ const [forward, setForward] = useState(null);+ useEffect(() => {+ if (!request?.url) return;+ const target = devServerTarget(request.url);+ if (target) {+ setNative({ ...target, key: request.id });+ onRequestHandled(request.id);+ return;+ }+ setNative(null);+ setForward(request);+ }, [request?.id, request?.url]);+ const forwarded = useCallback(+ (id) => {+ setForward((current) => (current?.id === id ? null : current));+ onRequestHandled(id);+ },+ [onRequestHandled],+ );+ if (native)+ return (+ <DevServerFrame+ key={`${native.port}:${native.key}`}+ runId={runId}+ port={native.port}+ path={native.path}+ onOpen={(url) => {+ const target = devServerTarget(url);+ if (target) setNative({ ...target, key: `${Date.now()}` });+ else {+ setNative(null);+ setForward({ id: `${Date.now()}-${Math.random()}`, url });+ }+ }}+ onScreencast={() => setNative(null)}+ onAddElement={onAddElement}+ />+ );+ return (+ <ScreencastBrowser+ runId={runId}+ request={forward}+ onRequestHandled={forwarded}+ onNative={(target) => setNative({ ...target, key: `${Date.now()}` })}+ onAddElement={onAddElement}+ />+ );+}++// DevServerFrame renders a dev server or a file of the workspace in an iframe+// served by the platform under /devpreview/: native input, scrolling and+// devtools.+function DevServerFrame({ runId, port, path, onOpen, onScreencast, onAddElement = null }) {+ const frameRef = useRef(null);+ const addressRef = useRef(null);+ const [address, setAddress] = useState(`${devServerOrigin(port)}${path}`);+ const [src] = useState(() => devServerURL(runId, port, path));+ const [loading, setLoading] = useState(true);+ const [loads, setLoads] = useState(0);+ const pick = useElementPicker(frameRef, loads);+ const current = () => {+ try {+ return frameRef.current?.contentWindow?.location || null;+ } catch {+ return null;+ }+ };+ // Client-side routing changes the address without a load event.+ useEffect(() => {+ const timer = setInterval(() => {+ const location = current();+ if (!location || location.href === "about:blank" || document.activeElement === addressRef.current) return;+ setAddress(devServerAddress(runId, port, location));+ }, 800);+ return () => clearInterval(timer);+ }, [runId, port]);+ const history = (step) => {+ try {+ frameRef.current?.contentWindow?.history.go(step);+ } catch {+ /* cross-origin page: ignore */+ }+ };+ const reload = () => {+ setLoading(true);+ try {+ frameRef.current.contentWindow.location.reload();+ } catch {+ frameRef.current.src = src;+ }+ };+ const openWindow = () => {+ const location = current();+ const href = location && location.href !== "about:blank" ? location.href : src;+ window.open(href, "_blank", "noopener");+ };+ const button = (icon, label, onClick, extra = "") => (+ <button type="button" className={`agent-browser-btn${extra}`} title={label} aria-label={label} onClick={onClick}>+ <HarnessIcon name={icon} size={15} />+ </button>+ );+ return (+ <div className="agent-browser is-native">+ <div className="agent-browser-bar">+ {button("arrow-left", "后退", () => history(-1))}+ {button("arrow-right", "前进", () => history(1))}+ {button("refresh", "刷新", reload)}+ <form+ className="agent-browser-address"+ onSubmit={(event) => {+ event.preventDefault();+ const target = address.trim();+ if (!target) return;+ const next = devServerTarget(/^[a-z][a-z0-9+.-]*:/i.test(target) ? target : `http://${target}`);+ if (next && next.port === port) {+ setLoading(true);+ frameRef.current.src = devServerURL(runId, port, next.path);+ addressRef.current?.blur();+ } else onOpen(target);+ }}+ >+ <HarnessIcon name="globe" size={13} />+ <input+ ref={addressRef}+ type="text"+ spellCheck={false}+ aria-label="网址"+ value={address}+ onChange={(event) => setAddress(event.target.value)}+ />+ {onAddElement ? <AnnotateButton picking={pick.picking} onClick={pick.toggle} /> : null}+ </form>+ {button("preview-expand", "在新窗口打开", openWindow)}+ {button("monitor", "切换到 Agent 浏览器画面", onScreencast)}+ </div>+ <div className="agent-browser-stage agent-browser-native">+ <iframe+ ref={frameRef}+ className="agent-browser-frame"+ title={port === "file" ? "工作区文件" : `开发服务器 localhost:${port}`}+ src={src}+ onLoad={() => {+ setLoading(false);+ setLoads((n) => n + 1);+ const location = current();+ if (location && document.activeElement !== addressRef.current)+ setAddress(devServerAddress(runId, port, location));+ }}+ />+ {loading ? (+ <div className="agent-browser-overlay agent-browser-loading" data-tone="idle">+ <span>正在打开 {devServerLabel(port)}…</span>+ </div>+ ) : null}+ <ElementPickLayer+ pick={pick}+ frameRef={frameRef}+ onAdd={(el) => {+ const location = current();+ onAddElement?.(elementReference(el, location ? devServerAddress(runId, port, location) : address));+ pick.clear();+ }}+ />+ </div>+ <div className="agent-browser-foot">+ <span>原生预览 · {port === "file" ? "工作区文件" : `端口 ${port}`}</span>+ <small role="status">+ {port === "file"+ ? "页面在你的浏览器里直接渲染;文件改动后点刷新"+ : "页面在你的浏览器里直接渲染;暂不支持热更新,代码改动后点刷新"}+ </small>+ </div>+ </div>+ );+}++// useElementPicker drives element selection inside a same-origin preview+// frame: hovering outlines the element under the pointer, a click selects+// it without reaching the page. The page's own DOM is left alone apart from a+// temporary crosshair cursor; outlines are drawn above the frame.+function useElementPicker(frameRef, loads) {+ const [picking, setPicking] = useState(false);+ const [hover, setHover] = useState(null);+ const [picked, setPicked] = useState(null);+ const [error, setError] = useState("");+ const [, setTick] = useState(0);+ const hovered = useRef(null);+ const frameDocument = () => {+ try {+ return frameRef.current?.contentDocument || null;+ } catch {+ return null;+ }+ };+ // Selection and outline follow the page while it scrolls or resizes.+ useEffect(() => {+ if (!picked && !picking) return undefined;+ const doc = frameDocument();+ const win = doc?.defaultView;+ if (!win) return undefined;+ const refresh = () => setTick((n) => n + 1);+ win.addEventListener("scroll", refresh, true);+ win.addEventListener("resize", refresh);+ return () => {+ win.removeEventListener("scroll", refresh, true);+ win.removeEventListener("resize", refresh);+ };+ }, [picked, picking, loads]);+ useEffect(() => {+ if (!picking) return undefined;+ const doc = frameDocument();+ if (!doc?.documentElement) {+ setError("当前页面不支持标记元素");+ setPicking(false);+ return undefined;+ }+ setError("");+ const cursor = doc.createElement("style");+ cursor.setAttribute("data-af-pick", "");+ cursor.textContent = "*{cursor:crosshair!important}";+ (doc.head || doc.documentElement).appendChild(cursor);+ const target = (event) => {+ const el = event.target;+ return el && el.nodeType === 1 && el !== doc.documentElement ? el : null;+ };+ const move = (event) => {+ const el = target(event);+ if (el === hovered.current) return;+ hovered.current = el;+ setHover(el ? { el } : null);+ };+ const block = (event) => {+ event.preventDefault();+ event.stopPropagation();+ event.stopImmediatePropagation?.();+ };+ const click = (event) => {+ block(event);+ const el = target(event);+ if (!el) return;+ setPicked({ el });+ setPicking(false);+ };+ const key = (event) => {+ if (event.key !== "Escape") return;+ block(event);+ setPicking(false);+ };+ const leave = () => {+ hovered.current = null;+ setHover(null);+ };+ const blocked = ["mousedown", "mouseup", "pointerdown", "pointerup", "dblclick", "contextmenu", "auxclick"];+ doc.addEventListener("mousemove", move, true);+ doc.addEventListener("click", click, true);+ doc.addEventListener("keydown", key, true);+ doc.documentElement.addEventListener("mouseleave", leave);+ blocked.forEach((name) => doc.addEventListener(name, block, true));+ return () => {+ doc.removeEventListener("mousemove", move, true);+ doc.removeEventListener("click", click, true);+ doc.removeEventListener("keydown", key, true);+ doc.documentElement.removeEventListener("mouseleave", leave);+ blocked.forEach((name) => doc.removeEventListener(name, block, true));+ cursor.remove();+ hovered.current = null;+ setHover(null);+ };+ }, [picking, loads]);+ // A navigation replaces the document the selection belonged to.+ useEffect(() => setPicked(null), [loads]);+ useEffect(() => {+ if (!picking && !picked) return undefined;+ const key = (event) => {+ if (event.key !== "Escape") return;+ setPicking(false);+ setPicked(null);+ };+ document.addEventListener("keydown", key);+ return () => document.removeEventListener("keydown", key);+ }, [picking, picked]);+ return {+ picking,+ hover,+ picked: picked?.el?.isConnected ? picked : null,+ error,+ toggle: () => {+ setPicked(null);+ setPicking((on) => !on);+ },+ repick: () => {+ setPicked(null);+ setPicking(true);+ },+ clear: () => setPicked(null),+ };+}++const outlineOf = (el, frame) => {+ const rect = el.getBoundingClientRect();+ return {+ left: frame.offsetLeft + rect.left,+ top: frame.offsetTop + rect.top,+ width: rect.width,+ height: rect.height,+ };+};++// AnnotateButton sits at the end of the address bar and turns element+// marking on and off.+function AnnotateButton({ picking, disabled = false, onClick }) {+ return (+ <button+ type="button"+ className={`agent-browser-annotate${picking ? " is-on" : ""}`}+ title={picking ? "退出标记(Esc)" : "在页面上标记元素,加入对话框"}+ aria-pressed={picking}+ disabled={disabled}+ onClick={onClick}+ >+ <HarnessIcon name="crosshair" size={14} />+ <span>标记元素</span>+ </button>+ );+}++// ElementPickLayer outlines elements of the native preview frame.+function ElementPickLayer({ pick, frameRef, onAdd }) {+ const frame = frameRef.current;+ const stage = frame?.parentElement;+ if (!frame || !stage) return null;+ const hover = pick.picking && pick.hover?.el?.isConnected ? pick.hover.el : null;+ const picked = pick.picked?.el || null;+ const box = (el) => (el ? { ...outlineOf(el, frame), label: elementLabel(el) } : null);+ return (+ <PickOverlay+ stage={stage}+ hover={box(hover)}+ picked={box(picked)}+ picking={pick.picking}+ error={pick.error}+ onAdd={() => onAdd(picked)}+ onRepick={pick.repick}+ onCancel={pick.clear}+ />+ );+}++// PickOverlay draws the hover outline and, once an element is picked, a+// popover that adds it to the composer. Boxes are in stage pixels.+function PickOverlay({ stage, hover, picked, picking, error, busy = false, onAdd, onRepick, onCancel }) {+ if (!stage) return null;+ const box = (o, className) => (+ <div className={className} style={{ left: o.left, top: o.top, width: o.width, height: o.height }} aria-hidden="true">+ <span+ className={`agent-browser-pick-tag${o.top < 22 ? " is-inside" : ""}${+ o.left + 260 > stage.clientWidth ? " is-end" : ""+ }`}+ >+ <span className="agent-browser-pick-label">{o.label}</span>+ <span className="agent-browser-pick-size">+ {Math.round(o.width)}×{Math.round(o.height)}+ </span>+ </span>+ </div>+ );+ let popover = null;+ if (picked) {+ const width = Math.min(280, stage.clientWidth - 16);+ const below = picked.top + picked.height + 8;+ const top = below + 92 <= stage.clientHeight ? below : Math.max(8, picked.top - 92 - 8);+ const left = Math.max(8, Math.min(picked.left, stage.clientWidth - width - 8));+ popover = (+ <div className="agent-browser-pick-popover" role="dialog" aria-label="选中的元素" style={{ top, left, width }}>+ <div className="agent-browser-pick-name">{picked.label}</div>+ <div className="agent-browser-pick-actions">+ <button type="button" className="agent-browser-pick-primary" disabled={busy} onClick={onAdd}>+ <HarnessIcon name="chat" size={14} />+ {busy ? "读取中…" : "加入对话框"}+ </button>+ <button type="button" className="agent-browser-pick-secondary" onClick={onRepick}>+ 重新选择+ </button>+ <button type="button" className="agent-browser-pick-secondary" onClick={onCancel}>+ 取消+ </button>+ </div>+ </div>+ );+ }+ return (+ <>+ {hover ? box(hover, "agent-browser-pick-outline") : null}+ {picked ? box(picked, "agent-browser-pick-outline is-picked") : null}+ {popover}+ {picking ? (+ <div className="agent-browser-pick-hint" role="status">+ 点击要标记的元素,Esc 退出+ </div>+ ) : null}+ {error ? (+ <div className="agent-browser-pick-hint" role="alert">+ {error}+ </div>+ ) : null}+ </>+ );+}++// useScreencastPicker selects elements on the Agent browser screencast. The+// page is inspected over CDP (operation "inspect"): hovering asks for the+// element under the pointer, one request at a time with only the latest point+// kept; a click asks again with the full description the composer sends.+function useScreencastPicker(runId, stageRef, screenRef) {+ const [picking, setPicking] = useState(false);+ const [hover, setHover] = useState(null);+ const [picked, setPicked] = useState(null);+ const [inspection, setInspection] = useState(null);+ const [busy, setBusy] = useState(false);+ const [error, setError] = useState("");+ const generation = useRef(0);+ const inflight = useRef(false);+ const pending = useRef(null);++ const inspect = useCallback(+ async (point, detail) => {+ const state = await api(`/api/runs/${encodeURIComponent(runId)}/browser`, {+ method: "POST",+ body: JSON.stringify({ operation: "inspect", point, detail }),+ });+ return state?.inspect || null;+ },+ [runId],+ );+ // box maps an inspection (page CSS px) onto the stage.+ const box = useCallback(+ (found) => {+ const stage = stageRef.current, screen = screenRef.current;+ if (!found?.rect || !found.viewport?.width || !stage || !screen) return null;+ const s = stage.getBoundingClientRect(), img = screen.getBoundingClientRect();+ const scale = img.width / found.viewport.width;+ return {+ left: img.left - s.left + (found.rect.x || 0) * scale,+ top: img.top - s.top + (found.rect.y || 0) * scale,+ width: found.rect.width * scale,+ height: found.rect.height * scale,+ label: found.label || "",+ };+ },+ [stageRef, screenRef],+ );+ const pointOf = (e) => {+ const rect = screenRef.current?.getBoundingClientRect();+ if (!rect?.width || !rect.height) return null;+ const x = (e.clientX - rect.left) / rect.width, y = (e.clientY - rect.top) / rect.height;+ return x < 0 || y < 0 || x > 1 || y > 1 ? null : { x, y };+ };+ const pump = useCallback(async () => {+ if (inflight.current || !pending.current) return;+ const { point, gen } = pending.current;+ pending.current = null;+ inflight.current = true;+ try {+ const found = await inspect(point, false);+ if (gen === generation.current) setHover(box(found));+ } catch (err) {+ if (gen === generation.current) setError(err.message || "无法检查页面元素");+ } finally {+ inflight.current = false;+ pump();+ }+ }, [inspect, box]);++ const clear = useCallback(() => {+ generation.current += 1;+ pending.current = null;+ setPicking(false);+ setHover(null);+ setPicked(null);+ setInspection(null);+ setBusy(false);+ setError("");+ }, []);+ const repick = useCallback(() => {+ clear();+ setPicking(true);+ }, [clear]);+ const active = picking || !!picked;+ useEffect(() => {+ if (!active) return undefined;+ // Capture phase: Escape leaves picking before the screencast forwards it.+ const onKey = (e) => {+ if (e.key !== "Escape") return;+ e.preventDefault();+ e.stopPropagation();+ clear();+ };+ document.addEventListener("keydown", onKey, true);+ return () => document.removeEventListener("keydown", onKey, true);+ }, [active, clear]);+ useEffect(() => clear, [runId, clear]);++ return {+ picking,+ active,+ hover,+ picked,+ inspection,+ busy,+ error,+ clear,+ repick,+ toggle: () => (picking ? clear() : repick()),+ // The page scrolled under an outline: drop it until the pointer moves.+ scrolled: () => {+ generation.current += 1;+ setHover(null);+ },+ leave: () => {+ generation.current += 1;+ pending.current = null;+ setHover(null);+ },+ move: (e) => {+ const point = pointOf(e);+ if (!point) return;+ setError("");+ pending.current = { point, gen: generation.current };+ pump();+ },+ select: async (e) => {+ const point = pointOf(e);+ if (!point) return;+ generation.current += 1;+ const gen = generation.current;+ pending.current = null;+ setPicking(false);+ setBusy(true);+ const s = stageRef.current?.getBoundingClientRect() || { left: 0, top: 0 };+ setPicked(hover ? { ...hover } : { left: e.clientX - s.left, top: e.clientY - s.top, width: 0, height: 0, label: "…" });+ try {+ const found = await inspect(point, true);+ if (gen !== generation.current) return;+ if (!found?.element) {+ setPicked(null);+ setPicking(true);+ setError("这个位置没有可选的元素");+ } else {+ setInspection(found);+ setPicked(box(found));+ }+ } catch (err) {+ if (gen !== generation.current) return;+ setPicked(null);+ setPicking(true);+ setError(err.message || "无法检查页面元素");+ } finally {+ if (gen === generation.current) setBusy(false);+ }+ },+ };+}++// ScreencastBrowser shows the Agent browser of the conversation's workspace:+// the browser Main Bot drives with workbench_browser_*, running on the+// execution machine next to the workspace. The user watches it live and can+// take over. Protocol and input handling follow the task panel in+// web/collaboration.js: one WebSocket carries frames and input, plain polling+// is the fallback.+function ScreencastBrowser({+ runId,+ request = null,+ onRequestHandled = () => {},+ onNative = () => {},+ onAddElement = null,+}) {const stageRef = useRef(null);const screenRef = useRef(null);const imeRef = useRef(null);@@ -27,6 +612,9 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =const [notice, setNotice] = useState("");const [address, setAddress] = useState("");const [denied, setDenied] = useState(false);+ const pick = useScreencastPicker(runId, stageRef, screenRef);+ const picking = useRef(false);+ picking.current = pick.active;useEffect(() => {if (!runId) return undefined;@@ -267,53 +855,24 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =queue.push(ev);flush();}- // Scrolling is shown locally first: the current frame slides by the pending- // delta until a frame captured after that scroll replaces it.- let wheel = null, wheelTimer = 0, settleTimer = 0, scrolls = [], shiftX = 0, shiftY = 0, frameLag = 80;+ // Scrolls are not previewed by sliding the current frame: that moved the+ // page's fixed parts and scrollbar with it and bared the stage below. The+ // stream delivers the scrolled frame in well under 100ms instead.+ let wheel = null, wheelTimer = 0;function point(e) {const rect = screen.getBoundingClientRect();return {- x: (e.clientX - rect.left + shiftX) / (rect.width || 1),- y: (e.clientY - rect.top + shiftY) / (rect.height || 1),+ x: (e.clientX - rect.left) / (rect.width || 1),+ y: (e.clientY - rect.top) / (rect.height || 1),};}- function applyShift() {- const scale = (screen.getBoundingClientRect().width || 1) / (state?.width || 1);- const limit = stage.clientHeight || 0;- let dx = wheel?.deltaX || 0, dy = wheel?.deltaY || 0;- for (const b of scrolls) {- dx += b.dx;- dy += b.dy;- }- shiftX = Math.max(-limit, Math.min(limit, -dx * scale));- shiftY = Math.max(-limit, Math.min(limit, -dy * scale));- screen.style.transform = shiftX || shiftY ? `translate(${shiftX}px,${shiftY}px)` : "";- }- function settleScroll(age) {- const now = Date.now();- scrolls = scrolls.filter((b) => now - b.at < age);- applyShift();- }function flushWheel() {wheelTimer = 0;if (!wheel) return;const ev = wheel;wheel = null;- scrolls.push({ dx: ev.deltaX, dy: ev.deltaY, at: Date.now() });push(ev);- applyShift();- clearTimeout(settleTimer);- settleTimer = setTimeout(() => settleScroll(400), 500);}- // A frame newer than the round trip already includes the scroll.- const onFrameLoad = () => {- const now = Date.now(), first = scrolls.find((b) => !b.seen);- if (first) {- first.seen = true;- frameLag = Math.round(frameLag * 0.7 + (now - first.at) * 0.3);- }- settleScroll(Math.max(50, Math.min(400, frameLag)));- };const mouse = (e, action) => ({type: "mouse",action,@@ -323,18 +882,21 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =clickCount: e.detail || 1,modifiers: modifiers(e),});+ // While picking, the capture layer owns the pointer: nothing reaches the page.+ const onScreen = (e) => !picking.current && (e.target === screen || e.target === stage);const onPointerDown = (e) => {- if (!state?.available || (e.target !== screen && e.target !== stage)) return;+ if (!state?.available || !onScreen(e)) return;e.preventDefault();stage.setPointerCapture?.(e.pointerId);ime.focus({ preventScroll: true });+ claim();push(mouse(e, "down"));};const onPointerUp = (e) => {- if (state?.available) push(mouse(e, "up"));+ if (state?.available && onScreen(e)) push(mouse(e, "up"));};const onPointerMove = (e) => {- if (!state?.available || readOnly) return;+ if (!state?.available || readOnly || !onScreen(e)) return;pendingMove = {type: "mouse",action: "move",@@ -354,6 +916,7 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =};const onWheel = (e) => {if (!state?.available || readOnly) return;+ pick.scrolled();e.preventDefault();const unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? state.height || 600 : 1;wheel = {@@ -363,13 +926,14 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =deltaY: (wheel?.deltaY || 0) + e.deltaY * unit,modifiers: modifiers(e),};- applyShift();+ claim();if (!wheelTimer) wheelTimer = setTimeout(flushWheel, 40);};const onContextMenu = (e) => e.preventDefault();const onStageFocus = () => ime.focus({ preventScroll: true });const onKeyDown = (e) => {if (e.isComposing || e.keyCode === 229) return;+ if (picking.current && e.key === "Escape") return;if ((e.metaKey || e.ctrlKey) && ["v", "c", "x"].includes(e.key.toLowerCase())) return;// Handled here so Escape or Tab reach the page instead of closing the panel.e.preventDefault();@@ -398,20 +962,43 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =if (ime.value) push({ type: "text", text: ime.value });ime.value = "";};+ // wantedSize is the Agent browser viewport that fills this panel.+ function wantedSize() {+ const rect = stage.getBoundingClientRect();+ if (!state?.available || rect.width < 120 || rect.height < 120) return null;+ const width = Math.round(Math.min(1600, Math.max(1024, rect.width)));+ const height = Math.round(Math.min(2000, Math.max(500, (width * rect.height) / rect.width)));+ // Frames carry up to the panel's device pixels: the Agent browser+ // renders at 2x, so a high-DPI panel is not shown upscaled frames.+ const frameWidth = Math.round(rect.width * Math.min(window.devicePixelRatio || 1, 2));+ return { width, height, frameWidth };+ }+ function sendSize(force = false) {+ if (!alive || readOnly) return;+ const size = wantedSize();+ if (!size) return;+ const key = `${size.width}x${size.height}@${size.frameWidth}`;+ if (key === sentSize && !force) return;+ sentSize = key;+ command({ operation: "resize", ...size });+ }function fit() {clearTimeout(sizeTimer);- sizeTimer = setTimeout(() => {- if (!alive || readOnly) return;- const rect = stage.getBoundingClientRect();- if (!state?.available || rect.width < 120 || rect.height < 120) return;- const width = Math.round(Math.min(1600, Math.max(1024, rect.width)));- const height = Math.round(Math.min(2000, Math.max(500, (width * rect.height) / rect.width)));- const frameWidth = Math.round(rect.width * Math.min(window.devicePixelRatio || 1, 1.5));- const key = `${width}x${height}@${frameWidth}`;- if (key === sentSize) return;- sentSize = key;- command({ operation: "resize", width, height, frameWidth });- }, 400);+ sizeTimer = setTimeout(sendSize, 400);+ }+ // Every panel watching the run shares one Agent browser, and the last one+ // to resize it wins. The panel being used takes the size back when it no+ // longer fits, but only on input, so two open panels never take turns.+ let claimedAt = 0;+ function claim() {+ const size = wantedSize();+ if (!size || readOnly || Date.now() - claimedAt < 1000) return;+ // Frames report the viewport with the page's scrollbar (about 15px)+ // while resize sets it without, so only a clearly different size counts.+ const off = (a, b) => !(a > 0) || Math.abs(a - b) > 24;+ if (!off(state.width, size.width) && !off(state.height, size.height)) return;+ claimedAt = Date.now();+ sendSize(true);}const listeners = [@@ -421,7 +1008,6 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =[stage, "wheel", onWheel, { passive: false }],[stage, "contextmenu", onContextMenu],[stage, "focus", onStageFocus],- [screen, "load", onFrameLoad],[ime, "keydown", onKeyDown],[ime, "keyup", onKeyUp],[ime, "compositionstart", onCompositionStart],@@ -441,7 +1027,7 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =controller.current = null;for (const [target, type, fn, options] of listeners) target.removeEventListener(type, fn, options);observer?.disconnect();- for (const timer of [noticeTimer, reconnectTimer, sizeTimer, moveTimer, wheelTimer, settleTimer]) clearTimeout(timer);+ for (const timer of [noticeTimer, reconnectTimer, sizeTimer, moveTimer, wheelTimer]) clearTimeout(timer);screen.onload = screen.onerror = null;ws?.close();ws = null;@@ -495,8 +1081,22 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =readOnly={denied}onChange={(event) => setAddress(event.target.value)}/>+ {onAddElement && !denied ? (+ <AnnotateButton picking={pick.picking} disabled={!page.available} onClick={pick.toggle} />+ ) : null}</form>{navButton("plus", "新标签页", "newTab")}+ {devServerTarget(address) ? (+ <button+ type="button"+ className="agent-browser-btn"+ title="在原生预览中打开(你的浏览器直接渲染,操作更流畅)"+ aria-label="在原生预览中打开"+ onClick={() => onNative(devServerTarget(address))}+ >+ <HarnessIcon name="preview-expand" size={15} />+ </button>+ ) : null}</div>{page.tabs.length > 1 ? (<div className="agent-browser-tabs" role="tablist" aria-label="浏览器标签页">@@ -555,6 +1155,32 @@ export default function AgentBrowser({ runId, request = null, onRequestHandled =) : null}</div>) : null}+ {pick.picking ? (+ <div+ className="agent-browser-pick-capture"+ data-testid="agent-browser-pick-capture"+ onPointerMove={pick.move}+ onPointerLeave={pick.leave}+ onPointerDown={(e) => e.preventDefault()}+ onClick={pick.select}+ />+ ) : null}+ {pick.active ? (+ <PickOverlay+ stage={stageRef.current}+ hover={pick.picking ? pick.hover : null}+ picked={pick.picked}+ picking={pick.picking}+ error={pick.error}+ busy={pick.busy}+ onAdd={() => {+ if (pick.inspection?.element) onAddElement?.(inspectionReference(pick.inspection));+ pick.clear();+ }}+ onRepick={pick.repick}+ onCancel={pick.clear}+ />+ ) : null}</div><div className="agent-browser-foot"><span>{status}</span>diff --git a/chatbot/src/App.jsx b/chatbot/src/App.jsxindex e61278a9..a70dee72 100644--- a/chatbot/src/App.jsx+++ b/chatbot/src/App.jsx@@ -54,6 +54,7 @@ import MentionPicker from "./MentionPicker";import Participants, { Avatar, authorFor } from "./Participants";import IssueContext from "./IssueContext";import { mentionAt, insertMention } from "./mentions.js";+import { elementTitle } from "./element-context.js";import HarnessIcon from "./HarnessIcon";import QueuedMessages from "./QueuedMessages";import "./message-queue.css";@@ -814,6 +815,16 @@ export default function App() {closeMenu();input.current?.focus();}+ // addElementReference takes an element picked in the native preview: it+ // rides along as a reference chip, without a token in the text.+ function addElementReference(ref) {+ if (references.length >= 12) {+ chat.setError("每轮最多引用 12 个对象。");+ return;+ }+ setReferences((old) => [...old, ref]);+ requestAnimationFrame(() => input.current?.focus());+ }function addReference(ref) {if (references.length >= 12 &&@@ -1509,8 +1520,16 @@ export default function App() {{message.references?.length ? (<div className="message-references">{message.references.map((ref) => (- <span key={ref.type + ref.id}>- @{ref.name || ref.id}+ <span+ key={ref.type + ref.id}+ title={ref.type === "element" ? elementTitle(ref) : undefined}+ >+ {ref.type === "element" ? (+ <HarnessIcon name="crosshair" size={12} />+ ) : (+ "@"+ )}+ {ref.name || ref.id}</span>))}</div>@@ -1843,9 +1862,15 @@ export default function App() {{references.map((ref) => (<spankey={ref.type + ref.id}- className="composer-attachment-file"+ className={`composer-attachment-file${ref.type === "element" ? " is-element" : ""}`}+ title={ref.type === "element" ? elementTitle(ref) : undefined}>- @{ref.name || ref.id}+ {ref.type === "element" ? (+ <HarnessIcon name="crosshair" size={13} />+ ) : (+ "@"+ )}+ {ref.name || ref.id}<buttonaria-label={`移除引用 ${ref.name || ref.id}`}onClick={() => {@@ -2134,6 +2159,7 @@ export default function App() {workspaceRunId={workspaceRunId}browserRequest={browserRequest}onBrowserRequestHandled={browserRequestHandled}+ onAddElement={chat.readOnly ? null : addElementReference}onClose={() => setPanel(false)}/>) : null}diff --git a/chatbot/src/ArtifactPanel.jsx b/chatbot/src/ArtifactPanel.jsxindex ba3c546a..536d0cb9 100644--- a/chatbot/src/ArtifactPanel.jsx+++ b/chatbot/src/ArtifactPanel.jsx@@ -192,6 +192,7 @@ export default function ArtifactPanel({workspaceRunId = "",browserRequest = null,onBrowserRequestHandled = () => {},+ onAddElement = null,}) {const root = useRef(null);const searchInput = useRef(null);@@ -577,6 +578,15 @@ export default function ArtifactPanel({runId={workspaceRunId}request={browserRequest}onRequestHandled={onBrowserRequestHandled}+ onAddElement={+ onAddElement &&+ ((ref) => {+ onAddElement(ref);+ // A compact panel covers the composer: go back so the+ // new chip is in view.+ if (layout.compact) onClose();+ })+ }/>) : (<div className="agent-browser-empty" role="status">diff --git a/chatbot/src/agent-browser-links.js b/chatbot/src/agent-browser-links.jsindex 37dd0c91..08685a22 100644--- a/chatbot/src/agent-browser-links.js+++ b/chatbot/src/agent-browser-links.js@@ -68,3 +68,49 @@ export function remarkWorkspaceLinks() {};return walk;}++// devServerTarget returns the port and path of a plain-HTTP address on the+// execution machine (a dev server), or null. Those pages are shown natively:+// the platform proxies the port, so the user's own browser renders them.+// Workspace files (file:///workspace/…) are served the same way; their+// "port" is "file".+export function devServerTarget(raw) {+ let url;+ try {+ url = new URL(String(raw || "").trim());+ } catch {+ return null;+ }+ if (url.protocol === "file:") {+ if (url.host || !url.pathname || url.pathname === "/") return null;+ return { port: devFile, path: `${url.pathname}${url.search}${url.hash}` };+ }+ if (url.protocol !== "http:" || !localHost.test(url.hostname)) return null;+ const port = Number(url.port || 80);+ if (!Number.isInteger(port) || port <= 0 || port > 65535) return null;+ return { port, path: `${url.pathname || "/"}${url.search}${url.hash}` };+}++const devFile = "file";++// devServerOrigin is how the page's address starts on the execution machine.+export const devServerOrigin = (port) => (port === devFile ? "file://" : `http://localhost:${port}`);++// devServerLabel names the target in the panel.+export const devServerLabel = (port) => (port === devFile ? "工作区文件" : `localhost:${port}`);++export const devServerPrefix = (runId, port) =>+ `/devpreview/${encodeURIComponent(runId)}/${port}`;++// devServerURL is where the platform serves the dev server page.+export const devServerURL = (runId, port, path = "/") =>+ `${devServerPrefix(runId, port)}${path.startsWith("/") ? path : `/${path}`}`;++// devServerAddress maps a proxied page location back to the address the+// dev server has on the execution machine.+export function devServerAddress(runId, port, location) {+ const prefix = devServerPrefix(runId, port);+ const path = location?.pathname || "/";+ const rest = path === prefix ? "/" : path.startsWith(`${prefix}/`) ? path.slice(prefix.length) : path;+ return `${devServerOrigin(port)}${rest}${location?.search || ""}${location?.hash || ""}`;+}diff --git a/chatbot/src/agent-browser.css b/chatbot/src/agent-browser.cssindex 4bc22237..54715d3b 100644--- a/chatbot/src/agent-browser.css+++ b/chatbot/src/agent-browser.css@@ -55,7 +55,8 @@background: var(--subtle);color: var(--muted);}-.agent-browser-address:focus-within {+/* Only typing an address highlights the bar, not its 标记元素 button. */+.agent-browser-address:has(input:focus) {border-color: var(--accent);}.agent-browser-address input {@@ -154,6 +155,21 @@filter: grayscale(1);opacity: 0.35;}+/* Native preview: the dev server page itself, rendered by this browser. */+.agent-browser-native {+ display: flex;+ background: #fff;+}+.agent-browser-frame {+ flex: 1;+ width: 100%;+ height: 100%;+ border: 0;+ background: #fff;+}+.agent-browser-loading {+ background: var(--subtle);+}.agent-browser-ime {position: absolute;top: 0;@@ -263,3 +279,186 @@[data-theme="dark"] .markdown .agent-browser-link code {color: #8ab4f8;}+/* Element picking in a native preview: outlines above the frame, a popover+ * that adds the picked element to the composer. */+.agent-browser-btn.is-on {+ background: var(--hover);+ color: var(--text);+}+.agent-browser-pick-outline {+ position: absolute;+ z-index: 2;+ box-sizing: border-box;+ border: 1.5px solid #3b82f6;+ border-radius: 3px;+ background: rgb(59 130 246 / 0.12);+ pointer-events: none;+}+.agent-browser-pick-outline.is-picked {+ border-width: 2px;+ background: rgb(59 130 246 / 0.08);+}+.agent-browser-pick-tag {+ position: absolute;+ display: flex;+ gap: 6px;+ bottom: 100%;+ left: -1.5px;+ max-width: 260px;+ margin-bottom: 4px;+ padding: 2px 6px;+ overflow: hidden;+ border-radius: 4px;+ background: #1f2937;+ color: #fff;+ font-size: 11px;+ line-height: 16px;+ white-space: nowrap;+ text-overflow: ellipsis;+}+/* The size stays visible; a long label is the part that gets cut. */+.agent-browser-pick-label {+ min-width: 0;+ overflow: hidden;+ text-overflow: ellipsis;+}+.agent-browser-pick-size {+ flex: none;+ opacity: 0.7;+}+.agent-browser-pick-tag.is-end {+ right: -1.5px;+ left: auto;+}+.agent-browser-pick-tag.is-inside.is-end {+ right: 0;+ left: auto;+ border-radius: 0 0 0 4px;+}+.agent-browser-pick-tag.is-inside {+ top: 0;+ bottom: auto;+ left: 0;+ margin: 0;+ border-radius: 0 0 4px 0;+}+.agent-browser-pick-popover {+ position: absolute;+ z-index: 3;+ box-sizing: border-box;+ padding: 10px;+ border: 1px solid var(--line);+ border-radius: 12px;+ background: var(--surface);+ box-shadow: 0 8px 28px rgb(0 0 0 / 0.14), 0 2px 6px rgb(0 0 0 / 0.06);+ color: var(--text);+ font-size: 12.5px;+ animation: agent-browser-pick-in 160ms cubic-bezier(0.2, 0.7, 0.3, 1);+}+.agent-browser-pick-name {+ margin-bottom: 8px;+ overflow: hidden;+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;+ font-size: 12px;+ white-space: nowrap;+ text-overflow: ellipsis;+}+.agent-browser-pick-actions {+ display: flex;+ gap: 6px;+}+.agent-browser-pick-actions button {+ display: inline-flex;+ align-items: center;+ gap: 5px;+ height: 28px;+ padding: 0 10px;+ border-radius: 7px;+ font-size: 12.5px;+ cursor: pointer;+ transition: background-color 120ms, color 120ms;+}+.agent-browser-pick-primary {+ border: 0;+ background: var(--accent);+ color: var(--surface);+}+.agent-browser-pick-primary:hover {+ background: color-mix(in srgb, var(--accent) 86%, var(--surface));+}+.agent-browser-pick-secondary {+ border: 1px solid var(--line);+ background: transparent;+ color: var(--muted);+}+.agent-browser-pick-secondary:hover {+ background: var(--hover);+ color: var(--text);+}+.agent-browser-pick-hint {+ position: absolute;+ bottom: 10px;+ left: 50%;+ z-index: 3;+ padding: 4px 10px;+ transform: translateX(-50%);+ border-radius: 999px;+ background: rgb(31 41 55 / 0.88);+ color: #fff;+ font-size: 12px;+ pointer-events: none;+}+@keyframes agent-browser-pick-in {+ from {+ opacity: 0;+ transform: translateY(-4px);+ }+}+@media (prefers-reduced-motion: reduce) {+ .agent-browser-pick-popover {+ animation: none;+ }+}+/* Element picking on the screencast: a transparent layer takes the pointer+ * so hovering and the pick click never reach the Agent browser. */+.agent-browser-pick-capture {+ position: absolute;+ inset: 0;+ z-index: 1;+ cursor: crosshair;+}+.agent-browser-pick-primary:disabled {+ opacity: 0.6;+ cursor: progress;+}+/* "标记元素" sits at the end of the address bar, like the page's own tool. */+.agent-browser-annotate {+ display: inline-flex;+ flex: none;+ align-items: center;+ gap: 4px;+ height: 22px;+ margin-right: -6px;+ padding: 0 7px;+ border: 0;+ border-radius: 999px;+ background: transparent;+ color: var(--muted);+ font: inherit;+ font-size: 12px;+ white-space: nowrap;+ cursor: pointer;+ transition: background-color 120ms, color 120ms;+}+.agent-browser-annotate:hover:not(:disabled) {+ background: var(--hover);+ color: var(--text);+}+.agent-browser-annotate.is-on {+ background: color-mix(in srgb, var(--accent) 14%, transparent);+ color: var(--accent);+}+.agent-browser-annotate:disabled {+ opacity: 0.4;+ cursor: default;+}diff --git a/chatbot/src/element-context.js b/chatbot/src/element-context.jsnew file mode 100644index 00000000..dbb4fa7b--- /dev/null+++ b/chatbot/src/element-context.js@@ -0,0 +1,239 @@+// Element context: what the user points at in a native preview, described so+// the Agent can find it in code (component and source file when the dev build+// exposes them) and in its own browser (a unique selector).++const styleKeys = [+ "display",+ "position",+ "width",+ "height",+ "margin",+ "padding",+ "gap",+ "flex-direction",+ "justify-content",+ "align-items",+ "color",+ "background-color",+ "font-size",+ "font-weight",+ "line-height",+ "border",+ "border-radius",+ "box-shadow",+ "opacity",+];++// Values that only restate the default are left out.+const boringStyle = /^(none|normal|auto|0px|0px none.*|rgba\(0, 0, 0, 0\))$/;+const styleDefaults = { position: "static", opacity: "1", "font-weight": "400" };+const layoutKeys = new Set(["gap", "flex-direction", "justify-content", "align-items"]);++const agentRefAttr = / data-agentflow-ref="[^"]*"/g;+const textLimit = 200;+const htmlLimit = 3000;++const clip = (value, limit) => {+ const text = String(value || "").replace(/\s+/g, " ").trim();+ return text.length > limit ? `${text.slice(0, limit)}…` : text;+};++const escapeIdent = (value) =>+ typeof CSS !== "undefined" && CSS.escape ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");++// Hashed or utility classes say little about where the element is; skip+// classes that look generated (css-1x2y3z, sc-abc123) and keep at most two.+const meaningfulClasses = (el) =>+ [...(el.classList || [])]+ .filter((c) => !/^(css|sc|jsx|svelte|emotion)-|^[a-z]{1,3}[-_]?(?=[0-9a-z]*[0-9])[0-9a-z]{5,}$/i.test(c) && c.length <= 40)+ .slice(0, 2);++function unique(doc, selector, el) {+ try {+ const found = doc.querySelectorAll(selector);+ return found.length === 1 && found[0] === el;+ } catch {+ return false;+ }+}++function step(el) {+ const tag = el.tagName.toLowerCase();+ const testId = el.getAttribute("data-testid") || el.getAttribute("data-test-id");+ if (testId) return `${tag}[data-testid="${testId.replace(/"/g, '\\"')}"]`;+ const classes = meaningfulClasses(el).map((c) => `.${escapeIdent(c)}`).join("");+ return `${tag}${classes}`;+}++function nth(el, base) {+ const parent = el.parentElement;+ if (!parent) return base;+ const same = [...parent.children].filter((c) => c.tagName === el.tagName);+ return same.length > 1 ? `${base}:nth-of-type(${same.indexOf(el) + 1})` : base;+}++// elementSelector returns a selector that matches only el in its document.+export function elementSelector(el) {+ const doc = el.ownerDocument;+ if (el.id && unique(doc, `#${escapeIdent(el.id)}`, el)) return `#${escapeIdent(el.id)}`;+ const parts = [];+ let node = el;+ while (node && node.nodeType === 1 && node !== doc.documentElement) {+ if (node !== el && node.id && unique(doc, `#${escapeIdent(node.id)}`, node)) {+ parts.unshift(`#${escapeIdent(node.id)}`);+ break;+ }+ parts.unshift(step(node));+ const selector = parts.join(" > ");+ if (unique(doc, selector, el)) return selector;+ parts[0] = nth(node, parts[0]);+ if (unique(doc, parts.join(" > "), el)) return parts.join(" > ");+ node = node.parentElement;+ }+ return parts.join(" > ");+}++// elementLabel is the short name shown on the composer chip.+export function elementLabel(el) {+ const tag = el.tagName.toLowerCase();+ const id = el.id ? `#${el.id}` : "";+ const cls = !id && meaningfulClasses(el)[0] ? `.${meaningfulClasses(el)[0]}` : "";+ const text = clip(el.innerText ?? el.textContent, 16);+ return `${tag}${id}${cls}${text ? ` 「${text}」` : ""}`;+}++function sourceOf(file, line, column) {+ if (!file) return "";+ return `${file}${line ? `:${line}` : ""}${line && column ? `:${column}` : ""}`;+}++function reactComponent(el) {+ const key = Object.keys(el).find((k) => k.startsWith("__reactFiber$") || k.startsWith("__reactInternalInstance$"));+ let fiber = key ? el[key] : null;+ let source = fiber?._debugSource || null;+ for (let i = 0; fiber && i < 40; i++, fiber = fiber.return) {+ source = source || fiber._debugSource || null;+ const type = fiber.type;+ if (typeof type === "function" || (type && typeof type === "object" && (type.render || type.type))) {+ const name = type.displayName || type.name || type.render?.displayName || type.render?.name || type.type?.name;+ if (name)+ return {+ framework: "React",+ component: name,+ source: sourceOf(source?.fileName, source?.lineNumber, source?.columnNumber),+ };+ }+ }+ return null;+}++function vueComponent(el) {+ const instance = el.__vueParentComponent;+ if (instance?.type) {+ const type = instance.type;+ const file = type.__file || "";+ const name = type.name || type.__name || (file ? file.split("/").pop().replace(/\.vue$/, "") : "");+ if (name || file) return { framework: "Vue", component: name, source: file };+ }+ const vm = el.__vue__;+ if (vm?.$options) {+ const file = vm.$options.__file || "";+ return { framework: "Vue", component: vm.$options.name || vm.$options._componentTag || "", source: file };+ }+ return null;+}++function svelteComponent(el) {+ const loc = el.__svelte_meta?.loc;+ if (!loc?.file) return null;+ return {+ framework: "Svelte",+ component: loc.file.split("/").pop().replace(/\.svelte$/, ""),+ source: sourceOf(loc.file, loc.line != null ? loc.line + 1 : 0, loc.column != null ? loc.column + 1 : 0),+ };+}++// componentOf finds the nearest framework component a dev build exposes on+// the element or its ancestors.+export function componentOf(el) {+ for (let node = el, depth = 0; node && node.nodeType === 1 && depth < 25; node = node.parentElement, depth++) {+ const found = reactComponent(node) || vueComponent(node) || svelteComponent(node);+ if (found) return found;+ }+ return null;+}++// describeElement collects what the Agent needs about el. url is the page+// address as the sandbox sees it (http://localhost:PORT/… or file://…).+export function describeElement(el, url) {+ const win = el.ownerDocument.defaultView;+ const rect = el.getBoundingClientRect();+ const computed = win.getComputedStyle(el);+ const styles = {};+ const layout = /flex|grid/.test(computed.getPropertyValue("display"));+ for (const key of styleKeys) {+ const value = computed.getPropertyValue(key);+ if (!layout && layoutKeys.has(key)) continue;+ if (!value || boringStyle.test(value) || styleDefaults[key] === value) continue;+ styles[key] = value;+ }+ // The Agent's page snapshots tag elements with data-agentflow-ref; they mean+ // nothing outside that snapshot.+ const html = (el.outerHTML || "").replace(agentRefAttr, "");+ const component = componentOf(el);+ return {+ url,+ title: clip(el.ownerDocument.title, 120),+ selector: elementSelector(el),+ tag: el.tagName.toLowerCase(),+ text: clip(el.innerText ?? el.textContent, textLimit),+ html: html.length > htmlLimit ? `${html.slice(0, htmlLimit)}…` : html,+ rect: {+ x: Math.round(rect.left),+ y: Math.round(rect.top),+ width: Math.round(rect.width),+ height: Math.round(rect.height),+ },+ viewport: { width: win.innerWidth, height: win.innerHeight },+ styles,+ ...(component?.component ? { component: component.component, framework: component.framework } : {}),+ ...(component?.source ? { source: component.source } : {}),+ };+}++let sequence = 0;++// elementReference wraps a description as a composer reference.+export function elementReference(el, url) {+ sequence += 1;+ return {+ type: "element",+ id: `el-${Date.now().toString(36)}-${sequence}`,+ name: elementLabel(el),+ element: describeElement(el, url),+ };+}++// inspectionReference wraps what the Agent browser reported for a picked+// element (internal/runtime/browser_inspect.go) as a composer reference.+export function inspectionReference(inspection) {+ sequence += 1;+ return {+ type: "element",+ id: `el-${Date.now().toString(36)}-${sequence}`,+ name: inspection.label || inspection.element?.tag || "元素",+ element: inspection.element,+ };+}++// elementTitle is the hover text of an element chip.+export function elementTitle(ref) {+ const e = ref?.element || {};+ return [+ e.component ? `${e.component}${e.source ? ` · ${e.source}` : ""}` : e.source,+ e.selector,+ e.url,+ ]+ .filter(Boolean)+ .join("\n");+}diff --git a/chatbot/src/styles.css b/chatbot/src/styles.cssindex 085319f1..04f201b5 100644--- a/chatbot/src/styles.css+++ b/chatbot/src/styles.css@@ -651,6 +651,17 @@ small {font-size: 11px;color: var(--muted);}+.message-references > span {+ display: inline-flex;+ align-items: center;+ gap: 3px;+ min-width: 0;+}+/* A page element picked in the native preview: the label is page text. */+.composer-attachment-file.is-element {+ max-width: 260px;+ white-space: nowrap;+}/* The send time stays visible; the copy and info buttons appear on hover. */.message-actions {display: flex;diff --git a/chatbot/tests/agent-browser.test.jsx b/chatbot/tests/agent-browser.test.jsxindex 2b241c3b..d12ccde7 100644--- a/chatbot/tests/agent-browser.test.jsx+++ b/chatbot/tests/agent-browser.test.jsx@@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";import AgentBrowser from "../src/AgentBrowser.jsx";import ArtifactPanel from "../src/ArtifactPanel.jsx";import { RichContent } from "../src/RichContent.jsx";-import { AgentBrowserLinks, agentBrowserTarget } from "../src/agent-browser-links.js";+import { AgentBrowserLinks, agentBrowserTarget, devServerAddress, devServerTarget } from "../src/agent-browser-links.js";class FakeSocket {static OPEN = 1;@@ -56,6 +56,26 @@ beforeEach(() => {});afterEach(() => vi.unstubAllGlobals());+describe("devServerTarget", () => {+ test("maps plain-HTTP execution machine addresses to a port and path", () => {+ expect(devServerTarget("http://localhost:5173/a/b?c=1#d")).toEqual({ port: 5173, path: "/a/b?c=1#d" });+ expect(devServerTarget("http://127.0.0.1:3000")).toEqual({ port: 3000, path: "/" });+ expect(devServerTarget("http://localhost/")).toEqual({ port: 80, path: "/" });+ for (const value of ["https://localhost:5173/", "http://example.com:5173/", "localhost:5173", "file:///", "file://host/a.html"])+ expect(devServerTarget(value)).toBeNull();+ expect(devServerAddress("run-1", 5173, { pathname: "/devpreview/run-1/5173/x", search: "?q", hash: "" })).toBe("http://localhost:5173/x?q");+ expect(devServerAddress("run-1", 5173, { pathname: "/x", search: "", hash: "" })).toBe("http://localhost:5173/x");+ });++ test("maps workspace files to the file channel", () => {+ expect(devServerTarget("file:///workspace/demo.html#top")).toEqual({ port: "file", path: "/workspace/demo.html#top" });+ expect(devServerTarget("file:///workspace/页面.html")).toEqual({ port: "file", path: "/workspace/%E9%A1%B5%E9%9D%A2.html" });+ expect(devServerAddress("run-1", "file", { pathname: "/devpreview/run-1/file/workspace/site/", search: "", hash: "" })).toBe(+ "file:///workspace/site/",+ );+ });+});+describe("agentBrowserTarget", () => {test("accepts only addresses on the execution machine", () => {expect(agentBrowserTarget("http://localhost:5173/sidebar-demo-2.html")).toBe("http://localhost:5173/sidebar-demo-2.html");@@ -83,7 +103,7 @@ describe("AgentBrowser", () => {test("streams the workspace browser and opens requested links", async () => {const handled = vi.fn();const { rerender, container } = render(- <AgentBrowser runId="run-1" request={{ id: "r1", url: "file:///workspace/demo.html" }} onRequestHandled={handled} />,+ <AgentBrowser runId="run-1" request={{ id: "r1", url: "https://example.org/demo.html" }} onRequestHandled={handled} />,);const socket = lastSocket();expect(socket.url).toBe(`ws://${location.host}/api/runs/run-1/browser/stream`);@@ -91,7 +111,7 @@ describe("AgentBrowser", () => {await waitFor(() => expect(fetchMock).toHaveBeenCalled());const [path, init] = fetchMock.mock.calls[0];expect(path).toBe("/api/runs/run-1/browser");- expect(JSON.parse(init.body)).toMatchObject({ operation: "navigate", url: "file:///workspace/demo.html", ensure: true });+ expect(JSON.parse(init.body)).toMatchObject({ operation: "navigate", url: "https://example.org/demo.html", ensure: true });expect(handled).toHaveBeenCalledWith("r1");act(() => {@@ -117,12 +137,12 @@ describe("AgentBrowser", () => {expect(socket.sent).toContainEqual({ operation: "activate", targetId: "t2" });rerender(- <AgentBrowser runId="run-1" request={{ id: "r2", url: "http://localhost:5173/" }} onRequestHandled={handled} />,+ <AgentBrowser runId="run-1" request={{ id: "r2", url: "https://example.com/" }} onRequestHandled={handled} />,);- expect(socket.sent).toContainEqual({ operation: "navigate", url: "http://localhost:5173/" });+ expect(socket.sent).toContainEqual({ operation: "navigate", url: "https://example.com/" });expect(handled).toHaveBeenCalledWith("r2");rerender(- <AgentBrowser runId="run-1" request={{ id: "r2", url: "http://localhost:5173/" }} onRequestHandled={handled} />,+ <AgentBrowser runId="run-1" request={{ id: "r2", url: "https://example.com/" }} onRequestHandled={handled} />,);expect(socket.sent.filter((m) => m.operation === "navigate")).toHaveLength(1);@@ -133,6 +153,72 @@ describe("AgentBrowser", () => {expect(fireEvent.keyDown(ime, { key: "Escape", code: "Escape", keyCode: 27 })).toBe(false);});+ test("opens dev server addresses natively and switches back to the screencast", async () => {+ const handled = vi.fn();+ const open = vi.spyOn(window, "open").mockImplementation(() => null);+ const { container } = render(+ <AgentBrowser runId="run-5" request={{ id: "n1", url: "http://localhost:5173/admin?tab=2" }} onRequestHandled={handled} />,+ );+ expect(handled).toHaveBeenCalledWith("n1");+ const frame = container.querySelector("iframe.agent-browser-frame");+ expect(frame).toHaveAttribute("src", "/devpreview/run-5/5173/admin?tab=2");+ expect(screen.getByLabelText("网址")).toHaveValue("http://localhost:5173/admin?tab=2");+ // No screencast while the page renders natively.+ expect(FakeSocket.instances).toHaveLength(0);+ fireEvent.click(screen.getByLabelText("在新窗口打开"));+ expect(open).toHaveBeenCalledWith(expect.stringContaining("/devpreview/run-5/5173/"), "_blank", "noopener");+ fireEvent.click(screen.getByLabelText("切换到 Agent 浏览器画面"));+ expect(container.querySelector("iframe.agent-browser-frame")).toBeNull();+ expect(lastSocket().url).toContain("/api/runs/run-5/browser/stream");+ });++ test("opens workspace files natively", async () => {+ const handled = vi.fn();+ const { container } = render(+ <AgentBrowser runId="run-6" request={{ id: "f1", url: "file:///workspace/demo.html" }} onRequestHandled={handled} />,+ );+ expect(handled).toHaveBeenCalledWith("f1");+ expect(container.querySelector("iframe.agent-browser-frame")).toHaveAttribute("src", "/devpreview/run-6/file/workspace/demo.html");+ expect(screen.getByLabelText("网址")).toHaveValue("file:///workspace/demo.html");+ expect(screen.getByText("原生预览 · 工作区文件")).toBeInTheDocument();+ expect(FakeSocket.instances).toHaveLength(0);+ });++ test("the panel in use takes the shared viewport back and never slides the frame", () => {+ const { container } = render(<AgentBrowser runId="run-5" />);+ const socket = lastSocket();+ const stage = container.querySelector(".agent-browser-stage");+ const img = container.querySelector(".agent-browser-screen");+ // A tall panel; another viewer last sized the browser to 1024×866.+ stage.getBoundingClientRect = () => ({ left: 0, top: 0, width: 600, height: 1200, right: 600, bottom: 1200, x: 0, y: 0 });+ img.getBoundingClientRect = () => ({ left: 0, top: 0, width: 600, height: 507, right: 600, bottom: 507, x: 0, y: 0 });+ act(() => {+ socket.open();+ socket.state({ ...live, width: 1024, height: 866 });+ });+ const resizes = () => socket.sent.filter((m) => m.operation === "resize");+ fireEvent.wheel(stage, { deltaY: 120 });+ expect(resizes()).toEqual([expect.objectContaining({ width: 1024, height: 2000 })]);+ expect(img.style.transform).toBe("");+ // Not again while the resize is on its way.+ fireEvent.wheel(stage, { deltaY: 120 });+ expect(resizes()).toHaveLength(1);+ });++ test("a panel whose size already fits leaves the viewport alone", () => {+ const { container } = render(<AgentBrowser runId="run-6" />);+ const socket = lastSocket();+ const stage = container.querySelector(".agent-browser-stage");+ stage.getBoundingClientRect = () => ({ left: 0, top: 0, width: 1024, height: 866, right: 1024, bottom: 866, x: 0, y: 0 });+ act(() => {+ socket.open();+ // Frames count the page's 15px scrollbar the resize left out.+ socket.state({ ...live, width: 1039, height: 866 });+ });+ fireEvent.wheel(stage, { deltaY: 120 });+ expect(socket.sent.filter((m) => m.operation === "resize")).toHaveLength(0);+ });+test("project viewers watch the browser without driving it", async () => {const { container } = render(<AgentBrowser runId="run-2" />);const socket = lastSocket();@@ -242,6 +328,44 @@ describe("side panel browser tab", () => {expect(FakeSocket.instances).toHaveLength(0);});+ test("a compact panel goes back to the chat after adding an element", () => {+ // A narrow window: the panel covers the conversation.+ const measure = vi+ .spyOn(HTMLElement.prototype, "getBoundingClientRect")+ .mockReturnValue({ left: 0, top: 0, width: 600, height: 800, right: 600, bottom: 800, x: 0, y: 0 });+ const onAddElement = vi.fn();+ const onClose = vi.fn();+ const props = {+ view: "browser",+ workspaceRunId: "run-9",+ browserRequest: { id: "b1", url: "http://localhost:5173/" },+ onBrowserRequestHandled: vi.fn(),+ onAddElement,+ onClose,+ };+ try {+ const { container } = render(panel(props));+ expect(screen.getByRole("button", { name: "返回聊天" })).toBeInTheDocument();+ const frame = container.querySelector("iframe.agent-browser-frame");+ const doc = frame.contentDocument;+ doc.open();+ doc.write(`<!doctype html><html><body><button>提交</button></body></html>`);+ doc.close();+ act(() => {+ fireEvent.load(frame);+ });+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));+ act(() => {+ doc.querySelector("button").dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true }));+ });+ fireEvent.click(screen.getByText("加入对话框"));+ expect(onAddElement).toHaveBeenCalledWith(expect.objectContaining({ type: "element" }));+ expect(onClose).toHaveBeenCalled();+ } finally {+ measure.mockRestore();+ }+ });+test("shows the workspace browser", async () => {const onViewChange = vi.fn();const { rerender } = render(panel({ view: "artifacts", workspaceRunId: "run-9", onViewChange }));diff --git a/chatbot/tests/devserver-cookie.test.js b/chatbot/tests/devserver-cookie.test.jsnew file mode 100644index 00000000..27fa2118--- /dev/null+++ b/chatbot/tests/devserver-cookie.test.js@@ -0,0 +1,65 @@+// The dev server intercept script (internal/preview/devserver_intercept.js)+// runs in the user's browser under /devpreview/{run}/{port}/. Its cookie+// setter must keep a page's cookies inside that preview: same port in another+// task, and the platform itself, never see them.+import { readFileSync } from "node:fs";+import { resolve } from "node:path";+import { afterEach, expect, test } from "vitest";++const source = readFileSync(resolve(__dirname, "../../internal/preview/devserver_intercept.js"), "utf8");+const raw = Object.getOwnPropertyDescriptor(Document.prototype, "cookie");+const browserCookies = () => raw.get.call(document);++function openPreview(run, port) {+ history.replaceState(null, "", `/devpreview/${run}/${port}/app/`);+ delete window.__AGENTFLOW_DEV_SERVER__;+ delete document.cookie;+ const prefix = `/devpreview/${run}/${port}`;+ // As preview.DevServerCookiePrefix fills it in.+ const cookiePrefix = `af_devpreview_${run}_${port}_`;+ (0, eval)(+ source+ .replace("__AF_PREFIX__", JSON.stringify(prefix))+ .replace("__AF_PORT__", String(port))+ .replace("__AF_COOKIE__", JSON.stringify(cookiePrefix)),+ );+}++afterEach(() => {+ history.replaceState(null, "", "/");+});++test("a page's root-path cookie stays inside its own preview", () => {+ openPreview("run-a", 3000);+ document.cookie = "sid=secret; Path=/; Domain=localhost";+ document.cookie = "theme=dark";+ expect(document.cookie).toContain("sid=secret");+ expect(document.cookie).toContain("theme=dark");+ expect(browserCookies()).toContain("af_devpreview_run-a_3000_sid=secret");++ // The platform root does not carry it…+ history.replaceState(null, "", "/chatbot/");+ expect(browserCookies()).not.toContain("sid=secret");+ // …and neither does another task serving the same port.+ openPreview("run-b", 3000);+ expect(browserCookies()).not.toContain("sid=secret");+ expect(document.cookie).not.toContain("sid");+ document.cookie = "sid=other; Path=/";+ expect(document.cookie).toBe("sid=other");++ // Back in the first preview its own cookie is intact.+ openPreview("run-a", 3000);+ expect(document.cookie).toContain("sid=secret");+ expect(document.cookie).not.toContain("sid=other");+});++test("paths already inside the preview are kept", () => {+ openPreview("run-c", 4000);+ document.cookie = "a=1; Path=/devpreview/run-c/4000/app";+ document.cookie = "b=2; path=/api";+ history.replaceState(null, "", "/devpreview/run-c/4000/api/x");+ expect(document.cookie).toContain("b=2");+ expect(document.cookie).not.toContain("a=1");+ history.replaceState(null, "", "/devpreview/run-c/4000/app/x");+ expect(document.cookie).toContain("a=1");+});diff --git a/chatbot/tests/element-picker.test.jsx b/chatbot/tests/element-picker.test.jsxnew file mode 100644index 00000000..23c05346--- /dev/null+++ b/chatbot/tests/element-picker.test.jsx@@ -0,0 +1,246 @@+import React from "react";+import { readFileSync } from "node:fs";+import { resolve } from "node:path";+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";+import { afterEach, describe, expect, test, vi } from "vitest";+import AgentBrowser from "../src/AgentBrowser.jsx";+import { componentOf, describeElement, elementLabel, elementSelector, elementTitle } from "../src/element-context.js";++function page(html) {+ document.body.innerHTML = html;+ return document;+}++describe("element context", () => {+ test("builds selectors that match only the element", () => {+ const doc = page(`+ <main id="app">+ <ul class="list"><li class="item">a</li><li class="item">b</li></ul>+ <button data-testid="save" class="css-1x2y3z primary">保存</button>+ <section><p>one</p><p>two</p></section>+ </main>`);+ const second = doc.querySelectorAll("li")[1];+ const save = doc.querySelector("button");+ const two = doc.querySelectorAll("p")[1];+ for (const el of [second, save, two, doc.getElementById("app")]) {+ const selector = elementSelector(el);+ expect(doc.querySelectorAll(selector)).toHaveLength(1);+ expect(doc.querySelector(selector)).toBe(el);+ }+ expect(elementSelector(save)).toBe('button[data-testid="save"]');+ expect(elementSelector(save)).not.toContain("css-1x2y3z");+ expect(elementLabel(save)).toBe("button.primary 「保存」");+ });++ test("finds the component a dev build exposes", () => {+ const doc = page(`<div id="root"><span>hi</span></div><b>vue</b><i>svelte</i>`);+ function SaveButton() {}+ doc.querySelector("span")["__reactFiber$abc"] = {+ type: "span",+ _debugSource: { fileName: "src/Save.tsx", lineNumber: 12, columnNumber: 3 },+ return: { type: SaveButton, return: null },+ };+ expect(componentOf(doc.querySelector("span"))).toEqual({+ framework: "React",+ component: "SaveButton",+ source: "src/Save.tsx:12:3",+ });+ doc.querySelector("b").__vueParentComponent = { type: { __name: "Toolbar", __file: "src/Toolbar.vue" } };+ expect(componentOf(doc.querySelector("b"))).toEqual({ framework: "Vue", component: "Toolbar", source: "src/Toolbar.vue" });+ doc.querySelector("i").__svelte_meta = { loc: { file: "src/Card.svelte", line: 4, column: 0 } };+ expect(componentOf(doc.querySelector("i"))).toEqual({ framework: "Svelte", component: "Card", source: "src/Card.svelte:5:1" });+ expect(componentOf(doc.getElementById("root"))).toBeNull();+ });++ test("describes the element for the Agent", () => {+ const doc = page(`<button id="go" style="color: red">${"长".repeat(400)}</button>`);+ const info = describeElement(doc.getElementById("go"), "http://localhost:5173/x");+ expect(info).toMatchObject({ url: "http://localhost:5173/x", selector: "#go", tag: "button" });+ expect(info.text.length).toBeLessThanOrEqual(201);+ expect(info.styles.color).toBe("rgb(255, 0, 0)");+ expect(elementTitle({ element: { ...info, component: "Go", source: "src/Go.tsx:1" } })).toBe(+ "Go · src/Go.tsx:1\n#go\nhttp://localhost:5173/x",+ );+ });+});++describe("element picking in the native preview", () => {+ test("picks an element in the frame and adds it to the composer", async () => {+ const add = vi.fn();+ const { container } = render(+ <AgentBrowser runId="run-7" request={{ id: "p1", url: "http://localhost:5173/" }} onAddElement={add} />,+ );+ const frame = container.querySelector("iframe.agent-browser-frame");+ const doc = frame.contentDocument;+ doc.open();+ doc.write(`<!doctype html><html><head></head><body><main><button class="primary">提交</button><a href="#x">link</a></main></body></html>`);+ doc.close();+ act(() => {+ fireEvent.load(frame);+ });+ // The button sits at the end of the address bar.+ expect(screen.getByRole("button", { name: "标记元素" }).closest(".agent-browser-address")).not.toBeNull();+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "true");+ expect(doc.querySelector("style[data-af-pick]")).not.toBeNull();+ const button = doc.querySelector("button");+ const pageClick = vi.fn();+ button.addEventListener("click", pageClick);+ act(() => {+ button.dispatchEvent(new MouseEvent("mousemove", { bubbles: true }));+ });+ expect(container.querySelector(".agent-browser-pick-outline")).not.toBeNull();+ act(() => {+ button.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true }));+ });+ // The page never sees clicks made while picking.+ expect(pageClick).not.toHaveBeenCalled();+ expect(doc.querySelector("style[data-af-pick]")).toBeNull();+ expect(screen.getByRole("dialog", { name: "选中的元素" })).toHaveTextContent("button.primary 「提交」");+ fireEvent.click(screen.getByText("加入对话框"));+ expect(add).toHaveBeenCalledTimes(1);+ expect(add.mock.calls[0][0]).toMatchObject({+ type: "element",+ name: "button.primary 「提交」",+ element: { tag: "button", selector: "button.primary", text: "提交" },+ });+ expect(screen.queryByRole("dialog", { name: "选中的元素" })).toBeNull();++ // Escape leaves picking mode.+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));+ act(() => {+ doc.dispatchEvent(new KeyboardEvent("keydown", { key: "Escape", bubbles: true }));+ });+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "false");+ });++ test("offers picking only when the composer can take elements", () => {+ render(<AgentBrowser runId="run-8" request={{ id: "p2", url: "http://localhost:5173/" }} />);+ expect(screen.queryByRole("button", { name: "标记元素" })).toBeNull();+ });+});++class PickSocket {+ static OPEN = 1;+ static last = null;+ constructor() {+ this.readyState = 0;+ this.sent = [];+ PickSocket.last = this;+ }+ send(data) {+ this.sent.push(JSON.parse(data));+ }+ close() {+ this.readyState = 3;+ }+ open() {+ this.readyState = PickSocket.OPEN;+ this.onopen?.();+ }+ state(state) {+ this.onmessage?.({ data: JSON.stringify({ type: "state", state }) });+ }+}++const rect = (left, top, width, height) => () => ({ left, top, width, height, right: left + width, bottom: top + height, x: left, y: top });++describe("element picking on the Agent browser screencast", () => {+ afterEach(() => vi.unstubAllGlobals());++ test("inspects the page over CDP and adds the picked element", async () => {+ vi.stubGlobal("WebSocket", PickSocket);+ // jsdom has no PointerEvent; without it pointer coordinates are dropped.+ if (!window.PointerEvent) vi.stubGlobal("PointerEvent", class extends MouseEvent {});+ const inspections = [];+ const save = { label: "button#save 「保存」", rect: { x: 100, y: 50, width: 200, height: 40 }, viewport: { width: 1280, height: 800 } };+ const fetchMock = vi.fn(async (_path, init) => {+ const body = JSON.parse(init.body);+ if (body.operation === "inspect") inspections.push(body);+ const inspect = body.operation === "inspect" ? { ...save, ...(body.detail ? { element: { tag: "button", selector: "#save", url: "http://localhost:3000/" } } : {}) } : undefined;+ return { ok: true, status: 200, json: async () => ({ available: true, inspect }) };+ });+ vi.stubGlobal("fetch", fetchMock);+ const add = vi.fn();+ const { container } = render(<AgentBrowser runId="run-9" onAddElement={add} />);+ const socket = PickSocket.last;+ act(() => {+ socket.open();+ socket.state({ available: true, url: "http://localhost:3000/", width: 1280, height: 800, tabs: [] });+ });+ const stage = container.querySelector(".agent-browser-stage");+ const img = container.querySelector(".agent-browser-screen");+ // The panel shows the 1280px page at half size, 10px below the stage top.+ stage.getBoundingClientRect = rect(0, 0, 640, 500);+ img.getBoundingClientRect = rect(0, 10, 640, 400);++ expect(screen.getByRole("button", { name: "标记元素" }).closest(".agent-browser-address")).not.toBeNull();+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));+ const capture = screen.getByTestId("agent-browser-pick-capture");+ const before = socket.sent.length;+ fireEvent.pointerMove(capture, { clientX: 160, clientY: 110 });+ await waitFor(() => expect(container.querySelector(".agent-browser-pick-outline")).not.toBeNull());+ expect(inspections[0]).toEqual({ operation: "inspect", point: { x: 0.25, y: 0.25 }, detail: false });+ const outline = container.querySelector(".agent-browser-pick-outline");+ expect(outline.style.left).toBe("50px");+ expect(outline.style.top).toBe("35px");+ expect(outline.style.width).toBe("100px");+ expect(outline.querySelector(".agent-browser-pick-label")).toHaveTextContent("button#save 「保存」");+ expect(outline.querySelector(".agent-browser-pick-size")).toHaveTextContent("100×20");++ fireEvent.click(capture, { clientX: 160, clientY: 110 });+ await waitFor(() => expect(screen.getByRole("button", { name: /加入对话框/ })).toBeEnabled());+ expect(inspections.at(-1)).toMatchObject({ detail: true });+ // Hovering and picking never reach the page as input.+ expect(socket.sent.slice(before).filter((m) => m.operation === "input")).toHaveLength(0);+ fireEvent.click(screen.getByRole("button", { name: /加入对话框/ }));+ expect(add).toHaveBeenCalledWith(+ expect.objectContaining({ type: "element", name: "button#save 「保存」", element: expect.objectContaining({ selector: "#save" }) }),+ );+ expect(screen.queryByRole("dialog", { name: "选中的元素" })).toBeNull();++ // Escape leaves picking before the screencast forwards it to the page.+ fireEvent.click(screen.getByRole("button", { name: "标记元素" }));+ const sent = socket.sent.length;+ fireEvent.keyDown(container.querySelector(".agent-browser-ime"), { key: "Escape", code: "Escape", keyCode: 27 });+ expect(screen.getByRole("button", { name: "标记元素" })).toHaveAttribute("aria-pressed", "false");+ expect(socket.sent).toHaveLength(sent);+ });++ test("the runtime inspect script describes the element at a point", () => {+ const go = readFileSync(resolve(__dirname, "../../internal/runtime/browser_inspect.go"), "utf8");+ const script = go.match(/const inspectScript = `([\s\S]*?)`/)[1];+ document.body.innerHTML = `<main id="app"><button class="primary" data-testid="save" data-agentflow-ref="e14">保存<i data-agentflow-ref="e15"></i></button></main>`;+ const button = document.querySelector("button");+ button.getBoundingClientRect = rect(10, 20, 80, 30);+ button["__reactFiber$x"] = { type: "button", _debugSource: { fileName: "src/Save.tsx", lineNumber: 4 }, return: { type: function SaveButton() {} } };+ const hits = [];+ document.elementFromPoint = (x, y) => {+ hits.push([x, y]);+ return button;+ };+ try {+ const out = (0, eval)(script)(0.5, 0.25, true);+ expect(hits[0]).toEqual([innerWidth * 0.5, innerHeight * 0.25]);+ expect(out.label).toBe("button.primary 「保存」");+ expect(out.rect).toEqual({ x: 10, y: 20, width: 80, height: 30 });+ // Same description as the native preview builds in the panel.+ const native = describeElement(button, location.href);+ expect(out.element).toMatchObject({+ selector: native.selector,+ tag: "button",+ text: "保存",+ component: "SaveButton",+ framework: "React",+ source: "src/Save.tsx:4",+ });+ expect(out.element.selector).toBe('button[data-testid="save"]');+ // The Agent's snapshot refs are dropped on both sides.+ expect(out.element.html).toBe('<button class="primary" data-testid="save">保存<i></i></button>');+ expect(native.html).toBe(out.element.html);+ expect((0, eval)(script)(0.5, 0.25, false).element).toBeUndefined();+ } finally {+ delete document.elementFromPoint;+ }+ });+});diff --git a/internal/domain/assistant.go b/internal/domain/assistant.goindex 0756aebd..4bbaf7ba 100644--- a/internal/domain/assistant.go+++ b/internal/domain/assistant.go@@ -76,6 +76,34 @@ type AssistantReference struct {Name string `json:"name,omitempty"`ProjectID string `json:"projectId,omitempty"`ProjectName string `json:"projectName,omitempty"`+ // Element describes a page element the user picked in the native+ // preview (Type "element"); it is the user's own context, not an object+ // the platform looks up.+ Element *AssistantPageElement `json:"element,omitempty"`+}++// AssistantPageElement is a page element picked in the native preview.+type AssistantPageElement struct {+ URL string `json:"url,omitempty"`+ Title string `json:"title,omitempty"`+ Selector string `json:"selector,omitempty"`+ Tag string `json:"tag,omitempty"`+ Text string `json:"text,omitempty"`+ HTML string `json:"html,omitempty"`+ Rect *ElementRect `json:"rect,omitempty"`+ Viewport *ElementRect `json:"viewport,omitempty"`+ Styles map[string]string `json:"styles,omitempty"`+ Component string `json:"component,omitempty"`+ Framework string `json:"framework,omitempty"`+ Source string `json:"source,omitempty"`+}++// ElementRect is a box in CSS pixels; a viewport only sets its size.+type ElementRect struct {+ X float64 `json:"x,omitempty"`+ Y float64 `json:"y,omitempty"`+ Width float64 `json:"width"`+ Height float64 `json:"height"`}type AssistantAttachment struct {diff --git a/internal/domain/runner_workspace.go b/internal/domain/runner_workspace.goindex 49651103..3fd99608 100644--- a/internal/domain/runner_workspace.go+++ b/internal/domain/runner_workspace.go@@ -141,6 +141,50 @@ type BrowserRequest struct {WithSnapshot bool `json:"withSnapshot,omitempty"`// Ensure starts the run browser when none is running yet.Ensure bool `json:"ensure,omitempty"`+ // HTTP carries one request to a dev server port inside the run sandbox+ // (operation "http"), so the user's own browser renders the page natively.+ HTTP *BrowserHTTPRequest `json:"http,omitempty"`+ // File is the absolute path of a workspace file as the run browser sees+ // it (operation "file"), served so the user's browser renders file://+ // pages natively. The response comes back in BrowserState.HTTP.+ File string `json:"file,omitempty"`+ // Point is where the panel inspects the page (operation "inspect"), as a+ // fraction of the viewport like input events; Detail asks for the full+ // element description rather than just its box and label.+ Point *BrowserPoint `json:"point,omitempty"`+ Detail bool `json:"detail,omitempty"`+}++// BrowserPoint is a position as a fraction of the viewport (0..1).+type BrowserPoint struct {+ X float64 `json:"x"`+ Y float64 `json:"y"`+}++// BrowserInspection is the element under a point of the Agent browser page.+// Rect is in CSS pixels of the top-level viewport.+type BrowserInspection struct {+ Label string `json:"label"`+ Rect ElementRect `json:"rect"`+ Viewport ElementRect `json:"viewport"`+ Element *AssistantPageElement `json:"element,omitempty"`+}++// BrowserHTTPRequest is one proxied request to http://localhost:Port in the+// run's network namespace. Path keeps the query string.+type BrowserHTTPRequest struct {+ Port int `json:"port"`+ Method string `json:"method"`+ Path string `json:"path"`+ Header map[string][]string `json:"header,omitempty"`+ Body []byte `json:"body,omitempty"`+}++// BrowserHTTPResponse is the dev server's answer to a BrowserHTTPRequest.+type BrowserHTTPResponse struct {+ Status int `json:"status"`+ Header map[string][]string `json:"header,omitempty"`+ Body []byte `json:"body,omitempty"`}// BrowserInputEvent coordinates are normalized to the frame (0..1).@@ -169,19 +213,29 @@ type BrowserTab struct {}type BrowserState struct {- Available bool `json:"available"`- Reason string `json:"reason,omitempty"`- Kind string `json:"kind,omitempty"`- Seq int64 `json:"seq"`- Frame string `json:"frame,omitempty"`- Width float64 `json:"width,omitempty"`- Height float64 `json:"height,omitempty"`- URL string `json:"url,omitempty"`- Title string `json:"title,omitempty"`- CanGoBack bool `json:"canGoBack,omitempty"`- CanGoForward bool `json:"canGoForward,omitempty"`- Tabs []BrowserTab `json:"tabs,omitempty"`- Snapshot *BrowserSnapshot `json:"snapshot,omitempty"`+ Available bool `json:"available"`+ Reason string `json:"reason,omitempty"`+ Kind string `json:"kind,omitempty"`+ Seq int64 `json:"seq"`+ Frame string `json:"frame,omitempty"`+ Width float64 `json:"width,omitempty"`+ Height float64 `json:"height,omitempty"`+ URL string `json:"url,omitempty"`+ Title string `json:"title,omitempty"`+ // ScrollX/Y and ContentWidth/Height (CSS px) bound how far the panel may+ // slide a frame ahead of the page when it predicts a scroll.+ ScrollX float64 `json:"scrollX,omitempty"`+ ScrollY float64 `json:"scrollY,omitempty"`+ ContentWidth float64 `json:"contentWidth,omitempty"`+ ContentHeight float64 `json:"contentHeight,omitempty"`+ CanGoBack bool `json:"canGoBack,omitempty"`+ CanGoForward bool `json:"canGoForward,omitempty"`+ Tabs []BrowserTab `json:"tabs,omitempty"`+ Snapshot *BrowserSnapshot `json:"snapshot,omitempty"`+ // HTTP answers operation "http"; Reason explains a missing answer.+ HTTP *BrowserHTTPResponse `json:"http,omitempty"`+ // Inspect answers operation "inspect"; nil when nothing is at the point.+ Inspect *BrowserInspection `json:"inspect,omitempty"`}// BrowserSnapshot is a text view of the active page for language models.diff --git a/internal/httpapi/assistant_browser.go b/internal/httpapi/assistant_browser.goindex 035d7180..6614454f 100644--- a/internal/httpapi/assistant_browser.go+++ b/internal/httpapi/assistant_browser.go@@ -45,7 +45,7 @@ func mainBotBrowserTools(s *AgentToolSession) []agentToolDescriptor {{"click", "用真实鼠标点击快照中的元素。返回点击后的新快照(ref 会刷新)。", `{"type":"object","properties":{"ref":{"type":"string"}},"required":["ref"],"additionalProperties":false}`},{"type", "清空并在快照中的输入元素里输入文本;submit=true 时随后按回车。返回新快照。", `{"type":"object","properties":{"ref":{"type":"string"},"text":{"type":"string"},"submit":{"type":"boolean"}},"required":["ref","text"],"additionalProperties":false}`},{"press", "在当前焦点按键:Enter、Tab、Escape、Backspace、Delete、Space、ArrowUp/Down/Left/Right、PageUp、PageDown、Home、End。返回新快照。", `{"type":"object","properties":{"key":{"type":"string"}},"required":["key"],"additionalProperties":false}`},- {"screenshot", "截取当前标签页画面(图像),用于确认布局、样式或快照无法表达的内容。截图会自动附在本轮回复里给用户看(显示在正文下方,每轮保留最近 4 张)。用户还能在对话侧栏「浏览器」页签实时查看和操作这个浏览器:回复里写出页面地址(file:///workspace/… 或 http://localhost:端口/…),用户点击即在该页签打开。", `{"type":"object","properties":{},"additionalProperties":false}`},+ {"screenshot", "截取当前标签页画面(图像),用于确认布局、样式或快照无法表达的内容。截图会自动附在本轮回复里给用户看(显示在正文下方,每轮保留最近 4 张)。用户还能在对话侧栏「浏览器」页签实时查看和操作这个浏览器:回复里写出页面地址(file:///workspace/… 或 http://localhost:端口/…),用户点击即在该页签打开;工作区文件和 http://localhost:端口 的开发服务器都由平台转发、在用户自己的浏览器里原生渲染(可在新窗口打开;开发服务器暂不支持热更新,改动后让用户刷新)。", `{"type":"object","properties":{},"additionalProperties":false}`},{"tabs", "管理标签页:list 列出,switch 切换到 tabId,new 新建,close 关闭 tabId(缺省当前页)。", `{"type":"object","properties":{"action":{"type":"string","enum":["list","switch","new","close"]},"tabId":{"type":"string"}},"required":["action"],"additionalProperties":false}`},}out := make([]agentToolDescriptor, 0, len(descriptors))diff --git a/internal/httpapi/assistant_page_element.go b/internal/httpapi/assistant_page_element.gonew file mode 100644index 00000000..1f2be568--- /dev/null+++ b/internal/httpapi/assistant_page_element.go@@ -0,0 +1,66 @@+package httpapi++import (+ "encoding/json"+ "fmt"+ "regexp"+ "sort"+ "strings"++ "agentflow/internal/domain"+)++// Page elements the user picks in the native preview ride along as+// references of type "element". Unlike other references they are not looked+// up: the browser describes the element, so the description is bounded and+// framed as the user's context here.++const pageElementReference = "element"++var pageElementIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)++// pageElementStyleLimit bounds the computed styles kept per element.+const pageElementStyleLimit = 32++func pageElementContext(ref domain.AssistantReference) (domain.AssistantReference, string, error) {+ if !pageElementIDPattern.MatchString(ref.ID) || ref.Element == nil {+ return ref, "", fmt.Errorf("页面元素引用无效")+ }+ in := ref.Element+ e := &domain.AssistantPageElement{+ URL: truncateRunes(strings.TrimSpace(in.URL), 2000),+ Title: truncateRunes(in.Title, 200),+ Selector: truncateRunes(in.Selector, 1000),+ Tag: truncateRunes(in.Tag, 40),+ Text: truncateRunes(in.Text, 500),+ HTML: truncateRunes(in.HTML, 4000),+ Rect: in.Rect,+ Viewport: in.Viewport,+ Component: truncateRunes(in.Component, 200),+ Framework: truncateRunes(in.Framework, 40),+ Source: truncateRunes(in.Source, 500),+ }+ if len(in.Styles) > 0 {+ keys := make([]string, 0, len(in.Styles))+ for k := range in.Styles {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ e.Styles = map[string]string{}+ for _, k := range keys {+ if len(e.Styles) == pageElementStyleLimit {+ break+ }+ e.Styles[truncateRunes(k, 60)] = truncateRunes(in.Styles[k], 200)+ }+ }+ name := strings.TrimSpace(truncateRunes(ref.Name, 80))+ if name == "" {+ name = e.Tag+ }+ canonical := domain.AssistantReference{Type: pageElementReference, ID: ref.ID, Name: name, Element: e}+ raw, _ := json.Marshal(e)+ text := "用户在对话侧栏的原生预览里选中的页面元素「" + name + "」,说明这一轮要改的位置。" ++ "有 source/component 时先按它找代码;需要核对时用 workbench_browser 打开 url 并按 selector 查看。元素信息:" + string(raw)+ return canonical, text, nil+}diff --git a/internal/httpapi/assistant_page_element_test.go b/internal/httpapi/assistant_page_element_test.gonew file mode 100644index 00000000..68253f96--- /dev/null+++ b/internal/httpapi/assistant_page_element_test.go@@ -0,0 +1,53 @@+package httpapi++import (+ "bytes"+ "strings"+ "testing"++ "agentflow/internal/domain"+)++func TestPickedPageElementsBecomeTurnContext(t *testing.T) {+ h, u := workbenchFixture(t)+ h.repo.PutRun(domain.Run{ID: "task", Status: "waiting", ExecutionContext: domain.TelemetryContext{ProjectID: "workbench"}, Nodes: map[string]*domain.NodeRun{}})+ element := func() *domain.AssistantPageElement {+ return &domain.AssistantPageElement{+ URL: "http://localhost:5173/settings",+ Selector: "#save",+ Tag: "button",+ Text: "保存",+ HTML: strings.Repeat("<b>x</b>", 2000),+ Component: "SaveButton",+ Source: "src/Settings.tsx:42",+ Styles: map[string]string{"color": "rgb(0, 0, 0)"},+ }+ }+ c, _ := workbenchContext(u, "POST", "/", bytes.NewBuffer(nil))+ // Task conversations accept them too: they are not task objects.+ req := assistantRequest{ProjectID: "workbench", RunID: "task", Messages: []assistantMessage{{Role: "user", Content: "改成次要按钮", References: []domain.AssistantReference{+ {Type: "element", ID: "el-1", Name: "button#save 「保存」", ProjectID: "forged", Element: element()},+ }}}}+ if err := h.prepareWorkbenchContext(c, &req, u); err != nil {+ t.Fatal(err)+ }+ ref := req.Messages[0].References[0]+ if ref.ProjectID != "" || ref.Element == nil || len([]rune(ref.Element.HTML)) > 4001 {+ t.Fatalf("canonical %+v", ref)+ }+ for _, want := range []string{"原生预览", "button#save 「保存」", "src/Settings.tsx:42", "SaveButton", "http://localhost:5173/settings", `"selector":"#save"`} {+ if !strings.Contains(req.contextText, want) {+ t.Fatalf("context misses %q: %s", want, req.contextText)+ }+ }++ for _, bad := range []domain.AssistantReference{+ {Type: "element", ID: "el-2"},+ {Type: "element", ID: "../x", Element: element()},+ } {+ req := assistantRequest{ProjectID: "workbench", Messages: []assistantMessage{{Role: "user", Content: "x", References: []domain.AssistantReference{bad}}}}+ if err := h.prepareWorkbenchContext(c, &req, u); err == nil {+ t.Fatalf("accepted %+v", bad)+ }+ }+}diff --git a/internal/httpapi/assistant_workbench.go b/internal/httpapi/assistant_workbench.goindex c2197dfd..bf109443 100644--- a/internal/httpapi/assistant_workbench.go+++ b/internal/httpapi/assistant_workbench.go@@ -278,6 +278,15 @@ func (h *assistantHandlers) prepareWorkbenchContext(c *gin.Context, req *assista}continue}+ if ref.Type == pageElementReference {+ canonical, text, err := pageElementContext(ref)+ if err != nil {+ return err+ }+ m.References[j] = canonical+ context.WriteString("引用(数据):" + text + "\n")+ continue+ }if req.RunID != "" {r, ok := h.repo.RunState(req.RunID)if !ok || !h.taskReferenceAllowed(r, ref.Type, ref.ID) {diff --git a/internal/httpapi/browser.go b/internal/httpapi/browser.goindex a574c934..b9c5a8d7 100644--- a/internal/httpapi/browser.go+++ b/internal/httpapi/browser.go@@ -66,6 +66,10 @@ func (h *browserHandlers) browser(c *gin.Context) {// executor is reported as an unavailable state rather than an error.func (h *browserHandlers) dispatch(ctx context.Context, run domain.Run, req domain.BrowserRequest) (json.RawMessage, string) {timeout := 15 * time.Second+ if req.Operation == "http" || req.Operation == "file" {+ // A dev server compiles a route on its first request.+ timeout = devServerTimeout+ }if req.Ensure {// The first browser of a host may still need its sidecar image built.timeout = 4 * time.Minutediff --git a/internal/httpapi/browser_devserver.go b/internal/httpapi/browser_devserver.gonew file mode 100644index 00000000..2d50310f--- /dev/null+++ b/internal/httpapi/browser_devserver.go@@ -0,0 +1,501 @@+package httpapi++import (+ "context"+ "encoding/json"+ "fmt"+ "html"+ "io"+ "net/http"+ "net/url"+ "path"+ "regexp"+ "strconv"+ "strings"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/preview"++ "github.com/gin-gonic/gin"+ "github.com/gorilla/websocket"+)++// Dev server preview: the user's own browser renders a dev server running in+// the run sandbox, under /devpreview/{runID}/{port}/ on the platform origin,+// so the page is native (input, scrolling, devtools) instead of a screencast.+// Like build previews it lives under a path prefix: markup, module imports and+// stylesheets are rewritten under it, the HTML gets the history shim and an+// intercept script, and requests a page still sends to root paths are+// redirected back under the prefix by Referer. Each request travels the+// browser operation channel ("http") to the executor hosting the sandbox.+// Workspace files the Agent opens as file:// are served the same way under+// /devpreview/{runID}/file/ (operation "file"), as they are, unrewritten.++const (+ devServerPathPrefix = "/devpreview/"+ devServerTimeout = 90 * time.Second+ devServerRequestLimit = 8 << 20+)++var devServerRunIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)++func devServerPrefix(runID string, port int) string {+ return devServerPathPrefix + runID + "/" + strconv.Itoa(port)+}++// parseDevServerPath splits /devpreview/{runID}/{port}/{rest}. rest keeps its+// escaping; slash reports whether anything, even "/", follows the port.+func parseDevServerPath(escaped string) (runID string, port int, rest string, slash, ok bool) {+ tail, found := strings.CutPrefix(escaped, devServerPathPrefix)+ if !found {+ return "", 0, "", false, false+ }+ runID, tail, found = strings.Cut(tail, "/")+ if !found || !devServerRunIDPattern.MatchString(runID) {+ return "", 0, "", false, false+ }+ rawPort, rest, slash := strings.Cut(tail, "/")+ port, err := strconv.Atoi(rawPort)+ if err != nil || port <= 0 || port > 65535 || strconv.Itoa(port) != rawPort {+ return "", 0, "", false, false+ }+ return runID, port, rest, slash, true+}++// devServerHost runs before routing, like the build preview host: it serves+// /devpreview/ and sends a dev page's root-path requests back under its prefix.+func (h *browserHandlers) devServerHost() gin.HandlerFunc {+ return func(c *gin.Context) {+ r := c.Request+ if runID, file, ok := parseDevFilePath(r.URL.Path); ok {+ h.serveDevFile(c, runID, file)+ c.Abort()+ return+ }+ runID, port, rest, slash, ok := parseDevServerPath(r.URL.EscapedPath())+ if !ok {+ if target, stray := devServerStrayTarget(r); stray {+ c.Redirect(http.StatusTemporaryRedirect, target)+ c.Abort()+ return+ }+ c.Next()+ return+ }+ if !slash {+ target := devServerPrefix(runID, port) + "/"+ if r.URL.RawQuery != "" {+ target += "?" + r.URL.RawQuery+ }+ c.Redirect(http.StatusFound, target)+ c.Abort()+ return+ }+ h.proxyDevServer(c, runID, port, "/"+rest)+ c.Abort()+ }+}++// devServerStrayTarget maps a request a dev page sent to a root path (module+// imports such as /src/App.tsx, assets, links, forms) under the page's+// prefix. Dev pages are served with Referrer-Policy: same-origin, so their+// requests name the page they come from.+func devServerStrayTarget(r *http.Request) (string, bool) {+ if strings.HasPrefix(r.URL.Path, devServerPathPrefix) || websocket.IsWebSocketUpgrade(r) {+ return "", false+ }+ ref, err := url.Parse(r.Referer())+ if err != nil || ref.Host == "" {+ return "", false+ }+ runID, port, _, _, ok := parseDevServerPath(ref.EscapedPath())+ if !ok {+ return "", false+ }+ switch r.Header.Get("Sec-Fetch-Site") {+ case "same-origin":+ case "":+ // Plain-HTTP deployments get no fetch metadata; compare hosts.+ if !strings.EqualFold(ref.Host, r.Host) {+ return "", false+ }+ default:+ return "", false+ }+ target := devServerPrefix(runID, port) + r.URL.EscapedPath()+ if r.URL.RawQuery != "" {+ target += "?" + r.URL.RawQuery+ }+ return target, true+}++func (h *browserHandlers) proxyDevServer(c *gin.Context, runID string, port int, path string) {+ r := c.Request+ if websocket.IsWebSocketUpgrade(r) {+ devServerFail(c, http.StatusNotImplemented, "开发服务器的实时连接(如热更新)暂不支持,修改代码后请刷新页面", false)+ return+ }+ run, ok := h.devServerRun(c, runID)+ if !ok {+ return+ }+ body, err := io.ReadAll(http.MaxBytesReader(c.Writer, r.Body, devServerRequestLimit))+ if err != nil {+ devServerFail(c, http.StatusRequestEntityTooLarge, "请求体过大", false)+ return+ }+ prefix := devServerPrefix(runID, port)+ target := path+ if r.URL.RawQuery != "" {+ target += "?" + r.URL.RawQuery+ }+ req := domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{+ Port: port, Method: r.Method, Path: target, Header: devServerRequestHeader(r, prefix, port), Body: body,+ }}+ res, ok := h.devServerFetch(c, run, req, "开发服务器暂不可用")+ if !ok {+ return+ }+ writeDevServerResponse(c, res, prefix, port)+}++// devServerRun resolves the run of a preview request for a user who may+// drive its workspace; otherwise it answers the request itself.+func (h *browserHandlers) devServerRun(c *gin.Context, runID string) (domain.Run, bool) {+ r := c.Request+ run, ok := h.repo.RunState(runID)+ if !ok {+ devServerFail(c, http.StatusNotFound, "运行不存在", false)+ return domain.Run{}, false+ }+ user, ok := h.requestUser(r, false)+ if !ok {+ devServerFail(c, http.StatusUnauthorized, "请先登录平台", false)+ return domain.Run{}, false+ }+ if !h.canControl(r, run, user.ID) {+ devServerFail(c, http.StatusForbidden, "无权访问该任务的开发服务器", false)+ return domain.Run{}, false+ }+ h.touchWorkspace(run.ID)+ return run, true+}++// devServerFetch runs a preview operation on the executor, starting the run+// browser when it is not up: dev server ports live in its network namespace+// and file:// paths are the ones it sees. Failures are answered here.+func (h *browserHandlers) devServerFetch(c *gin.Context, run domain.Run, req domain.BrowserRequest, unavailable string) (*domain.BrowserHTTPResponse, bool) {+ r := c.Request+ state, opErr := h.devServerCall(r.Context(), run, req)+ if state.HTTP == nil && !state.Available && opErr == "" {+ req.Ensure = true+ state, opErr = h.devServerCall(r.Context(), run, req)+ }+ if opErr != "" {+ if strings.Contains(opErr, "不支持的浏览器操作") {+ opErr = "执行机版本过旧,暂不支持原生预览,请升级执行机"+ }+ devServerFail(c, http.StatusBadGateway, opErr, false)+ return nil, false+ }+ if state.HTTP == nil {+ devServerFail(c, http.StatusBadGateway, firstNonEmpty(state.Reason, unavailable), devServerNavigation(r))+ return nil, false+ }+ return state.HTTP, true+}++func (h *browserHandlers) devServerCall(ctx context.Context, run domain.Run, req domain.BrowserRequest) (domain.BrowserState, string) {+ payload, opErr := h.dispatch(ctx, run, req)+ var state domain.BrowserState+ _ = json.Unmarshal(payload, &state)+ return state, opErr+}++func devServerNavigation(r *http.Request) bool {+ if mode := r.Header.Get("Sec-Fetch-Mode"); mode != "" {+ return mode == "navigate"+ }+ return r.Method == http.MethodGet && strings.Contains(r.Header.Get("Accept"), "text/html")+}++// Request headers that carry the platform's identity or the proxy chain stay+// on this side; the sandbox only sees what a browser talking to the dev+// server directly would send.+var devServerRequestHeaders = map[string]bool{+ "Accept": true, "Accept-Language": true, "Cache-Control": true, "Pragma": true,+ "Content-Type": true, "If-None-Match": true, "If-Modified-Since": true, "If-Match": true,+ "If-Unmodified-Since": true, "Range": true, "If-Range": true, "User-Agent": true,+ "Sec-Fetch-Dest": true, "Sec-Fetch-Mode": true, "Sec-Fetch-Site": true, "Sec-Fetch-User": true,+ "Sec-Ch-Ua": true, "Sec-Ch-Ua-Mobile": true, "Sec-Ch-Ua-Platform": true,+ "Upgrade-Insecure-Requests": true, "Purpose": true, "Sec-Purpose": true,+}++var devServerBlockedXHeaders = []string{"X-Forwarded-", "X-Real-", "X-Client-", "X-Rio-", "X-Tai-", "X-Ext-", "X-Original-"}++func devServerRequestHeader(r *http.Request, prefix string, port int) map[string][]string {+ out := map[string][]string{}+ for name, values := range r.Header {+ key := http.CanonicalHeaderKey(name)+ switch {+ case devServerRequestHeaders[key]:+ case key == "Authorization":+ // Browsers replay cached Basic credentials of the platform itself.+ if v := r.Header.Get(key); v == "" || strings.HasPrefix(strings.ToLower(v), "basic ") {+ continue+ }+ case strings.HasPrefix(key, "X-") && !devServerBlockedXHeader(key):+ default:+ continue+ }+ out[key] = values+ }+ local := "http://localhost:" + strconv.Itoa(port)+ if r.Header.Get("Origin") != "" {+ out["Origin"] = []string{local}+ }+ if ref, err := url.Parse(r.Referer()); err == nil {+ if refRun, refPort, rest, _, ok := parseDevServerPath(ref.EscapedPath()); ok && devServerPrefix(refRun, refPort) == prefix {+ referer := local + "/" + rest+ if ref.RawQuery != "" {+ referer += "?" + ref.RawQuery+ }+ out["Referer"] = []string{referer}+ }+ }+ if tags := devServerUpstreamETags(r.Header.Values("If-None-Match")); tags != "" {+ out["If-None-Match"] = []string{tags}+ } else {+ delete(out, "If-None-Match")+ }+ cookiePrefix := preview.DevServerCookiePrefix(prefix)+ var cookies []string+ for _, cookie := range r.Cookies() {+ if name, ok := strings.CutPrefix(cookie.Name, cookiePrefix); ok && name != "" {+ cookies = append(cookies, name+"="+cookie.Value)+ }+ }+ if len(cookies) > 0 {+ out["Cookie"] = []string{strings.Join(cookies, "; ")}+ }+ return out+}++// devServerRewriteVersion tags the ETags of rewritten responses: a cached+// copy rewritten by an older gateway must not be revalidated as current.+const devServerRewriteVersion = "-af1"++func devServerETag(etag string) string {+ if strings.HasSuffix(etag, `"`) {+ return strings.TrimSuffix(etag, `"`) + devServerRewriteVersion + `"`+ }+ return etag + devServerRewriteVersion+}++// devServerUpstreamETags keeps the If-None-Match tags this gateway version+// issued, restored to the dev server's own form.+func devServerUpstreamETags(values []string) string {+ var tags []string+ for _, value := range values {+ for _, tag := range strings.Split(value, ",") {+ tag = strings.TrimSpace(tag)+ quoted := strings.HasSuffix(tag, `"`)+ bare := strings.TrimSuffix(tag, `"`)+ if !strings.HasSuffix(bare, devServerRewriteVersion) {+ continue+ }+ bare = strings.TrimSuffix(bare, devServerRewriteVersion)+ if quoted {+ bare += `"`+ }+ tags = append(tags, bare)+ }+ }+ return strings.Join(tags, ", ")+}++func devServerBlockedXHeader(key string) bool {+ for _, prefix := range devServerBlockedXHeaders {+ if strings.HasPrefix(key, prefix) {+ return true+ }+ }+ return false+}++// Response headers that would reach beyond the preview: framing and CSP that+// block the side panel, HSTS and Clear-Site-Data for the platform origin, and+// Service-Worker-Allowed, which would let a worker control the platform.+var devServerDroppedResponseHeaders = map[string]bool{+ "Content-Length": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true,+ "X-Frame-Options": true, "Strict-Transport-Security": true, "Clear-Site-Data": true,+ "Service-Worker-Allowed": true, "Referrer-Policy": true, "Set-Cookie": true, "Location": true,+ "Alt-Svc": true, "Access-Control-Allow-Origin": true, "Access-Control-Allow-Credentials": true,+}++func writeDevServerResponse(c *gin.Context, res *domain.BrowserHTTPResponse, prefix string, port int) {+ header := c.Writer.Header()+ for name, values := range res.Header {+ key := http.CanonicalHeaderKey(name)+ if devServerDroppedResponseHeaders[key] {+ continue+ }+ for _, v := range values {+ header.Add(key, v)+ }+ }+ for name, values := range res.Header {+ switch http.CanonicalHeaderKey(name) {+ case "Location":+ if len(values) > 0 {+ header.Set("Location", preview.RewriteDevServerLocation(values[0], prefix, port))+ }+ case "Set-Cookie":+ for _, v := range values {+ header.Add("Set-Cookie", rewriteDevServerCookie(v, prefix))+ }+ }+ }+ if etag := header.Get("Etag"); etag != "" {+ header.Set("Etag", devServerETag(etag))+ }+ // Root-path requests find their way back by Referer.+ header.Set("Referrer-Policy", "same-origin")+ header.Set("X-Content-Type-Options", "nosniff")+ body := res.Body+ if header.Get("Content-Encoding") == "" {+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(header.Get("Content-Type"), ";")[0]))+ switch mediaType {+ case "text/html", "application/xhtml+xml":+ body = preview.RewriteDevServerHTML(body, prefix, port)+ case "text/javascript", "application/javascript", "application/x-javascript", "text/ecmascript", "application/ecmascript":+ body = preview.RewriteDevServerScript(body, prefix)+ case "text/css":+ body = preview.RewriteDevServerCSS(body, prefix)+ }+ }+ header.Set("Content-Length", strconv.Itoa(len(body)))+ status := res.Status+ if status < 100 || status > 999 {+ status = http.StatusBadGateway+ }+ c.Status(status)+ if c.Request.Method != http.MethodHead && status != http.StatusNoContent && status != http.StatusNotModified {+ _, _ = c.Writer.Write(body)+ }+}++// rewriteDevServerCookie namespaces a dev server cookie by run and port and keeps it+// on the platform host under the preview prefix.+func rewriteDevServerCookie(raw, prefix string) string {+ parts := strings.Split(raw, ";")+ out := []string{preview.DevServerCookiePrefix(prefix) + strings.TrimSpace(parts[0])}+ for _, attr := range parts[1:] {+ attr = strings.TrimSpace(attr)+ name, value, _ := strings.Cut(attr, "=")+ switch strings.ToLower(strings.TrimSpace(name)) {+ case "":+ continue+ case "domain":+ continue+ case "path":+ value = strings.TrimSpace(value)+ if !strings.HasPrefix(value, "/") {+ value = "/"+ }+ if value != prefix && !strings.HasPrefix(value, prefix+"/") {+ value = prefix + value+ }+ attr = "Path=" + value+ }+ out = append(out, attr)+ }+ return strings.Join(out, "; ")+}++func devServerFail(c *gin.Context, status int, message string, retry bool) {+ if !devServerNavigation(c.Request) {+ c.Data(status, "text/plain; charset=utf-8", []byte(message))+ return+ }+ refresh := ""+ hint := ""+ if retry {+ refresh = `<meta http-equiv="refresh" content="3">`+ hint = `<p class="hint">页面每 3 秒自动重试;让 Agent 启动开发服务器后这里会自动打开。</p>`+ }+ page := fmt.Sprintf(`<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">%s<title>开发服务器预览</title><style>body{margin:0;min-height:100vh;display:grid;place-items:center;font:14px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#3b3f45;background:#f7f8fa}@media(prefers-color-scheme:dark){body{color:#d7dae0;background:#1d1f23}}main{max-width:420px;padding:24px;text-align:center}.hint{opacity:.65;font-size:13px}</style></head><body><main><p>%s</p>%s</main></body></html>`,+ refresh, html.EscapeString(message), hint)+ c.Header("Cache-Control", "no-store")+ c.Data(status, "text/html; charset=utf-8", []byte(page))+}++// devServerFileSegment marks workspace files in place of a port:+// /devpreview/{runID}/file/{absolute path} serves what the run browser opens+// as file://{absolute path}, so relative links resolve as they do there.+const devServerFileSegment = "file"++// parseDevFilePath splits /devpreview/{runID}/file/{path} of an unescaped+// request path; file is absolute and cleaned, keeping a trailing slash.+func parseDevFilePath(p string) (runID, file string, ok bool) {+ tail, found := strings.CutPrefix(p, devServerPathPrefix)+ if !found {+ return "", "", false+ }+ runID, tail, found = strings.Cut(tail, "/")+ if !found || !devServerRunIDPattern.MatchString(runID) {+ return "", "", false+ }+ rest, found := strings.CutPrefix(tail, devServerFileSegment+"/")+ if !found || rest == "" {+ return "", "", false+ }+ file = path.Clean("/" + rest)+ if strings.HasSuffix(rest, "/") && file != "/" {+ file += "/"+ }+ return runID, file, true+}++func (h *browserHandlers) serveDevFile(c *gin.Context, runID, file string) {+ r := c.Request+ if r.Method != http.MethodGet && r.Method != http.MethodHead {+ devServerFail(c, http.StatusMethodNotAllowed, "工作区文件只支持读取", false)+ return+ }+ run, ok := h.devServerRun(c, runID)+ if !ok {+ return+ }+ res, ok := h.devServerFetch(c, run, domain.BrowserRequest{Operation: "file", File: file}, "工作区文件暂不可用")+ if !ok {+ return+ }+ header := c.Writer.Header()+ if res.Status == http.StatusMovedPermanently {+ // A directory: serve it with a slash so its relative links resolve.+ target := r.URL.EscapedPath() + "/"+ if r.URL.RawQuery != "" {+ target += "?" + r.URL.RawQuery+ }+ c.Redirect(http.StatusMovedPermanently, target)+ return+ }+ if v := res.Header["Content-Type"]; len(v) > 0 {+ header.Set("Content-Type", v[0])+ }+ // The Agent keeps editing these files; always revalidate.+ header.Set("Cache-Control", "no-cache")+ header.Set("Referrer-Policy", "same-origin")+ header.Set("X-Content-Type-Options", "nosniff")+ header.Set("Content-Length", strconv.Itoa(len(res.Body)))+ status := res.Status+ if status < 100 || status > 999 {+ status = http.StatusBadGateway+ }+ c.Status(status)+ if r.Method != http.MethodHead {+ _, _ = c.Writer.Write(res.Body)+ }+}diff --git a/internal/httpapi/browser_devserver_sandbox_live_test.go b/internal/httpapi/browser_devserver_sandbox_live_test.gonew file mode 100644index 00000000..ea1c6609--- /dev/null+++ b/internal/httpapi/browser_devserver_sandbox_live_test.go@@ -0,0 +1,198 @@+//go:build sandboxlive++package httpapi++import (+ "context"+ "encoding/json"+ "net"+ "net/http"+ "net/http/httptest"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/engine"+ "agentflow/internal/runtime"++ "github.com/gin-gonic/gin"+)++// The dev server binds 127.0.0.1 inside the sandbox and the port is not+// published. The preview reaches it through the browser sidecar, which+// shares that network namespace, the same way a run does.+func TestDevServerPreviewReachesDockerSandbox(t *testing.T) {+ if _, err := exec.LookPath("docker"); err != nil {+ t.Skip("docker not installed")+ }+ if err := exec.Command("docker", "info").Run(); err != nil {+ t.Skip("docker daemon not running")+ }+ const (+ runID = "live-devpreview"+ port = "4173"+ image = "agentflow-browser-c34051af5914b90c"+ sandbox = "lf-devpreview-sandbox"+ sidecar = "lf-devpreview-browser"+ pageToken = "sandbox-live-7f3a"+ fileToken = "workspace-live-marker"+ )+ if err := exec.Command("docker", "image", "inspect", image).Run(); err != nil {+ t.Fatalf("browser sidecar image %s is not present", image)+ }++ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)+ run, ok := a.repo.RunState("shared-run")+ if !ok {+ t.Fatal("fixture run missing")+ }+ run.ID = runID+ run.ExecutionContext.ExecutionMode = domain.ExecutionModeCloud+ if err := a.repo.PutRun(run); err != nil {+ t.Fatal(err)+ }++ // The Docker VM shares the home directory. macOS temp dirs and /tmp are+ // not visible inside containers, so the server script and workspace have+ // to live under the home directory.+ home, err := os.UserHomeDir()+ if err != nil {+ t.Fatal(err)+ }+ root := filepath.Join(home, "tmp", "lf-devpreview-live")+ _ = os.RemoveAll(root)+ ws := filepath.Join(root, "workspace")+ if err := os.MkdirAll(ws, 0o755); err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(root) })+ if err := os.WriteFile(filepath.Join(ws, "index.html"), []byte("<p>"+fileToken+"</p>"), 0o644); err != nil {+ t.Fatal(err)+ }+ serverJS := `const http=require("http");+const token=` + "`" + pageToken + "`" + `;+http.createServer((req,res)=>{+ if(req.url==="/headers"){+ const names=["host","cookie","x-forwarded-for","x-real-ip","x-app-trace","authorization"];+ const out={}; for(const n of names) out[n]=req.headers[n]||"";+ res.setHeader("Content-Type","application/json"); res.end(JSON.stringify(out)); return;+ }+ if(req.url==="/src/main.ts"){+ res.setHeader("Content-Type","text/javascript"); res.end('export default "from-sandbox"'); return;+ }+ res.setHeader("Content-Type","text/html; charset=utf-8");+ res.setHeader("Set-Cookie","sid=live; Path=/");+ res.end('<!doctype html><html><head></head><body><p>'+token+'</p><script type="module" src="/src/main.ts"></script></body></html>');+}).listen(4173,"127.0.0.1");`+ script := filepath.Join(root, "server.js")+ if err := os.WriteFile(script, []byte(serverJS), 0o644); err != nil {+ t.Fatal(err)+ }++ rm := func() {+ _ = exec.Command("docker", "rm", "-f", sidecar, sandbox).Run()+ }+ rm()+ t.Cleanup(rm)+ if out, err := exec.Command("docker", "run", "-d", "--name", sandbox,+ "-v", script+":/server.js:ro", "--entrypoint", "node", "node:22-bookworm-slim", "/server.js").CombinedOutput(); err != nil {+ t.Fatalf("start sandbox: %v: %s", err, out)+ }+ time.Sleep(300 * time.Millisecond)+ if state, _ := exec.Command("docker", "inspect", "-f", "{{.State.Status}}", sandbox).Output(); strings.TrimSpace(string(state)) != "running" {+ logs, _ := exec.Command("docker", "logs", sandbox).CombinedOutput()+ t.Fatalf("sandbox %s: %s", strings.TrimSpace(string(state)), logs)+ }+ if out, err := exec.Command("docker", "run", "-d", "--name", sidecar, "--init",+ "--label", "agentflow.browser.run="+runID,+ "--label", "agentflow.browser.owner="+sandbox,+ "--network", "container:"+sandbox,+ "--cap-drop=ALL", "--security-opt", "no-new-privileges",+ "--entrypoint", "sleep",+ "-v", ws+":/workspace",+ image, "600").CombinedOutput(); err != nil {+ t.Fatalf("start sidecar: %v: %s", err, out)+ }++ h := &browserHandlers{deps: a.deps}+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))+ current := owner+ g := gin.New()+ g.Use(func(c *gin.Context) {+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(current)))+ })+ g.Use(h.devServerHost())+ prefix := "/devpreview/" + runID + "/" + port+ serve := func(method, path string, header map[string]string) *httptest.ResponseRecorder {+ t.Helper()+ req := httptest.NewRequest(method, "http://platform.test"+path, nil)+ for k, v := range header {+ req.Header.Set(k, v)+ }+ rec := httptest.NewRecorder()+ g.ServeHTTP(rec, req)+ return rec+ }++ conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, 300*time.Millisecond)+ if err == nil {+ conn.Close()+ t.Fatal("sandbox port is reachable on the host; the preview would not be proving the network namespace")+ }+ if published, err := exec.Command("docker", "port", sandbox).Output(); err != nil || strings.TrimSpace(string(published)) != "" {+ t.Fatalf("sandbox published ports: %q err=%v", published, err)+ }++ rec := serve(http.MethodGet, prefix+"/", map[string]string{"Accept": "text/html", "Sec-Fetch-Mode": "navigate"})+ body := rec.Body.String()+ if rec.Code != 200 || !strings.Contains(body, pageToken) || !strings.Contains(body, `src="`+prefix+`/src/main.ts"`) || !strings.Contains(body, "data-af-dev-server") {+ t.Fatalf("page: %d %s", rec.Code, body)+ }+ if cookie := rec.Header().Get("Set-Cookie"); cookie != "af_devpreview_"+runID+"_"+port+"_sid=live; Path="+prefix+"/" {+ t.Fatalf("cookie %q", cookie)+ }++ rec = serve(http.MethodGet, prefix+"/src/main.ts", nil)+ if rec.Code != 200 || !strings.Contains(rec.Body.String(), `export default "from-sandbox"`) {+ t.Fatalf("module: %d %s", rec.Code, rec.Body.String())+ }++ rec = serve(http.MethodGet, prefix+"/headers", map[string]string{+ "Cookie": "af_devpreview_" + runID + "_" + port + "_sid=live; platform_session=secret",+ "X-Forwarded-For": "203.0.113.8",+ "X-Real-IP": "203.0.113.9",+ "X-App-Trace": "trace-from-browser",+ "Authorization": "Basic cGxhdGZvcm06cGFzcw==",+ "Origin": "http://platform.test",+ "Referer": "http://platform.test" + prefix + "/admin",+ })+ if rec.Code != 200 {+ t.Fatalf("headers: %d %s", rec.Code, rec.Body.String())+ }+ var seen map[string]string+ if err := json.Unmarshal(rec.Body.Bytes(), &seen); err != nil {+ t.Fatal(err)+ }+ if seen["host"] != "localhost:"+port || seen["cookie"] != "sid=live" || seen["x-forwarded-for"] != "" || seen["x-real-ip"] != "" || seen["authorization"] != "" || seen["x-app-trace"] != "trace-from-browser" {+ t.Fatalf("upstream headers %+v", seen)+ }++ rec = serve(http.MethodGet, "/devpreview/"+runID+"/file/workspace/index.html", nil)+ if rec.Code != 200 || !strings.Contains(rec.Body.String(), fileToken) {+ t.Fatalf("workspace file: %d %s", rec.Code, rec.Body.String())+ }++ rec = serve(http.MethodGet, "/devpreview/"+runID+"/4174/", map[string]string{"Sec-Fetch-Mode": "navigate"})+ if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "没有正在监听") {+ t.Fatalf("idle port: %d %s", rec.Code, rec.Body.String())+ }++ current = outsider+ if rec = serve(http.MethodGet, prefix+"/", nil); rec.Code != http.StatusForbidden {+ t.Fatalf("outsider: %d %s", rec.Code, rec.Body.String())+ }+}diff --git a/internal/httpapi/browser_devserver_test.go b/internal/httpapi/browser_devserver_test.gonew file mode 100644index 00000000..ae6edf24--- /dev/null+++ b/internal/httpapi/browser_devserver_test.go@@ -0,0 +1,316 @@+package httpapi++import (+ "context"+ "encoding/json"+ "net"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strconv"+ "strings"+ "testing"++ "agentflow/internal/domain"+ "agentflow/internal/engine"+ "agentflow/internal/runtime"++ "github.com/gin-gonic/gin"+)++// fakeHostBrowser registers a host browser for runID the way a host driver+// does, so dev server requests are served from this machine's loopback.+func fakeHostBrowser(t *testing.T, runID string) {+ t.Helper()+ fakeHostBrowserIn(t, runID, "")+}++// fakeHostBrowserIn also records the workspace the browser serves file:// from.+func fakeHostBrowserIn(t *testing.T, runID, workspace string) {+ t.Helper()+ devtools := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }))+ t.Cleanup(devtools.Close)+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(devtools.URL, "http://"))+ port, _ := strconv.Atoi(rawPort)+ dir := filepath.Join(os.TempDir(), "agentflow-browser", runID)+ _ = os.RemoveAll(dir)+ t.Cleanup(func() { _ = os.RemoveAll(dir) })+ if err := os.MkdirAll(dir, 0o700); err != nil {+ t.Fatal(err)+ }+ record, _ := json.Marshal(map[string]any{"pid": os.Getpid(), "owner": os.Getpid(), "webSocket": "ws://127.0.0.1:" + rawPort, "port": port, "workspace": workspace})+ if err := os.WriteFile(filepath.Join(dir, "endpoint.json"), record, 0o600); err != nil {+ t.Fatal(err)+ }+}++func TestDevServerPreviewProxiesTheSandboxPort(t *testing.T) {+ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)+ h := &browserHandlers{deps: a.deps}+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))+ fakeHostBrowser(t, "shared-run")++ var seen *http.Request+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ seen = r+ switch r.URL.Path {+ case "/":+ http.SetCookie(w, &http.Cookie{Name: "sid", Value: "1", Path: "/", Domain: "localhost"})+ w.Header().Set("X-Frame-Options", "DENY")+ w.Header().Set("Content-Type", "text/html")+ _, _ = w.Write([]byte(`<!doctype html><html><head><title>app</title></head><body><script type="module" src="/src/main.ts"></script></body></html>`))+ case "/login":+ http.Redirect(w, r, "http://localhost:"+r.Host[strings.LastIndex(r.Host, ":")+1:]+"/home?x=1", http.StatusFound)+ default:+ w.Header().Set("Content-Type", "text/javascript")+ _, _ = w.Write([]byte(`export default 1`))+ if r.URL.Path == "/src/app.ts" {+ _, _ = w.Write([]byte(`;import "/src/b.ts"`))+ }+ }+ }))+ defer dev.Close()+ _, devPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))+ prefix := "/devpreview/shared-run/" + devPort++ g := gin.New()+ user := owner+ g.Use(func(c *gin.Context) {+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(user)))+ })+ g.Use(h.devServerHost())+ g.NoRoute(func(c *gin.Context) { c.String(404, "platform") })+ serve := func(method, path string, header map[string]string) *httptest.ResponseRecorder {+ t.Helper()+ req := httptest.NewRequest(method, "http://platform.test"+path, nil)+ for k, v := range header {+ req.Header.Set(k, v)+ }+ rec := httptest.NewRecorder()+ g.ServeHTTP(rec, req)+ return rec+ }++ if rec := serve("GET", prefix, nil); rec.Code != 302 || rec.Header().Get("Location") != prefix+"/" {+ t.Fatalf("bare prefix: %d %q", rec.Code, rec.Header().Get("Location"))+ }+ rec := serve("GET", prefix+"/", map[string]string{"Accept": "text/html"})+ body := rec.Body.String()+ if rec.Code != 200 || !strings.Contains(body, "data-af-dev-server") || !strings.Contains(body, "data-af-preview-history") {+ t.Fatalf("document: %d %s", rec.Code, body)+ }+ if !strings.Contains(body, `src="`+prefix+`/src/main.ts"`) || strings.Contains(body, "<base") {+ t.Fatalf("module references must move under the prefix: %s", body)+ }+ if rec.Header().Get("X-Frame-Options") != "" || rec.Header().Get("Referrer-Policy") != "same-origin" {+ t.Fatalf("headers: %v", rec.Header())+ }+ if cookie := rec.Header().Get("Set-Cookie"); cookie != "af_devpreview_shared-run_"+devPort+"_sid=1; Path="+prefix+"/" {+ t.Fatalf("cookie: %q", cookie)+ }++ // A module import from the page comes back under the prefix by Referer.+ rec = serve("GET", "/src/main.ts?v=2", map[string]string{"Referer": "http://platform.test" + prefix + "/", "Sec-Fetch-Site": "same-origin"})+ if rec.Code != 307 || rec.Header().Get("Location") != prefix+"/src/main.ts?v=2" {+ t.Fatalf("stray import: %d %q", rec.Code, rec.Header().Get("Location"))+ }+ if rec := serve("GET", "/src/main.ts", map[string]string{"Referer": "http://platform.test/chatbot/", "Sec-Fetch-Site": "same-origin"}); rec.Code != 404 || rec.Body.String() != "platform" {+ t.Fatalf("platform request rerouted: %d", rec.Code)+ }+ if rec := serve("GET", "/src/main.ts", map[string]string{"Referer": "http://evil.test" + prefix + "/", "Sec-Fetch-Site": "cross-site"}); rec.Code != 404 {+ t.Fatalf("cross-site request rerouted: %d", rec.Code)+ }++ if rec := serve("GET", prefix+"/src/app.ts", nil); !strings.Contains(rec.Body.String(), `import "`+prefix+`/src/b.ts"`) {+ t.Fatalf("module imports: %s", rec.Body.String())+ }++ rec = serve("GET", prefix+"/login", nil)+ if rec.Code != 302 || rec.Header().Get("Location") != prefix+"/home?x=1" {+ t.Fatalf("redirect: %d %q", rec.Code, rec.Header().Get("Location"))+ }++ rec = serve("GET", prefix+"/src/main.ts", map[string]string{+ // Only this run's cookies reach its dev server: not the platform's,+ // not another run's on the same port, not the old port-only names.+ "Cookie": "af_devpreview_shared-run_" + devPort + "_sid=1; " + authCookieName + "=secret; " ++ "af_devpreview_other-run_" + devPort + "_leak=2; afdev" + devPort + "_old=3",+ "Origin": "http://platform.test",+ "Referer": "http://platform.test" + prefix + "/admin?a=1",+ "Staffname": "bob",+ "X-Forwarded-For": "10.0.0.1",+ "X-Requested-With": "XMLHttpRequest",+ "Authorization": "Basic cGxhdGZvcm06cGFzcw==",+ })+ if rec.Code != 200 || rec.Body.String() != "export default 1" {+ t.Fatalf("module: %d %s", rec.Code, rec.Body.String())+ }+ if got := seen.Header.Get("Cookie"); got != "sid=1" {+ t.Fatalf("upstream cookie %q", got)+ }+ if seen.Host != "localhost:"+devPort || seen.Header.Get("Origin") != "http://localhost:"+devPort || seen.Header.Get("Referer") != "http://localhost:"+devPort+"/admin?a=1" {+ t.Fatalf("upstream host/origin/referer: %q %q %q", seen.Host, seen.Header.Get("Origin"), seen.Header.Get("Referer"))+ }+ for _, name := range []string{"Staffname", "X-Forwarded-For", "Authorization"} {+ if seen.Header.Get(name) != "" {+ t.Fatalf("upstream got %s", name)+ }+ }+ if seen.Header.Get("X-Requested-With") != "XMLHttpRequest" {+ t.Fatal("app headers must pass")+ }++ if rec := serve("GET", prefix+"/", map[string]string{"Upgrade": "websocket", "Connection": "Upgrade", "Sec-Websocket-Version": "13"}); rec.Code != 501 {+ t.Fatalf("websocket: %d", rec.Code)+ }++ closed, _ := net.Listen("tcp", "127.0.0.1:0")+ _, idlePort, _ := net.SplitHostPort(closed.Addr().String())+ closed.Close()+ rec = serve("GET", "/devpreview/shared-run/"+idlePort+"/", map[string]string{"Sec-Fetch-Mode": "navigate"})+ if rec.Code != 502 || !strings.Contains(rec.Body.String(), "没有正在监听") || !strings.Contains(rec.Body.String(), `http-equiv="refresh"`) {+ t.Fatalf("idle port: %d %s", rec.Code, rec.Body.String())+ }++ user = outsider+ if rec := serve("GET", prefix+"/", nil); rec.Code != 403 {+ t.Fatalf("outsider: %d", rec.Code)+ }+}++func TestParseDevServerPath(t *testing.T) {+ for _, tc := range []struct {+ path string+ run string+ port int+ rest string+ slash bool+ ok bool+ }{+ {"/devpreview/run-1/5173/src/a.ts", "run-1", 5173, "src/a.ts", true, true},+ {"/devpreview/run-1/5173/", "run-1", 5173, "", true, true},+ {"/devpreview/run-1/5173", "run-1", 5173, "", false, true},+ {"/devpreview/run-1/05173/", "", 0, "", false, false},+ {"/devpreview/run-1/70000/", "", 0, "", false, false},+ {"/devpreview/run 1/80/", "", 0, "", false, false},+ {"/devpreview/run-1", "", 0, "", false, false},+ {"/preview/run-1/80/", "", 0, "", false, false},+ } {+ run, port, rest, slash, ok := parseDevServerPath(tc.path)+ if run != tc.run || port != tc.port || rest != tc.rest || slash != tc.slash || ok != tc.ok {+ t.Fatalf("%s: %q %d %q %v %v", tc.path, run, port, rest, slash, ok)+ }+ }+}++func TestDevServerETagsCarryTheRewriteVersion(t *testing.T) {+ if got := devServerETag(`W/"12-abc"`); got != `W/"12-abc-af1"` {+ t.Fatal(got)+ }+ if got := devServerUpstreamETags([]string{`W/"12-abc-af1", "old"`, `"x-af1"`}); got != `W/"12-abc", "x"` {+ t.Fatal(got)+ }+ if got := devServerUpstreamETags([]string{`"old"`}); got != "" {+ t.Fatal(got)+ }+}++func TestRewriteDevServerCookie(t *testing.T) {+ prefix := "/devpreview/r/3000"+ for raw, want := range map[string]string{+ "a=1; Path=/; Domain=localhost; HttpOnly": "af_devpreview_r_3000_a=1; Path=/devpreview/r/3000/; HttpOnly",+ "a=1; path=/api; SameSite=Lax": "af_devpreview_r_3000_a=1; Path=/devpreview/r/3000/api; SameSite=Lax",+ "a=1": "af_devpreview_r_3000_a=1",+ } {+ if got := rewriteDevServerCookie(raw, prefix); got != want {+ t.Fatalf("%q: %q", raw, got)+ }+ }+ _ = domain.BrowserRequest{}+}++func TestDevServerPreviewServesWorkspaceFiles(t *testing.T) {+ a, owner, _, outsider, _ := sharedWorkbenchFixture(t)+ h := &browserHandlers{deps: a.deps}+ h.engine = engine.New(h.repo, runtime.NewManager(t.TempDir()))+ ws := t.TempDir()+ for name, body := range map[string]string{"demo.html": `<link href="style.css"><h1>demo</h1>`, "style.css": "h1{}", "site/index.html": "<p>site</p>"} {+ _ = os.MkdirAll(filepath.Dir(filepath.Join(ws, name)), 0o755)+ if err := os.WriteFile(filepath.Join(ws, name), []byte(body), 0o644); err != nil {+ t.Fatal(err)+ }+ }+ fakeHostBrowserIn(t, "shared-run", ws)++ g := gin.New()+ user := owner+ g.Use(func(c *gin.Context) {+ c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestUserContextKey{}, domainUserToAuth(user)))+ })+ g.Use(h.devServerHost())+ g.NoRoute(func(c *gin.Context) { c.String(404, "platform") })+ serve := func(method, path string) *httptest.ResponseRecorder {+ t.Helper()+ rec := httptest.NewRecorder()+ g.ServeHTTP(rec, httptest.NewRequest(method, "http://platform.test"+path, nil))+ return rec+ }+ prefix := "/devpreview/shared-run/file" + ws++ rec := serve("GET", prefix+"/demo.html")+ if rec.Code != 200 || rec.Body.String() != `<link href="style.css"><h1>demo</h1>` || rec.Header().Get("Content-Type") != "text/html; charset=utf-8" {+ t.Fatalf("page: %d %q %v", rec.Code, rec.Body.String(), rec.Header())+ }+ if rec.Header().Get("Cache-Control") != "no-cache" || rec.Header().Get("Referrer-Policy") != "same-origin" {+ t.Fatalf("headers: %v", rec.Header())+ }+ if rec := serve("GET", prefix+"/style.css"); rec.Code != 200 || rec.Header().Get("Content-Type") != "text/css; charset=utf-8" {+ t.Fatalf("stylesheet: %d %v", rec.Code, rec.Header())+ }+ if rec := serve("GET", prefix+"/site"); rec.Code != 301 || rec.Header().Get("Location") != prefix+"/site/" {+ t.Fatalf("directory: %d %q", rec.Code, rec.Header().Get("Location"))+ }+ if rec := serve("GET", prefix+"/site/"); rec.Code != 200 || rec.Body.String() != "<p>site</p>" {+ t.Fatalf("index: %d %q", rec.Code, rec.Body.String())+ }+ if rec := serve("HEAD", prefix+"/site/"); rec.Code != 200 || rec.Body.Len() != 0 {+ t.Fatalf("head: %d %q", rec.Code, rec.Body.String())+ }+ if rec := serve("GET", prefix+"/missing.html"); rec.Code != 404 {+ t.Fatalf("missing: %d", rec.Code)+ }+ if err := os.WriteFile(filepath.Join(filepath.Dir(ws), "secret.txt"), []byte("secret"), 0o644); err != nil {+ t.Fatal(err)+ }+ if rec := serve("GET", prefix+"/../secret.txt"); rec.Code != 403 {+ t.Fatalf("outside: %d %q", rec.Code, rec.Body.String())+ }+ if rec := serve("POST", prefix+"/demo.html"); rec.Code != 405 {+ t.Fatalf("post: %d", rec.Code)+ }+ user = outsider+ if rec := serve("GET", prefix+"/demo.html"); rec.Code != 403 {+ t.Fatalf("outsider: %d", rec.Code)+ }+}++func TestParseDevFilePath(t *testing.T) {+ for _, tc := range []struct {+ path, run, file string+ ok bool+ }{+ {"/devpreview/run-1/file/workspace/demo.html", "run-1", "/workspace/demo.html", true},+ {"/devpreview/run-1/file/workspace/site/", "run-1", "/workspace/site/", true},+ {"/devpreview/run-1/file/workspace/a/../b.html", "run-1", "/workspace/b.html", true},+ {"/devpreview/run-1/file/", "", "", false},+ {"/devpreview/run-1/file", "", "", false},+ {"/devpreview/run-1/5173/file/a", "", "", false},+ {"/devpreview/run 1/file/a", "", "", false},+ } {+ run, file, ok := parseDevFilePath(tc.path)+ if run != tc.run || file != tc.file || ok != tc.ok {+ t.Fatalf("%s: %q %q %v", tc.path, run, file, ok)+ }+ }+}diff --git a/internal/httpapi/browser_stream.go b/internal/httpapi/browser_stream.goindex 8fbddef0..62256dce 100644--- a/internal/httpapi/browser_stream.go+++ b/internal/httpapi/browser_stream.go@@ -29,9 +29,10 @@ const (browserStreamPing = 20 * time.SecondbrowserStreamRunRefresh = 10 * time.SecondbrowserStreamRetry = 4 * time.Second- // An animated page screencasts at display rate; 25 fps is enough to- // follow the agent and keeps a busy page from saturating the link.- browserStreamFrameInterval = 40 * time.Millisecond+ // Frame long-polls kept waiting at the browser. Three cover a round trip+ // of a few frame intervals at the rates a sandbox browser renders.+ browserStreamFramePulls = 3+ browserStreamPullGap = 15 * time.Millisecond)type browserStreamMessage struct {@@ -169,37 +170,104 @@ func (s *browserStream) execute(ctx context.Context, cancel context.CancelFunc,}func (s *browserStream) frames(ctx context.Context, cancel context.CancelFunc) {- var last time.Time- for ctx.Err() == nil {- if wait := browserStreamFrameInterval - time.Since(last); wait > 0 {- select {- case <-ctx.Done():- return- case <-time.After(wait):- }- }- last = time.Now()- s.mu.Lock()- after := s.seq- s.mu.Unlock()+ pull := func(ctx context.Context, after int64) json.RawMessage {payload, _ := s.h.dispatch(ctx, s.currentRun(false), domain.BrowserRequest{Operation: "frame", After: after, WaitMs: 3000})- if ctx.Err() != nil {- return+ return payload+ }+ err := pipelineFrames(ctx, browserStreamFramePulls, pull, s.publish, func() int64 {+ s.mu.Lock()+ defer s.mu.Unlock()+ return s.seq+ }, func() bool {+ select {+ case <-ctx.Done():+ return false+ case <-time.After(browserStreamRetry):}- available, err := s.publish(payload)- if err != nil {- cancel()- return+ s.currentRun(true)+ return true+ })+ if err != nil {+ cancel()+ }+}++// pipelineFrames keeps pulls frame long-polls waiting at the browser, each+// for a later frame than the latest shown (latest, latest+1, …). A frame+// then leaves the moment it is captured instead of one round trip after the+// previous one arrived: a single poll delivered about half the frames a+// scrolling page produced.+//+// Thresholds always count from the latest frame shown, never from earlier+// polls: a poll that times out on an idle page is sent again for the same+// frame. Counting up per poll ran the thresholds ahead of the browser while+// nothing moved, and the runtime only returns an image newer than the+// threshold, so the next frames were never shown.+//+// publish reports whether the browser is available; while it is not, the+// polls drain and retry waits before the next round.+func pipelineFrames(+ ctx context.Context,+ pulls int,+ pull func(context.Context, int64) json.RawMessage,+ publish func(json.RawMessage) (bool, error),+ latest func() int64,+ retry func() bool,+) error {+ type answer struct {+ after int64+ payload json.RawMessage+ }+ results := make(chan answer, pulls)+ var issued time.Time+ waiting := map[int64]bool{}+ down := false+ for {+ for !down && len(waiting) < pulls && ctx.Err() == nil {+ // A spaced start keeps a runtime that answers at once (no frame+ // wait) from turning this into a hot loop.+ if wait := browserStreamPullGap - time.Since(issued); wait > 0 {+ select {+ case <-ctx.Done():+ return nil+ case <-time.After(wait):+ }+ }+ issued = time.Now()+ after := latest()+ for waiting[after] {+ after+++ }+ waiting[after] = true+ go func() {+ payload := pull(ctx, after)+ select {+ case results <- answer{after, payload}:+ case <-ctx.Done():+ }+ }()}- if available {+ if len(waiting) == 0 {+ if ctx.Err() != nil || !retry() {+ return nil+ }+ down = falsecontinue}+ var got answerselect {case <-ctx.Done():- return- case <-time.After(browserStreamRetry):+ return nil+ case got = <-results:+ }+ delete(waiting, got.after)+ available, err := publish(got.payload)+ if err != nil {+ return err+ }+ if !available {+ down = true}- s.currentRun(true)}}diff --git a/internal/httpapi/browser_stream_pipeline_test.go b/internal/httpapi/browser_stream_pipeline_test.gonew file mode 100644index 00000000..c1af21d5--- /dev/null+++ b/internal/httpapi/browser_stream_pipeline_test.go@@ -0,0 +1,162 @@+package httpapi++import (+ "context"+ "encoding/json"+ "fmt"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// fakeFrames is a browser behind a link with the given one-way delay. It+// captures a frame every interval, or, with interval 0, only when frame is+// raised. Like browserView.state it sends the image only when it is newer+// than the poll's threshold, and a poll gives up after timeout.+type fakeFrames struct {+ start time.Time+ interval time.Duration+ delay time.Duration+ timeout time.Duration+ frame atomic.Int64+ down atomic.Bool+ pulls atomic.Int64+}++func (f *fakeFrames) seq() int64 {+ if f.interval == 0 {+ return f.frame.Load()+ }+ return int64(time.Since(f.start) / f.interval)+}++func (f *fakeFrames) pull(ctx context.Context, after int64) json.RawMessage {+ f.pulls.Add(1)+ time.Sleep(f.delay)+ if f.down.Load() {+ return json.RawMessage(`{"available":false,"reason":"gone"}`)+ }+ timeout := f.timeout+ if timeout == 0 {+ timeout = time.Second+ }+ deadline := time.Now().Add(timeout)+ for f.seq() <= after && time.Now().Before(deadline) && ctx.Err() == nil {+ time.Sleep(time.Millisecond)+ }+ seq, frame := f.seq(), ""+ if seq > after {+ frame = "x"+ }+ payload := json.RawMessage(fmt.Sprintf(`{"available":true,"seq":%d,"frame":%q}`, seq, frame))+ time.Sleep(f.delay)+ return payload+}++// panelFrames stands in for browserStream.publish: it shows a frame only when+// it carries an image newer than the last one shown.+type panelFrames struct {+ mu sync.Mutex+ last int64+ shown int+}++func (p *panelFrames) publish(payload json.RawMessage) (bool, error) {+ var state struct {+ Available bool `json:"available"`+ Seq int64 `json:"seq"`+ Frame string `json:"frame"`+ }+ _ = json.Unmarshal(payload, &state)+ p.mu.Lock()+ defer p.mu.Unlock()+ if state.Frame != "" && state.Seq > p.last {+ p.last = state.Seq+ p.shown+++ }+ return state.Available, nil+}++func (p *panelFrames) latest() int64 {+ p.mu.Lock()+ defer p.mu.Unlock()+ return p.last+}++// deliver runs the pipeline for d and counts frames shown.+func deliver(t *testing.T, pulls int, f *fakeFrames, d time.Duration) int {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), d)+ defer cancel()+ panel := &panelFrames{}+ if err := pipelineFrames(ctx, pulls, f.pull, panel.publish, panel.latest, func() bool { return false }); err != nil {+ t.Fatal(err)+ }+ return panel.shown+}++func TestPipelinedFramePollsKeepUpWithTheBrowser(t *testing.T) {+ const run = 1500 * time.Millisecond+ single := deliver(t, 1, &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond, delay: 60 * time.Millisecond}, run)+ f := &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond, delay: 60 * time.Millisecond}+ piped := deliver(t, 3, f, run)+ produced := int(run / f.interval)+ // One poll waits a round trip (120ms) per frame; three waiting ahead+ // deliver up to three per round trip of the 50 captured meanwhile.+ if single > 15 || piped*2 < single*3 || piped*3 < produced {+ t.Fatalf("single poll showed %d, pipelined %d of %d produced", single, piped, produced)+ }+ // A jump in sequence makes the polls waiting below it stale; each comes+ // back once and is not re-sent, so polls stay within twice the frames.+ if pulls := f.pulls.Load(); pulls > int64(2*piped)+5 {+ t.Fatalf("%d polls for %d frames", pulls, piped)+ }+}++func TestPipelinedFramePollsRetryWhileTheBrowserIsDown(t *testing.T) {+ f := &fakeFrames{start: time.Now(), interval: 30 * time.Millisecond}+ f.down.Store(true)+ retries := 0+ published := 0+ err := pipelineFrames(context.Background(), 3, f.pull, func(json.RawMessage) (bool, error) {+ published+++ return false, nil+ }, func() int64 { return 0 }, func() bool {+ retries+++ return retries < 2+ })+ if err != nil {+ t.Fatal(err)+ }+ // Each round sends the polls once and drains them before waiting.+ if retries != 2 || published != 6 || f.pulls.Load() != 6 {+ t.Fatalf("retries=%d published=%d pulls=%d", retries, published, f.pulls.Load())+ }+}++func TestPipelinedFramePollsShowTheFirstFrameAfterAnIdlePage(t *testing.T) {+ f := &fakeFrames{timeout: 20 * time.Millisecond}+ f.frame.Store(1)+ panel := &panelFrames{}+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ go func() { _ = pipelineFrames(ctx, 3, f.pull, panel.publish, panel.latest, func() bool { return false }) }()+ waitFor := func(seq int64) {+ t.Helper()+ for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(2 * time.Millisecond) {+ if panel.latest() >= seq {+ return+ }+ }+ t.Fatalf("panel stuck at frame %d, browser at %d", panel.latest(), seq)+ }+ waitFor(1)+ // A still page: every poll times out many times over.+ time.Sleep(400 * time.Millisecond)+ f.frame.Store(2)+ waitFor(2)+ time.Sleep(200 * time.Millisecond)+ f.frame.Store(3)+ waitFor(3)+}diff --git a/internal/httpapi/server.go b/internal/httpapi/server.goindex c777dcba..f0c30a7e 100644--- a/internal/httpapi/server.go+++ b/internal/httpapi/server.go@@ -202,6 +202,7 @@ func newWithNotifications(s *repo.Store, e *engine.Engine, syncSvc *tracker.Serv// 405/404;gin 默认会 301 重定向到无斜杠路径,行为不同,关闭之。g.RedirectTrailingSlash = falseg.Use(api.previews.previewHost())+ g.Use(api.browser.devServerHost())// gin 只有在显式注册 NoMethod 后才会对"路径匹配但方法不符"返回 405,// 否则落入 404;这里同时把 405 响应统一成 JSON 形态。g.NoMethod(func(c *gin.Context) {diff --git a/internal/preview/devserver.go b/internal/preview/devserver.gonew file mode 100644index 00000000..dd680f3b--- /dev/null+++ b/internal/preview/devserver.go@@ -0,0 +1,184 @@+package preview++import (+ _ "embed"+ "encoding/json"+ "net/url"+ "regexp"+ "strconv"+ "strings"+)++//go:embed devserver_intercept.js+var devServerInterceptJS string++// devServerScriptMax bounds rewritten scripts; pre-bundled dependencies of a+// dev server are larger than built HTML.+const devServerScriptMax = 16 << 20++// Root-absolute ES module specifiers: `from "/x"`, `import "/x"`, `import("/x")`.+var devServerImportRe = regexp.MustCompile(`(\bfrom\s*|\bimport\s*\(?\s*)(["'])(/[^/"'\s][^"'\s]*)(["'])`)+var devServerScriptRe = regexp.MustCompile(`(?is)(<script\b[^>]*>)(.*?)(</script>)`)++// RewriteDevServerHTML prepares a dev server document for its path prefix,+// the way build previews are served: root references in markup move under+// the prefix, the history shim hides the prefix from client routers, and the+// intercept script keeps runtime requests, sockets and cookies inside it. No+// <base> is injected: pages resolve relative URLs against their own address.+//+// Module specifiers must be rewritten, not just redirected: a module loaded+// through a redirect keys the module map by the root URL while import.meta.url+// is the prefixed one, and anything importing import.meta.url (React Refresh+// does) would evaluate the module a second time. Requests that still go to+// root paths are redirected under the prefix by the gateway.+func RewriteDevServerHTML(body []byte, prefix string, port int) []byte {+ if len(body) > maxRewrittenHTML {+ return body+ }+ prefix = strings.TrimRight(prefix, "/")+ html := rewriteReferrerMeta(string(body))+ html = devServerScriptRe.ReplaceAllStringFunc(html, func(m string) string {+ parts := devServerScriptRe.FindStringSubmatch(m)+ if len(parts) != 4 || strings.Contains(parts[1], "data-af-") {+ return m+ }+ return parts[1] + rewriteDevServerImports(parts[2], prefix) + parts[3]+ })+ html = rewriteRootRelativeRefs(html, prefix)+ html = rewriteCSSURLs(html, prefix)+ html = injectDevServerIntercept(html, prefix, port)+ html = injectPreviewHistory(html, prefix, true)+ return []byte(html)+}++// RewriteDevServerScript moves root-absolute module specifiers of a dev+// server script under the prefix.+func RewriteDevServerScript(body []byte, prefix string) []byte {+ if len(body) > devServerScriptMax {+ return body+ }+ prefix = strings.TrimRight(prefix, "/")+ src := rewriteDevServerImports(string(body), prefix)+ if strings.Contains(src, "__vite__css") {+ // Vite serves imported stylesheets as modules that inject a <style>.+ src = rewriteCSSURLs(src, prefix)+ }+ return []byte(src)+}++// RewriteDevServerCSS moves root url() references under the prefix.+func RewriteDevServerCSS(body []byte, prefix string) []byte {+ if len(body) > devServerScriptMax {+ return body+ }+ return []byte(rewriteCSSURLs(string(body), strings.TrimRight(prefix, "/")))+}++func rewriteDevServerImports(src, prefix string) string {+ if prefix == "" || !strings.Contains(src, "/") {+ return src+ }+ return devServerImportRe.ReplaceAllStringFunc(src, func(m string) string {+ parts := devServerImportRe.FindStringSubmatch(m)+ if len(parts) != 5 || parts[2] != parts[4] {+ return m+ }+ ref := parts[3]+ if ref == prefix || strings.HasPrefix(ref, prefix+"/") {+ return m+ }+ return parts[1] + parts[2] + prefix + ref + parts[4]+ })+}++// DevServerCookiePrefix namespaces the cookies of one dev server preview+// (its /devpreview/{run}/{port} prefix): the gateway forwards only these to+// the dev server, and the intercept script shows only these to the page.+// With the run in the name, two tasks serving the same port never share a+// cookie, whatever path it was set for.+func DevServerCookiePrefix(prefix string) string {+ var b strings.Builder+ b.WriteString("af_")+ for _, r := range strings.Trim(prefix, "/") {+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' {+ b.WriteRune(r)+ } else {+ b.WriteByte('_')+ }+ }+ b.WriteByte('_')+ return b.String()+}++func injectDevServerIntercept(html, prefix string, port int) string {+ if strings.Contains(html, `data-af-dev-server`) || strings.Contains(devServerInterceptJS, "</script>") {+ return html+ }+ prefixJSON, err := json.Marshal(prefix)+ if err != nil {+ return html+ }+ cookieJSON, _ := json.Marshal(DevServerCookiePrefix(prefix))+ script := strings.NewReplacer("__AF_PREFIX__", string(prefixJSON), "__AF_PORT__", strconv.Itoa(port),+ "__AF_COOKIE__", string(cookieJSON)).Replace(devServerInterceptJS)+ tag := `<script data-af-dev-server>` + script + `</script>`+ // First thing in the document: module scripts and inline boot code must+ // already see the patched fetch/WebSocket.+ lower := strings.ToLower(html)+ if i := strings.Index(lower, "<head>"); i >= 0 {+ at := i + len("<head>")+ return html[:at] + tag + html[at:]+ }+ if i := strings.Index(lower, "<head "); i >= 0 {+ if end := strings.Index(html[i:], ">"); end >= 0 {+ at := i + end + 1+ return html[:at] + tag + html[at:]+ }+ }+ if i := strings.Index(lower, "<html"); i >= 0 {+ if end := strings.Index(html[i:], ">"); end >= 0 {+ at := i + end + 1+ return html[:at] + tag + html[at:]+ }+ }+ return tag + html+}++// RewriteDevServerLocation maps a redirect target of the dev server back+// under the prefix: root paths and http://localhost:PORT addresses move,+// other origins stay as they are.+func RewriteDevServerLocation(raw, prefix string, port int) string {+ raw = strings.TrimSpace(raw)+ prefix = strings.TrimRight(prefix, "/")+ if raw == "" {+ return raw+ }+ u, err := url.Parse(raw)+ if err != nil {+ return raw+ }+ if u.IsAbs() || strings.HasPrefix(raw, "//") {+ host := strings.ToLower(u.Hostname())+ local := host == "localhost" || host == "127.0.0.1" || host == "0.0.0.0" || host == "::1"+ p := u.Port()+ if p == "" {+ p = map[string]string{"https": "443"}[u.Scheme]+ if p == "" {+ p = "80"+ }+ }+ if !local || p != strconv.Itoa(port) {+ return raw+ }+ u.Scheme, u.Host, u.User = "", "", nil+ } else if !strings.HasPrefix(u.Path, "/") {+ return raw+ }+ if u.Path != prefix && !strings.HasPrefix(u.Path, prefix+"/") {+ u.Path = prefix + u.Path+ if u.RawPath != "" {+ u.RawPath = prefix + u.RawPath+ }+ }+ return u.String()+}diff --git a/internal/preview/devserver_intercept.js b/internal/preview/devserver_intercept.jsnew file mode 100644index 00000000..b3e3be0d--- /dev/null+++ b/internal/preview/devserver_intercept.js@@ -0,0 +1,229 @@+(function () {+ // Dev server pages run under a path prefix on the platform origin. This+ // keeps their requests, sockets and cookies inside that prefix: root paths+ // and http://localhost:PORT addresses are moved under it, and cookies are+ // namespaced so the page neither sees nor sends platform cookies.+ var root = __AF_PREFIX__;+ var port = String(__AF_PORT__);+ var loc = window.location;+ if (window.__AGENTFLOW_DEV_SERVER__) return;+ window.__AGENTFLOW_DEV_SERVER__ = { prefix: root, port: Number(port) };+ var localHosts = { localhost: 1, "127.0.0.1": 1, "0.0.0.0": 1, "[::1]": 1 };+ function inside(path) {+ return path === root || path.indexOf(root + "/") === 0;+ }+ function fix(href) {+ if (href == null || href === "") return href;+ var u;+ try {+ u = new URL(String(href), loc.href);+ } catch (e) {+ return href;+ }+ var socket = u.protocol === "ws:" || u.protocol === "wss:";+ if (!socket && u.protocol !== "http:" && u.protocol !== "https:") return href;+ var changed = false;+ if (localHosts[u.hostname] && (u.port || (socket || u.protocol === "http:" ? "80" : "443")) === port) {+ u = new URL(u.pathname + u.search + u.hash, loc.href);+ if (socket) u.protocol = loc.protocol === "https:" ? "wss:" : "ws:";+ changed = true;+ } else if (socket && u.host === loc.host) {+ changed = false;+ } else if (u.origin !== loc.origin) {+ return href;+ }+ if (!inside(u.pathname)) {+ u.pathname = root + u.pathname;+ changed = true;+ }+ return changed ? u.href : href;+ }+ var ofetch = window.fetch;+ if (ofetch) {+ window.fetch = function (input, init) {+ try {+ if (typeof Request !== "undefined" && input instanceof Request) {+ var next = fix(input.url);+ if (next !== input.url) input = new Request(next, input);+ } else if (input != null) {+ input = fix(input instanceof URL ? input.href : input);+ }+ } catch (e) {}+ return ofetch.call(this, input, init);+ };+ }+ var oopen = XMLHttpRequest.prototype.open;+ XMLHttpRequest.prototype.open = function (m, u) {+ if (arguments.length > 1 && u != null) {+ try {+ arguments[1] = fix(u instanceof URL ? u.href : u);+ } catch (e) {}+ }+ return oopen.apply(this, arguments);+ };+ if (navigator.sendBeacon) {+ var obeacon = navigator.sendBeacon.bind(navigator);+ navigator.sendBeacon = function (u, d) {+ return obeacon(fix(u), d);+ };+ }+ if (window.EventSource) {+ var ES = window.EventSource;+ var PatchedES = function (u, o) {+ return new ES(fix(u), o);+ };+ PatchedES.prototype = ES.prototype;+ PatchedES.CONNECTING = ES.CONNECTING;+ PatchedES.OPEN = ES.OPEN;+ PatchedES.CLOSED = ES.CLOSED;+ window.EventSource = PatchedES;+ }+ // Live reload sockets cannot cross the platform yet. A socket that fails+ // makes Vite poll the (reachable) dev server and reload the page forever,+ // so HMR clients get one that simply never connects; the page works and+ // shows edits on refresh.+ function hmrSocket(u, protocols) {+ var list = protocols == null ? [] : [].concat(protocols);+ for (var i = 0; i < list.length; i++) if (/^vite-(hmr|ping)$/.test(String(list[i]))) return true;+ try {+ return /\/_next\/webpack-hmr$/.test(new URL(u).pathname);+ } catch (e) {+ return false;+ }+ }+ if (window.WebSocket) {+ var WS = window.WebSocket;+ var Idle = function (u) {+ var t = typeof EventTarget === "function" ? new EventTarget() : document.createDocumentFragment();+ t.url = String(u);+ t.readyState = 0;+ t.protocol = "";+ t.extensions = "";+ t.bufferedAmount = 0;+ t.binaryType = "blob";+ t.onopen = t.onmessage = t.onerror = t.onclose = null;+ t.send = function () {};+ t.close = function () {+ t.readyState = 3;+ };+ t.CONNECTING = 0;+ t.OPEN = 1;+ t.CLOSING = 2;+ t.CLOSED = 3;+ return t;+ };+ var PatchedWS = function (u, p) {+ var next = fix(u instanceof URL ? u.href : u);+ if (hmrSocket(next, p)) return Idle(next);+ return p === undefined ? new WS(next) : new WS(next, p);+ };+ PatchedWS.prototype = WS.prototype;+ PatchedWS.CONNECTING = WS.CONNECTING;+ PatchedWS.OPEN = WS.OPEN;+ PatchedWS.CLOSING = WS.CLOSING;+ PatchedWS.CLOSED = WS.CLOSED;+ window.WebSocket = PatchedWS;+ }+ // Cookies are namespaced by run and port and kept under the preview+ // prefix, like the gateway does for Set-Cookie: a page writing+ // "sid=1; Path=/" must not leave it at the platform root, where every+ // other preview (and the platform) would see it.+ var cookiePrefix = __AF_COOKIE__;+ function scopeCookie(v) {+ var parts = String(v).split(";");+ var out = [cookiePrefix + parts[0].replace(/^\s+/, "")];+ for (var i = 1; i < parts.length; i++) {+ var attr = parts[i].replace(/^\s+|\s+$/g, "");+ var eq = attr.indexOf("=");+ var name = (eq < 0 ? attr : attr.slice(0, eq)).toLowerCase();+ if (!name || name === "domain") continue;+ if (name === "path") {+ var path = eq < 0 ? "" : attr.slice(eq + 1).replace(/^\s+/, "");+ if (path.charAt(0) !== "/") path = "/";+ if (!inside(path)) path = root + path;+ attr = "Path=" + path;+ }+ out.push(attr);+ }+ return out.join("; ");+ }+ var cookie = Object.getOwnPropertyDescriptor(Document.prototype, "cookie");+ if (cookie && cookie.configurable !== false && cookie.get && cookie.set) {+ Object.defineProperty(document, "cookie", {+ configurable: true,+ get: function () {+ return String(cookie.get.call(document))+ .split(/;\s*/)+ .filter(function (c) {+ return c.indexOf(cookiePrefix) === 0;+ })+ .map(function (c) {+ return c.slice(cookiePrefix.length);+ })+ .join("; ");+ },+ set: function (v) {+ cookie.set.call(document, scopeCookie(v));+ },+ });+ }+ // While the history shim hides the prefix from a booting router, root+ // paths resolve against the bare origin and requests carry no usable+ // Referer, so resource URLs set from script are moved here. Anchors are+ // left alone: routers read them, and the shim routes their clicks.+ var resourceAttrs = { src: 1, href: 1, poster: 1, data: 1 };+ function resource(el, name) {+ var tag = el && el.tagName;+ if (!tag || !resourceAttrs[name]) return false;+ if (name === "href") return tag === "LINK" || tag === "use" || tag === "image";+ return tag !== "A" && tag !== "AREA" && tag !== "FORM" && tag !== "BASE";+ }+ var osetAttribute = Element.prototype.setAttribute;+ Element.prototype.setAttribute = function (name, value) {+ try {+ var lower = String(name).toLowerCase();+ if (resource(this, lower) && typeof value === "string" && value.charAt(0) === "/") value = fix(value);+ } catch (e) {}+ return osetAttribute.call(this, name, value);+ };+ function patchProperty(Ctor, prop) {+ var proto = Ctor && Ctor.prototype;+ var desc = proto && Object.getOwnPropertyDescriptor(proto, prop);+ if (!desc || !desc.set || desc.configurable === false) return;+ Object.defineProperty(proto, prop, {+ configurable: true,+ enumerable: desc.enumerable,+ get: desc.get,+ set: function (value) {+ try {+ if (typeof value === "string" || value instanceof URL) value = fix(String(value));+ } catch (e) {}+ return desc.set.call(this, value);+ },+ });+ }+ [+ [window.HTMLImageElement, "src"],+ [window.HTMLScriptElement, "src"],+ [window.HTMLLinkElement, "href"],+ [window.HTMLSourceElement, "src"],+ [window.HTMLMediaElement, "src"],+ [window.HTMLVideoElement, "poster"],+ [window.HTMLIFrameElement, "src"],+ [window.HTMLTrackElement, "src"],+ [window.HTMLEmbedElement, "src"],+ [window.HTMLObjectElement, "data"],+ [window.HTMLInputElement, "src"],+ ].forEach(function (pair) {+ patchProperty(pair[0], pair[1]);+ });+ var oopenWindow = window.open;+ window.open = function (u) {+ if (arguments.length > 0 && u != null && u !== "") {+ try {+ arguments[0] = fix(u instanceof URL ? u.href : u);+ } catch (e) {}+ }+ return oopenWindow.apply(this, arguments);+ };+})();diff --git a/internal/preview/devserver_test.go b/internal/preview/devserver_test.gonew file mode 100644index 00000000..0cf38e05--- /dev/null+++ b/internal/preview/devserver_test.go@@ -0,0 +1,98 @@+package preview++import (+ "strings"+ "testing"+)++func TestRewriteDevServerHTML(t *testing.T) {+ in := `<!doctype html><html><head><meta name="referrer" content="no-referrer"><script type="module" src="/@vite/client"></script></head><body><img src="/logo.svg"></body></html>`+ out := string(RewriteDevServerHTML([]byte(in), "/devpreview/r/5173/", 5173))+ intercept := strings.Index(out, "<script data-af-dev-server>")+ if intercept < 0 || intercept > strings.Index(out, `src="/devpreview/r/5173/@vite/client"`) {+ t.Fatalf("intercept must run before page scripts: %s", out)+ }+ if !strings.Contains(out, `var root = "/devpreview/r/5173"`) || !strings.Contains(out, `String(5173)`) {+ t.Fatalf("intercept placeholders: %s", out)+ }+ if !strings.Contains(out, "data-af-preview-history") || !strings.Contains(out, `content="same-origin"`) {+ t.Fatalf("history shim / referrer: %s", out)+ }+ if !strings.Contains(out, `src="/devpreview/r/5173/logo.svg"`) || !strings.Contains(out, `src="/devpreview/r/5173/@vite/client"`) || strings.Contains(out, "<base") {+ t.Fatalf("markup references: %s", out)+ }+ if again := string(RewriteDevServerHTML([]byte(out), "/devpreview/r/5173", 5173)); strings.Count(again, "data-af-dev-server") != 1 {+ t.Fatal("rewrite must be idempotent")+ }+}++func TestRewriteDevServerModuleSpecifiers(t *testing.T) {+ prefix := "/devpreview/r/5173"+ html := string(RewriteDevServerHTML([]byte(`<head><script type="module">import { injectIntoGlobalHook } from "/@react-refresh";</script></head>`), prefix, 5173))+ if !strings.Contains(html, `from "/devpreview/r/5173/@react-refresh"`) {+ t.Fatalf("inline module: %s", html)+ }+ js := `import React from "/node_modules/.vite/deps/react.js?v=1";import "/src/a.css";import{x}from'/src/x.ts';` ++ `const m = import ( "/src/lazy.tsx" );export * from "/src/re.ts";` ++ `Array.from("/not/a/module");fetch("/api");importScripts("/sw.js");import "//cdn.example.com/a.js";import "./rel.js";` ++ `import "/devpreview/r/5173/done.js";`+ out := string(RewriteDevServerScript([]byte(js), prefix))+ for _, want := range []string{+ `from "/devpreview/r/5173/node_modules/.vite/deps/react.js?v=1"`,+ `import "/devpreview/r/5173/src/a.css"`,+ `from'/devpreview/r/5173/src/x.ts'`,+ `import ( "/devpreview/r/5173/src/lazy.tsx" )`,+ `export * from "/devpreview/r/5173/src/re.ts"`,+ `Array.from("/not/a/module")`,+ `fetch("/api")`,+ `importScripts("/sw.js")`,+ `import "//cdn.example.com/a.js"`,+ `import "./rel.js"`,+ `import "/devpreview/r/5173/done.js"`,+ } {+ if !strings.Contains(out, want) {+ t.Fatalf("missing %s in %s", want, out)+ }+ }+ viteCSS := string(RewriteDevServerScript([]byte(`const __vite__css = ".card{background:url(/bg.svg)}"`), prefix))+ if !strings.Contains(viteCSS, `url(/devpreview/r/5173/bg.svg)`) {+ t.Fatalf("vite css module: %s", viteCSS)+ }+ if plain := string(RewriteDevServerScript([]byte(`const s = "url(/x)"`), prefix)); plain != `const s = "url(/x)"` {+ t.Fatalf("plain script urls must stay: %s", plain)+ }+ if css := string(RewriteDevServerCSS([]byte(`.a{background:url(/bg.svg)}`), prefix)); css != `.a{background:url(/devpreview/r/5173/bg.svg)}` {+ t.Fatalf("css: %s", css)+ }+}++func TestRewriteDevServerLocation(t *testing.T) {+ prefix := "/devpreview/r/5173"+ for raw, want := range map[string]string{+ "/login?next=/a": prefix + "/login?next=/a",+ "http://localhost:5173/home#x": prefix + "/home#x",+ "http://127.0.0.1:5173": prefix,+ prefix + "/already": prefix + "/already",+ "http://localhost:3000/other": "http://localhost:3000/other",+ "https://sso.example.com/login": "https://sso.example.com/login",+ "relative/path": "relative/path",+ } {+ if got := RewriteDevServerLocation(raw, prefix, 5173); got != want {+ t.Fatalf("%q: got %q want %q", raw, got, want)+ }+ }+}++func TestDevServerCookiePrefixSeparatesRunsOnOnePort(t *testing.T) {+ a, b := DevServerCookiePrefix("/devpreview/run-a/3000"), DevServerCookiePrefix("/devpreview/run-b/3000/")+ if a != "af_devpreview_run-a_3000_" || b != "af_devpreview_run-b_3000_" {+ t.Fatalf("prefixes %q %q", a, b)+ }+ if got := DevServerCookiePrefix("/devpreview/r.x;y/file"); got != "af_devpreview_r_x_y_file_" {+ t.Fatalf("cookie names allow only token characters: %q", got)+ }+ html := string(RewriteDevServerHTML([]byte("<html><head></head><body></body></html>"), "/devpreview/run-a/3000", 3000))+ if !strings.Contains(html, `var cookiePrefix = "af_devpreview_run-a_3000_"`) || strings.Contains(html, "__AF_") {+ t.Fatalf("intercept script must carry the run's cookie namespace")+ }+}diff --git a/internal/runtime/browser.go b/internal/runtime/browser.goindex 090fef08..a1eb90e0 100644--- a/internal/runtime/browser.go+++ b/internal/runtime/browser.go@@ -63,6 +63,8 @@ type browserEndpoint struct {Kind stringContainer stringWebSocket string+ // Workspace is the host directory a local browser serves file:// from.+ Workspace string}func (e browserEndpoint) dial(ctx context.Context) (cdpTransport, error) {@@ -130,6 +132,18 @@ func browserUsageHint(servers []domain.MCPServerDefinition) string {return ""}+// BrowserDeviceScale is the device scale factor run browsers render at+// (AGENTFLOW_BROWSER_SCALE, 1 to 3, default 2). The task panel shows their+// screencast on high-DPI screens, where 1x frames are upscaled and blurred;+// screencast frames follow the browser's real scale, not an emulated one.+func BrowserDeviceScale() string {+ raw := strings.TrimSpace(os.Getenv("AGENTFLOW_BROWSER_SCALE"))+ if value, err := strconv.ParseFloat(raw, 64); err == nil && value >= 1 && value <= 3 {+ return strconv.FormatFloat(value, 'f', -1, 64)+ }+ return "2"+}+func dockerBrowserContainerName(base string) string { return base + "-browser" }var browserImageMu sync.Mutex@@ -234,7 +248,7 @@ func (s *dockerSandbox) ensureBrowser(ctx context.Context, runID string, sink Ev"--label", browserRunLabel + "=" + runID, "--label", browserOwnerLabel + "=" + s.name,"--network", "container:" + s.name, "--cap-drop=ALL", "--security-opt", "no-new-privileges","--shm-size", "512m", "-v", s.workspace + ":/workspace", "-w", "/workspace",- "-e", "AGENTFLOW_BROWSER_OUTPUT=/workspace/.agentflow/browser", image}+ "-e", "AGENTFLOW_BROWSER_OUTPUT=/workspace/.agentflow/browser", "-e", "AGENTFLOW_BROWSER_SCALE=" + BrowserDeviceScale(), image}if out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput(); err != nil {return domain.MCPServerDefinition{}, fmt.Errorf("启动浏览器容器失败: %w: %s", err, tailOutput(out))}@@ -323,7 +337,7 @@ func ensureHostBrowser(ctx context.Context, workspace, runID string) (domain.MCP_ = os.Remove(filepath.Join(profile, "DevToolsActivePort"))args := []string{"--headless=new", "--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0","--user-data-dir=" + profile, "--no-first-run", "--no-default-browser-check", "--disable-background-networking",- "--window-size=1280,800", "--lang=zh-CN"}+ "--window-size=1280,800", "--lang=zh-CN", "--force-device-scale-factor=" + BrowserDeviceScale()}if goruntime.GOOS == "linux" && os.Geteuid() == 0 {args = append(args, "--no-sandbox", "--disable-dev-shm-usage")}@@ -508,7 +522,7 @@ func lookupBrowserEndpoint(ctx context.Context, runID string) (browserEndpoint,if raw, err := os.ReadFile(filepath.Join(hostBrowserDir(runID), "endpoint.json")); err == nil {var record hostBrowserRecordif json.Unmarshal(raw, &record) == nil && record.PID > 0 && syscall.Kill(record.PID, 0) == nil && hostDevToolsAlive(ctx, record.Port) {- return browserEndpoint{Kind: "local", WebSocket: record.WebSocket}, true+ return browserEndpoint{Kind: "local", WebSocket: record.WebSocket, Workspace: record.Workspace}, true}}if !dockerAvailable() {diff --git a/internal/runtime/browser_agent.go b/internal/runtime/browser_agent.goindex b6b6e7f6..caada619 100644--- a/internal/runtime/browser_agent.go+++ b/internal/runtime/browser_agent.go@@ -211,10 +211,33 @@ func (v *browserView) screenshot(ctx context.Context) (string, error) {if session == "" {return "", errors.New("没有可操作的标签页")}+ // The browser renders at a high device scale for the panel; the model+ // gets CSS-pixel screenshots, which carry the same content in a quarter+ // of the image.+ params := map[string]any{"format": "jpeg", "quality": 75}+ var metrics struct {+ CSSVisualViewport struct {+ PageX float64 `json:"pageX"`+ PageY float64 `json:"pageY"`+ ClientWidth float64 `json:"clientWidth"`+ ClientHeight float64 `json:"clientHeight"`+ } `json:"cssVisualViewport"`+ }+ var ratio struct {+ Result struct {+ Value float64 `json:"value"`+ } `json:"result"`+ }+ if v.client.call(ctx, session, "Page.getLayoutMetrics", nil, &metrics) == nil &&+ v.client.call(ctx, session, "Runtime.evaluate", map[string]any{"expression": "window.devicePixelRatio", "returnByValue": true}, &ratio) == nil &&+ ratio.Result.Value > 1 && metrics.CSSVisualViewport.ClientWidth > 0 {+ vp := metrics.CSSVisualViewport+ params["clip"] = map[string]any{"x": vp.PageX, "y": vp.PageY, "width": vp.ClientWidth, "height": vp.ClientHeight, "scale": 1 / ratio.Result.Value}+ }var shot struct {Data string `json:"data"`}- if err := v.client.call(ctx, session, "Page.captureScreenshot", map[string]any{"format": "jpeg", "quality": 75}, &shot); err != nil {+ if err := v.client.call(ctx, session, "Page.captureScreenshot", params, &shot); err != nil {return "", err}return shot.Data, nildiff --git a/internal/runtime/browser_assets/entrypoint.sh b/internal/runtime/browser_assets/entrypoint.shindex 0992ff5a..6bf2c12d 100644--- a/internal/runtime/browser_assets/entrypoint.sh+++ b/internal/runtime/browser_assets/entrypoint.sh@@ -10,7 +10,8 @@ mkdir -p "$profile" "$output"chromium --headless=new --no-sandbox --disable-dev-shm-usage --disable-gpu \--no-first-run --no-default-browser-check --disable-background-networking \--remote-debugging-address=127.0.0.1 --remote-debugging-port="${AGENTFLOW_BROWSER_CDP_PORT:-9222}" \- --user-data-dir="$profile" --window-size=1280,800 --lang=zh-CN about:blank >/tmp/chromium.log 2>&1+ --user-data-dir="$profile" --window-size=1280,800 --lang=zh-CN \+ --force-device-scale-factor="${AGENTFLOW_BROWSER_SCALE:-2}" about:blank >/tmp/chromium.log 2>&1sleep 1done) &diff --git a/internal/runtime/browser_devserver.go b/internal/runtime/browser_devserver.gonew file mode 100644index 00000000..6102903e--- /dev/null+++ b/internal/runtime/browser_devserver.go@@ -0,0 +1,317 @@+package runtime++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "io"+ "log"+ "net"+ "net/http"+ "os/exec"+ "strconv"+ "strings"+ "sync"+ "syscall"+ "time"++ "agentflow/internal/domain"+)++// Dev server access: the user's own browser renders a dev server that runs+// in the run sandbox, and this side forwards each HTTP request to+// localhost:port in the sandbox's network namespace. Docker sandboxes are+// reached through the browser sidecar (it shares that namespace and ships+// node); host drivers reach the port directly.++const (+ // DevServerBodyLimit keeps a response, base64 encoded, inside the runner+ // socket's 20 MiB message limit.+ DevServerBodyLimit = 12 << 20+ devServerEndpointTTL = 15 * time.Second+ devServerClientIdle = 10 * time.Minute+)++// ErrDevServerUnreachable means nothing accepted a connection on the port.+var ErrDevServerUnreachable = errors.New("dev server unreachable")++type devServerClient struct {+ endpoint browserEndpoint+ checked time.Time+ used time.Time+ client *http.Client+}++var devServerClients = struct {+ sync.Mutex+ items map[string]*devServerClient+}{items: map[string]*devServerClient{}}++// devServerDial is swapped in tests.+var devServerDial = func(ctx context.Context, endpoint browserEndpoint, port int) (net.Conn, error) {+ if endpoint.Kind == "docker" {+ return dialDockerPort(ctx, endpoint.Container, port)+ }+ var d net.Dialer+ return d.DialContext(ctx, "tcp", net.JoinHostPort("localhost", strconv.Itoa(port)))+}++// devServerLookup is swapped in tests.+var devServerLookup = lookupBrowserEndpoint++func handleDevServerHTTP(ctx context.Context, runID string, req *domain.BrowserHTTPRequest) (domain.BrowserState, error) {+ if req == nil || req.Port <= 0 || req.Port > 65535 || !strings.HasPrefix(req.Path, "/") {+ return domain.BrowserState{}, errors.New("无效的开发服务器请求")+ }+ client, endpoint, ok := devServerClientFor(ctx, runID)+ if !ok {+ return domain.BrowserState{Available: false, Reason: "Agent 浏览器尚未启动:开发服务器预览需要它所在的沙箱网络"}, nil+ }+ method := strings.ToUpper(strings.TrimSpace(req.Method))+ if method == "" {+ method = http.MethodGet+ }+ target := "http://localhost:" + strconv.Itoa(req.Port) + req.Path+ out, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(req.Body))+ if err != nil {+ return domain.BrowserState{}, err+ }+ for name, values := range req.Header {+ if devServerHopHeader(name) || strings.EqualFold(name, "Host") {+ continue+ }+ for _, v := range values {+ out.Header.Add(name, v)+ }+ }+ // Dev servers check Host against their allow list (Vite allowedHosts).+ out.Host = "localhost:" + strconv.Itoa(req.Port)+ res, err := client.Do(out)+ if err != nil {+ if errors.Is(err, ErrDevServerUnreachable) {+ forgetDevServerEndpoint(runID, endpoint)+ }+ var unreachable devServerDialError+ if errors.As(err, &unreachable) {+ if devServerRefused(unreachable.err) {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("端口 %d 上没有正在监听的开发服务器", req.Port)}, nil+ }+ log.Printf("browser dev server dial run=%s kind=%s container=%s port=%d: %v", runID, endpoint.Kind, endpoint.Container, req.Port, unreachable.err)+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("连接端口 %d 失败:%v", req.Port, unreachable.err)}, nil+ }+ if ctx.Err() != nil {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("端口 %d 的开发服务器响应超时", req.Port)}, nil+ }+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "访问开发服务器失败:" + err.Error()}, nil+ }+ defer res.Body.Close()+ body, err := io.ReadAll(io.LimitReader(res.Body, DevServerBodyLimit+1))+ if err != nil {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "读取开发服务器响应失败:" + err.Error()}, nil+ }+ if len(body) > DevServerBodyLimit {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("响应超过 %d MB,预览暂不支持", DevServerBodyLimit>>20)}, nil+ }+ header := map[string][]string{}+ for name, values := range res.Header {+ if devServerHopHeader(name) || strings.EqualFold(name, "Content-Length") {+ continue+ }+ header[name] = values+ }+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{Status: res.StatusCode, Header: header, Body: body}}, nil+}++var devServerHop = map[string]bool{+ "connection": true, "keep-alive": true, "proxy-connection": true, "proxy-authenticate": true,+ "proxy-authorization": true, "te": true, "trailer": true, "transfer-encoding": true, "upgrade": true,+}++func devServerHopHeader(name string) bool { return devServerHop[strings.ToLower(name)] }++func devServerClientFor(ctx context.Context, runID string) (*http.Client, browserEndpoint, bool) {+ now := time.Now()+ devServerClients.Lock()+ for id, c := range devServerClients.items {+ if now.Sub(c.used) > devServerClientIdle {+ c.client.CloseIdleConnections()+ delete(devServerClients.items, id)+ }+ }+ cached := devServerClients.items[runID]+ if cached != nil && now.Sub(cached.checked) < devServerEndpointTTL {+ cached.used = now+ devServerClients.Unlock()+ return cached.client, cached.endpoint, true+ }+ devServerClients.Unlock()+ endpoint, ok := devServerLookup(ctx, runID)+ if !ok {+ return nil, browserEndpoint{}, false+ }+ devServerClients.Lock()+ defer devServerClients.Unlock()+ cached = devServerClients.items[runID]+ if cached != nil && cached.endpoint == endpoint {+ cached.checked, cached.used = now, now+ return cached.client, endpoint, true+ }+ if cached != nil {+ cached.client.CloseIdleConnections()+ }+ transport := &http.Transport{+ DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {+ _, rawPort, err := net.SplitHostPort(addr)+ if err != nil {+ return nil, err+ }+ port, err := strconv.Atoi(rawPort)+ if err != nil {+ return nil, err+ }+ conn, err := devServerDial(ctx, endpoint, port)+ if err != nil {+ return nil, devServerDialError{err}+ }+ return conn, nil+ },+ // Each docker connection is a process; keep a few warm per run so a+ // page load does not pay a docker exec per request.+ MaxIdleConnsPerHost: 8,+ IdleConnTimeout: 90 * time.Second,+ ResponseHeaderTimeout: 2 * time.Minute,+ ForceAttemptHTTP2: false,+ }+ client := &http.Client{+ Transport: transport,+ CheckRedirect: func(*http.Request, []*http.Request) error {+ return http.ErrUseLastResponse+ },+ }+ devServerClients.items[runID] = &devServerClient{endpoint: endpoint, checked: now, used: now, client: client}+ return client, endpoint, true+}++func forgetDevServerEndpoint(runID string, endpoint browserEndpoint) {+ devServerClients.Lock()+ defer devServerClients.Unlock()+ if c := devServerClients.items[runID]; c != nil && c.endpoint == endpoint {+ c.checked = time.Time{}+ }+}++// devServerRefused tells a port nobody listens on from a relay that could+// not reach the sandbox at all: only refusals (and a loopback family the+// sandbox lacks) mean the dev server is not up.+func devServerRefused(err error) bool {+ if errors.Is(err, syscall.ECONNREFUSED) {+ return true+ }+ refused := false+ for _, code := range strings.Split(strings.TrimSpace(err.Error()), ",") {+ switch strings.TrimSpace(code) {+ case "ECONNREFUSED":+ refused = true+ case "EADDRNOTAVAIL", "ENETUNREACH", "EAFNOSUPPORT":+ default:+ return false+ }+ }+ return refused+}++type devServerDialError struct{ err error }++func (e devServerDialError) Error() string { return e.err.Error() }+func (e devServerDialError) Unwrap() error { return ErrDevServerUnreachable }++// devServerPipeScript connects stdio to localhost:port inside the sidecar.+// It prints one 0x01 byte once connected, so a refused port fails the dial+// instead of surfacing later as an empty response. Vite and friends may bind+// either loopback family, so ::1 is tried after 127.0.0.1; a failure+// reports each attempt's error code, comma separated.+const devServerPipeScript = `const net=require("net");const port=+process.argv[1];const hosts=["127.0.0.1","::1"];const codes=[];` ++ `(function go(){const s=net.connect({port,host:hosts.shift()});let up=false;` ++ `s.once("connect",()=>{up=true;process.stdout.write("\x01");process.stdin.pipe(s);s.on("data",d=>process.stdout.write(d));s.on("close",()=>process.stdout.write("",()=>process.exit(0)));});` ++ `s.on("error",e=>{if(up)return;codes.push(String(e.code||e));if(hosts.length)return go();process.stderr.write(codes.join(","));process.exit(2);});})();`++func dialDockerPort(ctx context.Context, container string, port int) (net.Conn, error) {+ return dialExecPort(ctx, "docker", "exec", "-i", container, "node", "-e", devServerPipeScript, strconv.Itoa(port))+}++// dialExecPort runs a command that relays its stdio to a TCP port.+func dialExecPort(ctx context.Context, name string, args ...string) (net.Conn, error) {+ cmd := exec.Command(name, args...)+ stdin, err := cmd.StdinPipe()+ if err != nil {+ return nil, err+ }+ stdout, err := cmd.StdoutPipe()+ if err != nil {+ return nil, err+ }+ var stderr bytes.Buffer+ cmd.Stderr = &stderr+ if err := cmd.Start(); err != nil {+ return nil, err+ }+ conn := &execConn{cmd: cmd, stdin: stdin, stdout: stdout}+ ready := make(chan error, 1)+ go func() {+ var b [1]byte+ _, err := io.ReadFull(stdout, b[:])+ if err == nil && b[0] != 1 {+ err = errors.New("unexpected handshake")+ }+ ready <- err+ }()+ select {+ case err := <-ready:+ if err != nil {+ _ = conn.Close()+ if msg := strings.TrimSpace(stderr.String()); msg != "" {+ return nil, errors.New(msg)+ }+ return nil, err+ }+ return conn, nil+ case <-ctx.Done():+ _ = conn.Close()+ return nil, ctx.Err()+ case <-time.After(15 * time.Second):+ _ = conn.Close()+ return nil, errors.New("连接开发服务器超时")+ }+}++// execConn is a net.Conn over a child process's stdio.+type execConn struct {+ cmd *exec.Cmd+ stdin io.WriteCloser+ stdout io.ReadCloser+ once sync.Once+}++func (c *execConn) Read(p []byte) (int, error) { return c.stdout.Read(p) }+func (c *execConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }+func (c *execConn) Close() error {+ c.once.Do(func() {+ _ = c.stdin.Close()+ if c.cmd.Process != nil {+ _ = c.cmd.Process.Kill()+ }+ go func() { _ = c.cmd.Wait() }()+ })+ return nil+}+func (c *execConn) LocalAddr() net.Addr { return execAddr{} }+func (c *execConn) RemoteAddr() net.Addr { return execAddr{} }+func (c *execConn) SetDeadline(time.Time) error { return nil }+func (c *execConn) SetReadDeadline(time.Time) error { return nil }+func (c *execConn) SetWriteDeadline(time.Time) error { return nil }++type execAddr struct{}++func (execAddr) Network() string { return "exec" }+func (execAddr) String() string { return "docker-exec" }diff --git a/internal/runtime/browser_devserver_test.go b/internal/runtime/browser_devserver_test.gonew file mode 100644index 00000000..f8b99f07--- /dev/null+++ b/internal/runtime/browser_devserver_test.go@@ -0,0 +1,122 @@+package runtime++import (+ "bufio"+ "context"+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "os/exec"+ "strconv"+ "strings"+ "syscall"+ "testing"++ "agentflow/internal/domain"+)++func TestDevServerPipeScriptRelaysHTTP(t *testing.T) {+ if _, err := exec.LookPath("node"); err != nil {+ t.Skip("node not installed")+ }+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.WriteString(w, "hello "+r.URL.Path)+ }))+ defer dev.Close()+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))+ conn, err := dialExecPort(context.Background(), "node", "-e", devServerPipeScript, rawPort)+ if err != nil {+ t.Fatal(err)+ }+ defer conn.Close()+ for i := 0; i < 2; i++ { // keep-alive: two requests on one relay+ req, _ := http.NewRequest("GET", "http://localhost:"+rawPort+"/p"+strconv.Itoa(i), nil)+ if err := req.Write(conn); err != nil {+ t.Fatal(err)+ }+ res, err := http.ReadResponse(bufio.NewReader(conn), req)+ if err != nil {+ t.Fatal(err)+ }+ body, _ := io.ReadAll(res.Body)+ if string(body) != "hello /p"+strconv.Itoa(i) {+ t.Fatalf("body %q", body)+ }+ }++ closed, _ := net.Listen("tcp", "127.0.0.1:0")+ _, idle, _ := net.SplitHostPort(closed.Addr().String())+ closed.Close()+ if _, err := dialExecPort(context.Background(), "node", "-e", devServerPipeScript, idle); err == nil || !strings.Contains(err.Error(), "ECONNREFUSED") {+ t.Fatalf("refused port: %v", err)+ }+}++func TestHandleDevServerHTTPThroughEndpoint(t *testing.T) {+ dev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/plain")+ w.Header().Set("Connection", "keep-alive")+ _, _ = io.WriteString(w, r.Method+" "+r.URL.RequestURI()+" "+r.Host)+ }))+ defer dev.Close()+ _, rawPort, _ := net.SplitHostPort(strings.TrimPrefix(dev.URL, "http://"))+ port, _ := strconv.Atoi(rawPort)+ lookup := devServerLookup+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{Kind: "local"}, true }+ t.Cleanup(func() { devServerLookup = lookup })++ state, err := HandleBrowser(context.Background(), "dev-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Method: "POST", Path: "/a?b=1"}})+ if err != nil || state.HTTP == nil {+ t.Fatalf("state=%+v err=%v", state, err)+ }+ if got := string(state.HTTP.Body); got != "POST /a?b=1 localhost:"+rawPort {+ t.Fatalf("body %q", got)+ }+ if _, hop := state.HTTP.Header["Connection"]; hop {+ t.Fatal("hop-by-hop header leaked")+ }+ if _, err := HandleBrowser(context.Background(), "dev-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Path: "no-slash"}}); err == nil {+ t.Fatal("invalid path accepted")+ }+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{}, false }+ devServerClients.Lock()+ delete(devServerClients.items, "other-run")+ devServerClients.Unlock()+ if state, _ := HandleBrowser(context.Background(), "other-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: port, Path: "/"}}); state.Available || state.HTTP != nil {+ t.Fatalf("missing browser: %+v", state)+ }+}++func TestDevServerDialFailuresAreNotReportedAsIdlePorts(t *testing.T) {+ for raw, want := range map[string]bool{+ "ECONNREFUSED,ECONNREFUSED": true,+ "ECONNREFUSED,EADDRNOTAVAIL": true,+ "EADDRNOTAVAIL,ENETUNREACH": false,+ "ETIMEDOUT,ECONNREFUSED": false,+ "exit status 126": false,+ "连接开发服务器超时": false,+ } {+ if got := devServerRefused(errors.New(raw)); got != want {+ t.Fatalf("%q: %v", raw, got)+ }+ }+ if !devServerRefused(fmt.Errorf("dial: %w", syscall.ECONNREFUSED)) {+ t.Fatal("host refusal")+ }++ lookup, dial := devServerLookup, devServerDial+ t.Cleanup(func() { devServerLookup, devServerDial = lookup, dial })+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {+ return browserEndpoint{Kind: "docker", Container: "dial-sidecar"}, true+ }+ devServerDial = func(context.Context, browserEndpoint, int) (net.Conn, error) {+ return nil, errors.New(`OCI runtime exec failed: exec: "node": executable file not found`)+ }+ state, err := HandleBrowser(context.Background(), "dial-run", domain.BrowserRequest{Operation: "http", HTTP: &domain.BrowserHTTPRequest{Port: 8000, Path: "/"}})+ if err != nil || state.HTTP != nil || !strings.Contains(state.Reason, "连接端口 8000 失败") || !strings.Contains(state.Reason, "executable file not found") {+ t.Fatalf("state=%+v err=%v", state, err)+ }+}diff --git a/internal/runtime/browser_file.go b/internal/runtime/browser_file.gonew file mode 100644index 00000000..6a70cecb--- /dev/null+++ b/internal/runtime/browser_file.go@@ -0,0 +1,165 @@+package runtime++import (+ "context"+ "errors"+ "fmt"+ "mime"+ "net/http"+ "os"+ "os/exec"+ "path"+ "path/filepath"+ "strconv"+ "strings"++ "agentflow/internal/domain"+)++// Workspace file access: pages the Agent opens as file:// are served to the+// user's own browser so they render natively, like dev servers. Paths are+// the ones the run browser sees (/workspace/... in a docker sidecar, the host+// workspace for a host browser) and must resolve inside that workspace.++// dockerWorkspaceRoot is where the browser sidecar mounts the workspace.+const dockerWorkspaceRoot = "/workspace"++// Outcomes shared by the sidecar script (exit codes) and the host reader.+const (+ workspaceFileOK = 0+ workspaceFileMissing = 44+ workspaceFileOutside = 45+ workspaceFileDirectory = 46 // a directory asked for without a trailing slash+ workspaceFileTooLarge = 47+ workspaceFileNoIndexHTML = 48+)++// workspaceFileScript prints a workspace file inside the sidecar:+// node -e script ROOT PATH LIMIT. Symlinks are resolved before the+// containment check, and a directory serves its index.html.+const workspaceFileScript = `const fs=require("fs"),path=require("path");const[root0,p,limit]=process.argv.slice(1);` ++ `const real=f=>{try{return fs.realpathSync(f)}catch(e){process.exit(44)}};const root=real(root0);` ++ `const inside=r=>r===root||r.startsWith(root.endsWith("/")?root:root+"/");` ++ `let r=real(p);if(!inside(r))process.exit(45);let st=fs.statSync(r);` ++ `if(st.isDirectory()){if(!p.endsWith("/"))process.exit(46);const i=path.join(r,"index.html");if(!fs.existsSync(i))process.exit(48);` ++ `r=real(i);if(!inside(r))process.exit(45);st=fs.statSync(r);}` ++ `if(!st.isFile())process.exit(44);if(st.size>+limit)process.exit(47);process.stdout.write(fs.readFileSync(r),()=>process.exit(0));`++// workspaceFileExec runs the script in a sidecar; swapped in tests.+var workspaceFileExec = func(ctx context.Context, container, file string) ([]byte, int, error) {+ out, err := exec.CommandContext(ctx, "docker", "exec", container, "node", "-e", workspaceFileScript,+ dockerWorkspaceRoot, file, strconv.Itoa(DevServerBodyLimit)).Output()+ var exit *exec.ExitError+ if errors.As(err, &exit) && exit.ExitCode() >= workspaceFileMissing && exit.ExitCode() <= workspaceFileNoIndexHTML {+ return nil, exit.ExitCode(), nil+ }+ if err != nil {+ return nil, 0, err+ }+ return out, workspaceFileOK, nil+}++func handleWorkspaceFile(ctx context.Context, runID, file string) (domain.BrowserState, error) {+ if !strings.HasPrefix(file, "/") || strings.ContainsRune(file, 0) {+ return domain.BrowserState{}, errors.New("无效的工作区文件路径")+ }+ endpoint, ok := devServerLookup(ctx, runID)+ if !ok {+ return domain.BrowserState{Available: false, Reason: "Agent 浏览器尚未启动:工作区文件预览需要它所在的执行环境"}, nil+ }+ var (+ body []byte+ code int+ err error+ )+ if endpoint.Kind == "docker" {+ body, code, err = workspaceFileExec(ctx, endpoint.Container, file)+ } else if endpoint.Workspace == "" {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "无法确定 Agent 浏览器的工作区目录"}, nil+ } else {+ body, code = serveWorkspaceFile(endpoint.Workspace, file, DevServerBodyLimit)+ }+ if err != nil {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: "读取工作区文件失败:" + err.Error()}, nil+ }+ text := func(status int, message string) (domain.BrowserState, error) {+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{+ Status: status, Header: map[string][]string{"Content-Type": {"text/plain; charset=utf-8"}}, Body: []byte(message),+ }}, nil+ }+ switch code {+ case workspaceFileOK:+ case workspaceFileMissing:+ return text(http.StatusNotFound, "文件不存在:"+file)+ case workspaceFileOutside:+ return text(http.StatusForbidden, "只能预览工作区内的文件")+ case workspaceFileDirectory:+ // The gateway adds the slash, so relative links resolve in the directory.+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{Status: http.StatusMovedPermanently}}, nil+ case workspaceFileNoIndexHTML:+ return text(http.StatusNotFound, "目录下没有 index.html:"+file)+ case workspaceFileTooLarge:+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, Reason: fmt.Sprintf("文件超过 %d MB,预览暂不支持", DevServerBodyLimit>>20)}, nil+ }+ name := file+ if strings.HasSuffix(name, "/") {+ name += "index.html"+ }+ contentType := mime.TypeByExtension(path.Ext(name))+ if contentType == "" {+ contentType = http.DetectContentType(body)+ }+ return domain.BrowserState{Available: true, Kind: endpoint.Kind, HTTP: &domain.BrowserHTTPResponse{+ Status: http.StatusOK, Header: map[string][]string{"Content-Type": {contentType}}, Body: body,+ }}, nil+}++// serveWorkspaceFile is the host browser's counterpart of workspaceFileScript.+func serveWorkspaceFile(root, file string, limit int64) ([]byte, int) {+ realRoot, err := filepath.EvalSymlinks(root)+ if err != nil {+ return nil, workspaceFileMissing+ }+ inside := func(p string) bool {+ rel, err := filepath.Rel(realRoot, p)+ return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))+ }+ real, err := filepath.EvalSymlinks(filepath.FromSlash(file))+ if err != nil {+ return nil, workspaceFileMissing+ }+ if !inside(real) {+ return nil, workspaceFileOutside+ }+ info, err := os.Stat(real)+ if err != nil {+ return nil, workspaceFileMissing+ }+ if info.IsDir() {+ if !strings.HasSuffix(file, "/") {+ return nil, workspaceFileDirectory+ }+ index, err := filepath.EvalSymlinks(filepath.Join(real, "index.html"))+ if err != nil {+ return nil, workspaceFileNoIndexHTML+ }+ if !inside(index) {+ return nil, workspaceFileOutside+ }+ if info, err = os.Stat(index); err != nil {+ return nil, workspaceFileNoIndexHTML+ }+ real = index+ }+ if !info.Mode().IsRegular() {+ return nil, workspaceFileMissing+ }+ if info.Size() > limit {+ return nil, workspaceFileTooLarge+ }+ body, err := os.ReadFile(real)+ if err != nil {+ return nil, workspaceFileMissing+ }+ return body, workspaceFileOK+}diff --git a/internal/runtime/browser_file_test.go b/internal/runtime/browser_file_test.gonew file mode 100644index 00000000..ea034199--- /dev/null+++ b/internal/runtime/browser_file_test.go@@ -0,0 +1,151 @@+package runtime++import (+ "context"+ "errors"+ "os"+ "os/exec"+ "path/filepath"+ "strconv"+ "strings"+ "testing"++ "agentflow/internal/domain"+)++// workspaceFileTree lays out a workspace plus a file outside it.+func workspaceFileTree(t *testing.T) string {+ t.Helper()+ base := t.TempDir()+ root := filepath.Join(base, "ws")+ for name, body := range map[string]string{+ "ws/demo.html": "<h1>demo</h1>",+ "ws/site/index.html": "<p>site</p>",+ "ws/site/app.js": "console.log(1)",+ "ws/empty/.keep": "",+ "ws/big.txt": strings.Repeat("x", 64),+ "secret.txt": "secret",+ } {+ p := filepath.Join(base, filepath.FromSlash(name))+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {+ t.Fatal(err)+ }+ }+ if err := os.Symlink(filepath.Join(base, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil {+ t.Fatal(err)+ }+ return root+}++var workspaceFileCases = []struct {+ path string+ code int+ body string+}{+ {"/demo.html", workspaceFileOK, "<h1>demo</h1>"},+ {"/site/", workspaceFileOK, "<p>site</p>"},+ {"/site/app.js", workspaceFileOK, "console.log(1)"},+ {"/site", workspaceFileDirectory, ""},+ {"/empty/", workspaceFileNoIndexHTML, ""},+ {"/missing.html", workspaceFileMissing, ""},+ {"/leak.txt", workspaceFileOutside, ""},+ {"/../secret.txt", workspaceFileOutside, ""},+ {"/big.txt", workspaceFileTooLarge, ""},+}++func TestServeWorkspaceFileStaysInTheWorkspace(t *testing.T) {+ root := workspaceFileTree(t)+ for _, tc := range workspaceFileCases {+ body, code := serveWorkspaceFile(root, root+tc.path, 32)+ if code != tc.code || string(body) != tc.body {+ t.Fatalf("%s: code %d body %q", tc.path, code, body)+ }+ }+}++// The sidecar script must agree with the host reader on every case.+func TestWorkspaceFileScriptMatchesHostReader(t *testing.T) {+ if _, err := exec.LookPath("node"); err != nil {+ t.Skip("node not installed")+ }+ root := workspaceFileTree(t)+ for _, tc := range workspaceFileCases {+ out, err := exec.Command("node", "-e", workspaceFileScript, root, root+tc.path, "32").Output()+ code := 0+ var exit *exec.ExitError+ if errors.As(err, &exit) {+ code = exit.ExitCode()+ } else if err != nil {+ t.Fatal(err)+ }+ if code != tc.code || (code == 0 && string(out) != tc.body) {+ t.Fatalf("%s: code %d body %q", tc.path, code, out)+ }+ }+}++func TestHandleWorkspaceFileResponses(t *testing.T) {+ root := workspaceFileTree(t)+ lookup, execFile := devServerLookup, workspaceFileExec+ t.Cleanup(func() { devServerLookup, workspaceFileExec = lookup, execFile })+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {+ return browserEndpoint{Kind: "docker", Container: "sidecar"}, true+ }+ var seen []string+ workspaceFileExec = func(_ context.Context, container, file string) ([]byte, int, error) {+ seen = append(seen, container+" "+file)+ body, code := serveWorkspaceFile(root, root+strings.TrimPrefix(file, dockerWorkspaceRoot), 32)+ return body, code, nil+ }+ get := func(file string) domain.BrowserState {+ t.Helper()+ state, err := HandleBrowser(context.Background(), "file-run", domain.BrowserRequest{Operation: "file", File: file})+ if err != nil {+ t.Fatal(err)+ }+ return state+ }+ for _, tc := range []struct {+ file, contentType string+ status int+ }{+ {"/workspace/demo.html", "text/html; charset=utf-8", 200},+ {"/workspace/site/", "text/html; charset=utf-8", 200},+ {"/workspace/site/app.js", "text/javascript; charset=utf-8", 200},+ {"/workspace/site", "", 301},+ {"/workspace/missing.html", "text/plain; charset=utf-8", 404},+ {"/workspace/leak.txt", "text/plain; charset=utf-8", 403},+ } {+ state := get(tc.file)+ if state.HTTP == nil || state.HTTP.Status != tc.status {+ t.Fatalf("%s: %+v", tc.file, state)+ }+ if got := strings.Join(state.HTTP.Header["Content-Type"], ""); got != tc.contentType {+ t.Fatalf("%s: content type %q", tc.file, got)+ }+ }+ if state := get("/workspace/big.txt"); state.HTTP != nil || !strings.Contains(state.Reason, strconv.Itoa(DevServerBodyLimit>>20)+" MB") {+ t.Fatalf("too large: %+v", state)+ }+ if seen[0] != "sidecar /workspace/demo.html" {+ t.Fatalf("exec: %v", seen)+ }+ if _, err := HandleBrowser(context.Background(), "file-run", domain.BrowserRequest{Operation: "file", File: "relative.html"}); err == nil {+ t.Fatal("relative path accepted")+ }++ // A host browser reads its own workspace directly.+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) {+ return browserEndpoint{Kind: "local", Workspace: root}, true+ }+ if state := get(root + "/site/"); state.HTTP == nil || string(state.HTTP.Body) != "<p>site</p>" {+ t.Fatalf("host: %+v", state)+ }+ devServerLookup = func(context.Context, string) (browserEndpoint, bool) { return browserEndpoint{}, false }+ if state := get("/workspace/demo.html"); state.Available || state.HTTP != nil {+ t.Fatalf("no browser: %+v", state)+ }+}diff --git a/internal/runtime/browser_inspect.go b/internal/runtime/browser_inspect.gonew file mode 100644index 00000000..8c888355--- /dev/null+++ b/internal/runtime/browser_inspect.go@@ -0,0 +1,161 @@+package runtime++import (+ "context"+ "errors"+ "fmt"++ "agentflow/internal/domain"+)++// Element inspection for the panel's pick mode on the Agent browser+// screencast: the page reports the element under a point, its box for the+// outline and, on a pick, the description the composer sends to the Agent.+// The description follows chatbot/src/element-context.js, which does the same+// for native previews in the user's own browser; keep the two in step.++const inspectScript = `(function (fx, fy, detail) {+ var doc = document, x = fx * innerWidth, y = fy * innerHeight, ox = 0, oy = 0, el = null;+ for (var depth = 0; depth < 8; depth++) {+ var hit = doc.elementFromPoint(x, y);+ while (hit && hit.shadowRoot) {+ var inner = hit.shadowRoot.elementFromPoint(x, y);+ if (!inner || inner === hit) break;+ hit = inner;+ }+ if (!hit) break;+ el = hit;+ if (hit.tagName !== "IFRAME" && hit.tagName !== "FRAME") break;+ var child = null;+ try { child = hit.contentDocument; } catch (e) {}+ if (!child || !child.documentElement) break;+ var box = hit.getBoundingClientRect();+ var dx = box.left + hit.clientLeft, dy = box.top + hit.clientTop;+ x -= dx; y -= dy; ox += dx; oy += dy; doc = child;+ }+ if (!el || el === document.documentElement) return null;+ var clip = function (value, limit) {+ var text = String(value || "").replace(/\s+/g, " ").trim();+ return text.length > limit ? text.slice(0, limit) + "…" : text;+ };+ var esc = function (v) { return window.CSS && CSS.escape ? CSS.escape(v) : String(v).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); };+ var classes = function (node) {+ return Array.prototype.filter.call(node.classList || [], function (c) {+ return !/^(css|sc|jsx|svelte|emotion)-|^[a-z]{1,3}[-_]?(?=[0-9a-z]*[0-9])[0-9a-z]{5,}$/i.test(c) && c.length <= 40;+ }).slice(0, 2);+ };+ var label = function (node) {+ var tag = node.tagName.toLowerCase(), id = node.id ? "#" + node.id : "";+ var cls = !id && classes(node)[0] ? "." + classes(node)[0] : "";+ var text = clip(node.innerText != null ? node.innerText : node.textContent, 16);+ return tag + id + cls + (text ? " 「" + text + "」" : "");+ };+ var rect = el.getBoundingClientRect();+ var out = {+ label: label(el),+ rect: { x: Math.round(rect.left + ox), y: Math.round(rect.top + oy), width: Math.round(rect.width), height: Math.round(rect.height) },+ viewport: { width: innerWidth, height: innerHeight }+ };+ if (!detail) return out;+ var owner = el.ownerDocument;+ var unique = function (selector) {+ try { var found = owner.querySelectorAll(selector); return found.length === 1 && found[0] === el; } catch (e) { return false; }+ };+ var step = function (node) {+ var tag = node.tagName.toLowerCase();+ var testId = node.getAttribute("data-testid") || node.getAttribute("data-test-id");+ if (testId) return tag + '[data-testid="' + testId.replace(/"/g, '\\"') + '"]';+ return tag + classes(node).map(function (c) { return "." + esc(c); }).join("");+ };+ var nth = function (node, base) {+ var parent = node.parentElement;+ if (!parent) return base;+ var same = Array.prototype.filter.call(parent.children, function (c) { return c.tagName === node.tagName; });+ return same.length > 1 ? base + ":nth-of-type(" + (same.indexOf(node) + 1) + ")" : base;+ };+ var selector = function () {+ if (el.id && unique("#" + esc(el.id))) return "#" + esc(el.id);+ var parts = [], node = el;+ while (node && node.nodeType === 1 && node !== owner.documentElement) {+ if (node !== el && node.id && owner.querySelectorAll("#" + esc(node.id)).length === 1) { parts.unshift("#" + esc(node.id)); break; }+ parts.unshift(step(node));+ if (unique(parts.join(" > "))) return parts.join(" > ");+ parts[0] = nth(node, parts[0]);+ if (unique(parts.join(" > "))) return parts.join(" > ");+ node = node.parentElement;+ }+ return parts.join(" > ");+ };+ var source = function (file, line, column) {+ return file ? file + (line ? ":" + line : "") + (line && column ? ":" + column : "") : "";+ };+ var component = function () {+ for (var node = el, i = 0; node && node.nodeType === 1 && i < 25; node = node.parentElement, i++) {+ var key = Object.keys(node).find(function (k) { return k.indexOf("__reactFiber$") === 0 || k.indexOf("__reactInternalInstance$") === 0; });+ if (key) {+ var fiber = node[key], src = fiber && fiber._debugSource;+ for (var j = 0; fiber && j < 40; j++, fiber = fiber.return) {+ src = src || fiber._debugSource;+ var t = fiber.type;+ if (typeof t === "function" || (t && typeof t === "object" && (t.render || t.type))) {+ var name = t.displayName || t.name || (t.render && (t.render.displayName || t.render.name)) || (t.type && t.type.name);+ if (name) return { framework: "React", component: name, source: src ? source(src.fileName, src.lineNumber, src.columnNumber) : "" };+ }+ }+ }+ var vue = node.__vueParentComponent;+ if (vue && vue.type) {+ var file = vue.type.__file || "";+ return { framework: "Vue", component: vue.type.name || vue.type.__name || file.split("/").pop().replace(/\.vue$/, ""), source: file };+ }+ if (node.__vue__ && node.__vue__.$options) {+ var o = node.__vue__.$options;+ return { framework: "Vue", component: o.name || o._componentTag || "", source: o.__file || "" };+ }+ var loc = node.__svelte_meta && node.__svelte_meta.loc;+ if (loc && loc.file) {+ return { framework: "Svelte", component: loc.file.split("/").pop().replace(/\.svelte$/, ""), source: source(loc.file, loc.line != null ? loc.line + 1 : 0, loc.column != null ? loc.column + 1 : 0) };+ }+ }+ return null;+ };+ var computed = owner.defaultView.getComputedStyle(el), styles = {};+ var layout = /flex|grid/.test(computed.getPropertyValue("display"));+ var defaults = { position: "static", opacity: "1", "font-weight": "400" };+ ["display", "position", "width", "height", "margin", "padding", "gap", "flex-direction", "justify-content", "align-items",+ "color", "background-color", "font-size", "font-weight", "line-height", "border", "border-radius", "box-shadow", "opacity"].forEach(function (k) {+ var v = computed.getPropertyValue(k);+ if (!layout && ["gap", "flex-direction", "justify-content", "align-items"].indexOf(k) >= 0) return;+ if (!v || /^(none|normal|auto|0px|0px none.*|rgba\(0, 0, 0, 0\))$/.test(v) || defaults[k] === v) return;+ styles[k] = v;+ });+ // Snapshot refs (data-agentflow-ref) mean nothing outside their snapshot.+ var html = (el.outerHTML || "").replace(/ data-agentflow-ref="[^"]*"/g, ""), found = component();+ out.element = {+ url: location.href,+ title: clip(document.title, 120),+ selector: selector(),+ tag: el.tagName.toLowerCase(),+ text: clip(el.innerText != null ? el.innerText : el.textContent, 200),+ html: html.length > 3000 ? html.slice(0, 3000) + "…" : html,+ rect: out.rect,+ viewport: out.viewport,+ styles: styles+ };+ if (found && found.component) { out.element.component = found.component; out.element.framework = found.framework; }+ if (found && found.source) out.element.source = found.source;+ return out;+})`++// inspect reports the element at point (a fraction of the viewport).+func (v *browserView) inspect(ctx context.Context, point *domain.BrowserPoint, detail bool) (*domain.BrowserInspection, error) {+ if point == nil {+ return nil, errors.New("缺少检查位置")+ }+ var out *domain.BrowserInspection+ expression := fmt.Sprintf("%s(%g, %g, %t)", inspectScript, clampUnit(point.X), clampUnit(point.Y), detail)+ if err := v.evaluate(ctx, expression, &out); err != nil {+ return nil, err+ }+ return out, nil+}diff --git a/internal/runtime/browser_inspect_test.go b/internal/runtime/browser_inspect_test.gonew file mode 100644index 00000000..98c09f4f--- /dev/null+++ b/internal/runtime/browser_inspect_test.go@@ -0,0 +1,50 @@+package runtime++import (+ "context"+ "encoding/json"+ "strings"+ "testing"++ "agentflow/internal/domain"+)++func TestInspectReportsTheElementAtAPoint(t *testing.T) {+ transport := newFakeCDPTransport()+ transport.results = map[string]string{+ "Runtime.evaluate": `{"result":{"value":{"label":"button.btn 「保存」","rect":{"x":10,"y":20,"width":55,"height":35},"viewport":{"width":1280,"height":800},+ "element":{"url":"https://example.com/","selector":"#save","tag":"button","source":"src/Save.tsx:3"}}}}`,+ }+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}+ view.client = newCDPClient(transport, view.onEvent)+ defer view.client.close()++ if got, err := view.inspect(context.Background(), nil, true); err == nil || got != nil {+ t.Fatal("missing point accepted")+ }+ got, err := view.inspect(context.Background(), &domain.BrowserPoint{X: 0.25, Y: 1.7}, true)+ if err != nil || got == nil {+ t.Fatalf("got=%+v err=%v", got, err)+ }+ if got.Label != "button.btn 「保存」" || got.Rect.Width != 55 || got.Viewport.Width != 1280 || got.Element == nil || got.Element.Source != "src/Save.tsx:3" {+ t.Fatalf("inspection %+v %+v", got, got.Element)+ }+ transport.mu.Lock()+ last := transport.sent[len(transport.sent)-1]+ transport.mu.Unlock()+ var params struct {+ Expression string `json:"expression"`+ }+ _ = json.Unmarshal(last.Params, ¶ms)+ // Points are clamped to the viewport like input events.+ if !strings.HasSuffix(params.Expression, "})(0.25, 1, true)") {+ t.Fatalf("expression ends %q", params.Expression[len(params.Expression)-40:])+ }++ transport.mu.Lock()+ transport.results["Runtime.evaluate"] = `{"result":{"value":null}}`+ transport.mu.Unlock()+ if got, err := view.inspect(context.Background(), &domain.BrowserPoint{X: 0.5, Y: 0.5}, false); err != nil || got != nil {+ t.Fatalf("empty point: %+v %v", got, err)+ }+}diff --git a/internal/runtime/browser_motion_test.go b/internal/runtime/browser_motion_test.gonew file mode 100644index 00000000..3b0f13ed--- /dev/null+++ b/internal/runtime/browser_motion_test.go@@ -0,0 +1,95 @@+package runtime++import (+ "bytes"+ "context"+ "encoding/base64"+ "fmt"+ "image/jpeg"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "agentflow/internal/domain"+)++// Chromium re-sends the current page when the screencast restarts, so a+// static page still ends on a sharp frame after a scroll.+func TestBrowserScrollEndsOnASharpFrame(t *testing.T) {+ if testing.Short() {+ t.Skip("short mode")+ }+ if _, err := FindChromium(); err != nil {+ t.Skip("chromium not installed")+ }+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/html; charset=utf-8")+ fmt.Fprint(w, `<title>静态</title><h1>不会动的页面</h1>`)+ }))+ defer page.Close()+ workspace := t.TempDir()+ runID := "browser-motion-" + fmt.Sprint(time.Now().UnixNano())+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()+ if _, err := ensureHostBrowser(ctx, workspace, runID); err != nil {+ t.Fatal(err)+ }+ defer stopHostBrowsers(workspace)+ if _, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "resize", FrameWidth: 800}); err != nil {+ t.Fatal(err)+ }+ state, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "navigate", URL: page.URL})+ if err != nil {+ t.Fatal(err)+ }+ // nextWidth waits for a frame after seq and returns its pixel width.+ nextWidth := func(after int64) (int64, int) {+ t.Helper()+ for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); {+ st, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "frame", After: after, WaitMs: 2000})+ if err != nil {+ t.Fatal(err)+ }+ if st.Seq > after && st.Frame != "" {+ raw, _ := base64.StdEncoding.DecodeString(st.Frame)+ config, err := jpeg.DecodeConfig(bytes.NewReader(raw))+ if err != nil {+ t.Fatal(err)+ }+ return st.Seq, config.Width+ }+ }+ t.Fatal("no frame")+ return 0, 0+ }+ seq, width := nextWidth(state.Seq)+ // Let the page finish painting at full width.+ for width < 700 {+ seq, width = nextWidth(seq)+ }+ if _, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "input", Events: []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}}}); err != nil {+ t.Fatal(err)+ }+ // A full-width frame already in flight can arrive before the screencast+ // restarts at the lighter size. Keep reading until a light frame shows up.+ sawLight := false+ for deadline := time.Now().Add(8 * time.Second); time.Now().Before(deadline); {+ seq, width = nextWidth(seq)+ if width <= 400 {+ sawLight = true+ break+ }+ }+ if !sawLight {+ t.Fatalf("no light frame while scrolling (last %dpx)", width)+ }+ // More light frames may follow while the scroll animates; once input+ // settles a full-width frame arrives although nothing changed.+ for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {+ if seq, width = nextWidth(seq); width >= 700 {+ return+ }+ }+ t.Fatalf("no sharp frame after the scroll settled (last %dpx)", width)+}diff --git a/internal/runtime/browser_test.go b/internal/runtime/browser_test.goindex e74a8c38..71b2676a 100644--- a/internal/runtime/browser_test.go+++ b/internal/runtime/browser_test.go@@ -10,6 +10,7 @@ import ("strings""sync""testing"+ "time""agentflow/internal/domain")@@ -84,6 +85,8 @@ type fakeCDPTransport struct {replies chan []byteclosed chan struct{}once sync.Once+ // results answers methods with a canned result instead of {}.+ results map[string]string}func newFakeCDPTransport() *fakeCDPTransport {@@ -107,7 +110,11 @@ func (f *fakeCDPTransport) write(body []byte) error {f.mu.Lock()f.sent = append(f.sent, msg)f.mu.Unlock()- reply, _ := json.Marshal(cdpMessage{ID: msg.ID, Result: json.RawMessage(`{}`)})+ result := `{}`+ if r := f.results[msg.Method]; r != "" {+ result = r+ }+ reply, _ := json.Marshal(cdpMessage{ID: msg.ID, Result: json.RawMessage(result)})f.replies <- replyreturn nil}@@ -134,20 +141,28 @@ func TestBrowserViewInputMapsToCDP(t *testing.T) {}transport.mu.Lock()defer transport.mu.Unlock()- if len(transport.sent) != 5 {- t.Fatalf("sent %d messages: %+v", len(transport.sent), transport.sent)+ // The wheel also switches the screencast to light frames; only input+ // events are checked here.+ var sent []cdpMessage+ for _, msg := range transport.sent {+ if strings.HasPrefix(msg.Method, "Input.") {+ sent = append(sent, msg)+ }+ }+ if len(sent) != 5 {+ t.Fatalf("sent %d input messages: %+v", len(sent), transport.sent)}params := func(i int) map[string]any {var out map[string]any- _ = json.Unmarshal(transport.sent[i].Params, &out)+ _ = json.Unmarshal(sent[i].Params, &out)return out}- for _, msg := range transport.sent {+ for _, msg := range sent {if msg.SessionID != "page-1" {t.Fatalf("input must target the attached page session: %+v", msg)}}- if p := params(0); transport.sent[0].Method != "Input.dispatchMouseEvent" || p["type"] != "mousePressed" || p["x"] != 500.0 || p["y"] != 125.0 {+ if p := params(0); sent[0].Method != "Input.dispatchMouseEvent" || p["type"] != "mousePressed" || p["x"] != 500.0 || p["y"] != 125.0 {t.Fatalf("mouse: %s %v", transport.sent[0].Method, p)}if p := params(1); p["type"] != "mouseWheel" || p["x"] != 1000.0 || p["y"] != 0.0 || p["deltaY"] != 120.0 {@@ -159,7 +174,139 @@ func TestBrowserViewInputMapsToCDP(t *testing.T) {if p := params(3); p["type"] != "rawKeyDown" || p["text"] != nil {t.Fatalf("control key: %v", p)}- if p := params(4); transport.sent[4].Method != "Input.insertText" || p["text"] != "你好" {- t.Fatalf("text: %s %v", transport.sent[4].Method, p)+ if p := params(4); sent[4].Method != "Input.insertText" || p["text"] != "你好" {+ t.Fatalf("text: %s %v", sent[4].Method, p)+ }+}++func TestBrowserViewReportsScrollAndContentSize(t *testing.T) {+ transport := newFakeCDPTransport()+ transport.results = map[string]string{"Page.getLayoutMetrics": `{"cssContentSize":{"width":1000,"height":3200}}`}+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}+ view.client = newCDPClient(transport, view.onEvent)+ defer view.client.close()+ frame, _ := json.Marshal(map[string]any{"data": "AAAA", "sessionId": 7, "metadata": map[string]any{"deviceWidth": 1000, "deviceHeight": 800, "scrollOffsetX": 0, "scrollOffsetY": 640}})+ view.onEvent(cdpMessage{Method: "Page.screencastFrame", SessionID: "page-1", Params: frame})+ deadline := time.Now().Add(2 * time.Second)+ for {+ state := view.state(0)+ if state.ContentHeight == 3200 {+ if state.ScrollY != 640 || state.Height != 800 || state.ContentWidth != 1000 {+ t.Fatalf("state %+v", state)+ }+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("content size never arrived: %+v", state)+ }+ time.Sleep(10 * time.Millisecond)+ }+ // A second frame within the refresh interval does not read metrics again.+ view.onEvent(cdpMessage{Method: "Page.screencastFrame", SessionID: "page-1", Params: frame})+ time.Sleep(50 * time.Millisecond)+ transport.mu.Lock()+ defer transport.mu.Unlock()+ reads := 0+ for _, msg := range transport.sent {+ if msg.Method == "Page.getLayoutMetrics" {+ reads+++ }+ }+ if reads != 1 {+ t.Fatalf("layout metric reads: %d", reads)+ }+}++func TestBrowserDeviceScale(t *testing.T) {+ for raw, want := range map[string]string{"": "2", "1.5": "1.5", "3": "3", "0.5": "2", "x": "2", "4": "2"} {+ t.Setenv("AGENTFLOW_BROWSER_SCALE", raw)+ if got := BrowserDeviceScale(); got != want {+ t.Fatalf("%q: %s", raw, got)+ }+ }+}++func TestAgentScreenshotsStayInCSSPixels(t *testing.T) {+ transport := newFakeCDPTransport()+ transport.results = map[string]string{+ "Page.getLayoutMetrics": `{"cssVisualViewport":{"pageX":0,"pageY":300,"clientWidth":1280,"clientHeight":800}}`,+ "Runtime.evaluate": `{"result":{"value":2}}`,+ "Page.captureScreenshot": `{"data":"AAAA"}`,+ }+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1"}+ view.client = newCDPClient(transport, view.onEvent)+ defer view.client.close()+ if data, err := view.screenshot(context.Background()); err != nil || data != "AAAA" {+ t.Fatalf("data=%q err=%v", data, err)+ }+ transport.mu.Lock()+ defer transport.mu.Unlock()+ last := transport.sent[len(transport.sent)-1]+ var params struct {+ Clip map[string]float64 `json:"clip"`+ }+ _ = json.Unmarshal(last.Params, ¶ms)+ if last.Method != "Page.captureScreenshot" || params.Clip["scale"] != 0.5 || params.Clip["y"] != 300 || params.Clip["width"] != 1280 {+ t.Fatalf("%s %s", last.Method, last.Params)+ }+}++func TestBrowserViewCapturesLighterFramesWhileScrolling(t *testing.T) {+ settle := browserMotionSettle+ browserMotionSettle = 60 * time.Millisecond+ defer func() { browserMotionSettle = settle }()+ transport := newFakeCDPTransport()+ view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1", width: 1000, height: 500, frameWidth: 1600}+ view.client = newCDPClient(transport, view.onEvent)+ defer view.client.close()+ starts := func() []map[string]any {+ transport.mu.Lock()+ defer transport.mu.Unlock()+ var out []map[string]any+ for _, msg := range transport.sent {+ if msg.Method == "Page.startScreencast" {+ var p map[string]any+ _ = json.Unmarshal(msg.Params, &p)+ out = append(out, p)+ }+ }+ return out+ }+ wheel := []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}}+ ctx := context.Background()++ // Hovering and clicking keep sharp frames.+ if err := view.input(ctx, []domain.BrowserInputEvent{{Type: "mouse", Action: "move", X: 0.5, Y: 0.5}, {Type: "mouse", Action: "down", X: 0.5, Y: 0.5, Buttons: 1}}); err != nil {+ t.Fatal(err)+ }+ if got := starts(); len(got) != 0 {+ t.Fatalf("hover restarted the screencast: %v", got)+ }+ // A scroll switches to light frames once, however many follow.+ for range 3 {+ if err := view.input(ctx, wheel); err != nil {+ t.Fatal(err)+ }+ time.Sleep(20 * time.Millisecond)+ }+ got := starts()+ if len(got) != 1 || got[0]["maxWidth"] != 800.0 || got[0]["quality"] != 60.0 {+ t.Fatalf("scroll frames: %v", got)+ }+ // Quiet input returns to sharp frames at the panel width.+ deadline := time.Now().Add(2 * time.Second)+ for len(starts()) < 2 && time.Now().Before(deadline) {+ time.Sleep(10 * time.Millisecond)+ }+ got = starts()+ if len(got) != 2 || got[1]["maxWidth"] != 1600.0 || got[1]["quality"] != 80.0 {+ t.Fatalf("settled frames: %v", got)+ }+ // Dragging counts as motion too.+ if err := view.input(ctx, []domain.BrowserInputEvent{{Type: "mouse", Action: "move", X: 0.4, Y: 0.5, Buttons: 1}}); err != nil {+ t.Fatal(err)+ }+ if got := starts(); len(got) != 3 || got[2]["quality"] != 60.0 {+ t.Fatalf("drag frames: %v", got)}}diff --git a/internal/runtime/browser_view.go b/internal/runtime/browser_view.goindex 2b11bc9d..82896957 100644--- a/internal/runtime/browser_view.go+++ b/internal/runtime/browser_view.go@@ -5,6 +5,7 @@ import ("encoding/json""errors""fmt"+ "math""net/url""strings""sync"@@ -28,19 +29,29 @@ type browserView struct {endpoint browserEndpointclient *cdpClient- mu sync.Mutex- targets map[string]*browserTarget- active string- session string- frame string- seq int64- width float64- height float64- lastUsed time.Time- changed chan struct{}- follow bool+ mu sync.Mutex+ targets map[string]*browserTarget+ active string+ session string+ frame string+ seq int64+ width float64+ height float64+ // Document scroll position from frame metadata and content size from+ // layout metrics, refreshed at most every browserContentRefresh.+ scrollX, scrollY float64+ contentWidth, contentHeight float64+ contentAt time.Time+ contentBusy bool+ lastUsed time.Time+ changed chan struct{}+ follow bool// frameWidth is the panel's device pixel width; 0 keeps the default cap.frameWidth int+ // motion is set while the panel scrolls or drags: frames are captured+ // lighter until the input settles (see markMotion).+ motion bool+ motionGen int64// History flags are cached per active page URL: re-reading them on every// frame would add a CDP round trip to each one.historyKey string@@ -57,6 +68,13 @@ var browserViewReads = map[string]bool{"": true, "status": true, "frame": true}// HandleBrowser serves one task panel browser request for a run whose browser// lives on this host (docker sidecar or host Chromium).func HandleBrowser(ctx context.Context, runID string, req domain.BrowserRequest) (domain.BrowserState, error) {+ if req.Operation == "http" {+ // Dev server preview needs the sandbox network, not the CDP view.+ return handleDevServerHTTP(ctx, runID, req.HTTP)+ }+ if req.Operation == "file" {+ return handleWorkspaceFile(ctx, runID, req.File)+ }view, err := openBrowserView(ctx, runID)if err != nil {return domain.BrowserState{Available: false, Reason: err.Error()}, nil@@ -115,6 +133,12 @@ func HandleBrowser(ctx context.Context, runID string, req domain.BrowserRequest)if err := view.client.call(ctx, "", "Target.closeTarget", map[string]any{"targetId": target}, nil); err != nil {return view.state(req.After), err}+ case "inspect":+ inspection, err := view.inspect(ctx, req.Point, req.Detail)+ // Inspection never needs a frame: the panel already shows one.+ state := view.state(math.MaxInt64)+ state.Inspect = inspection+ return state, errcase "snapshot", "screenshot", "click", "type", "press":snap, frame, err := view.agentOperation(ctx, req)state := view.state(req.After)@@ -307,8 +331,11 @@ func (v *browserView) onEvent(msg cdpMessage) {Data string `json:"data"`SessionID int64 `json:"sessionId"`Metadata struct {- DeviceWidth float64 `json:"deviceWidth"`- DeviceHeight float64 `json:"deviceHeight"`+ DeviceWidth float64 `json:"deviceWidth"`+ DeviceHeight float64 `json:"deviceHeight"`+ ScrollOffsetX float64 `json:"scrollOffsetX"`+ ScrollOffsetY float64 `json:"scrollOffsetY"`+ PageScaleFactor float64 `json:"pageScaleFactor"`} `json:"metadata"`}if json.Unmarshal(msg.Params, &ev) != nil {@@ -316,12 +343,20 @@ func (v *browserView) onEvent(msg cdpMessage) {}v.mu.Lock()current := msg.SessionID == v.session+ refresh := falseif current {v.frame, v.width, v.height = ev.Data, ev.Metadata.DeviceWidth, ev.Metadata.DeviceHeight+ v.scrollX, v.scrollY = ev.Metadata.ScrollOffsetX, ev.Metadata.ScrollOffsetYv.seq++v.notify()+ if !v.contentBusy && time.Since(v.contentAt) > browserContentRefresh {+ v.contentBusy, refresh = true, true+ }}v.mu.Unlock()+ if refresh {+ go v.refreshContentSize(msg.SessionID)+ }go func() {ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)defer cancel()@@ -330,6 +365,29 @@ func (v *browserView) onEvent(msg cdpMessage) {}}+// browserContentRefresh bounds layout metric reads: content size changes+// rarely, and one read per frame would add a CDP round trip to each.+const browserContentRefresh = time.Second++func (v *browserView) refreshContentSize(session string) {+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ var metrics struct {+ CSSContentSize struct {+ Width float64 `json:"width"`+ Height float64 `json:"height"`+ } `json:"cssContentSize"`+ }+ err := v.client.call(ctx, session, "Page.getLayoutMetrics", nil, &metrics)+ v.mu.Lock()+ defer v.mu.Unlock()+ v.contentBusy = false+ v.contentAt = time.Now()+ if err == nil && session == v.session {+ v.contentWidth, v.contentHeight = metrics.CSSContentSize.Width, metrics.CSSContentSize.Height+ }+}+func (v *browserView) latestTarget() string {v.mu.Lock()defer v.mu.Unlock()@@ -401,13 +459,64 @@ func (v *browserView) attach(ctx context.Context, targetID string) error {func (v *browserView) screencastParams() map[string]any {v.mu.Lock()- width := v.frameWidth+ width, motion := v.frameWidth, v.motionv.mu.Unlock()- if width <= 0 || width > 1600 {- width = 1600+ if width <= 0 || width > 2560 {+ width = 2560+ }+ // Text needs a cleaner JPEG than photos; 80 keeps edges sharp at about+ // twice the bytes of 60.+ quality := 80+ if motion {+ // Moving content is not read: half the pixels at quality 60 cut a+ // text page frame to about a fifth, so the sandbox renders and the+ // link carries several times as many while scrolling.+ width, quality = width/2, browserMotionQuality}width = max(width, 320)- return map[string]any{"format": "jpeg", "quality": 60, "maxWidth": width, "maxHeight": min(2*width, 2000), "everyNthFrame": 1}+ return map[string]any{"format": "jpeg", "quality": quality, "maxWidth": width, "maxHeight": min(2*width, 4000), "everyNthFrame": 1}+}++const browserMotionQuality = 60++// browserMotionSettle is how long after the last scroll or drag the+// screencast returns to sharp frames.+var browserMotionSettle = 400 * time.Millisecond++// markMotion switches the screencast to light frames for scrolling and+// dragging, and back once input has been quiet for browserMotionSettle.+// Restarting the screencast makes Chromium send the current page again, so+// the view ends on a sharp frame even when nothing moves afterwards.+func (v *browserView) markMotion(ctx context.Context) {+ v.mu.Lock()+ v.motionGen+++ gen, start, session := v.motionGen, !v.motion, v.session+ v.motion = true+ v.mu.Unlock()+ if start && session != "" {+ v.restartScreencast(ctx, session)+ }+ time.AfterFunc(browserMotionSettle, func() {+ v.mu.Lock()+ if gen != v.motionGen || !v.motion {+ v.mu.Unlock()+ return+ }+ v.motion = false+ session := v.session+ v.mu.Unlock()+ if session == "" {+ return+ }+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ v.restartScreencast(ctx, session)+ })+}++func (v *browserView) restartScreencast(ctx context.Context, session string) {+ _ = v.client.call(ctx, session, "Page.stopScreencast", nil, nil)+ _ = v.client.call(ctx, session, "Page.startScreencast", v.screencastParams(), nil)}// setFrameWidth restarts the screencast when the panel's pixel width changes.@@ -469,7 +578,8 @@ func (v *browserView) waitFrame(ctx context.Context, after int64, waitMs int) {func (v *browserView) state(after int64) domain.BrowserState {v.mu.Lock()defer v.mu.Unlock()- state := domain.BrowserState{Available: !v.client.closed(), Kind: v.endpoint.Kind, Seq: v.seq, Width: v.width, Height: v.height}+ state := domain.BrowserState{Available: !v.client.closed(), Kind: v.endpoint.Kind, Seq: v.seq, Width: v.width, Height: v.height,+ ScrollX: v.scrollX, ScrollY: v.scrollY, ContentWidth: v.contentWidth, ContentHeight: v.contentHeight}if !state.Available {state.Reason = "浏览器连接已断开"}@@ -620,6 +730,12 @@ func (v *browserView) input(ctx context.Context, events []domain.BrowserInputEveif width <= 0 || height <= 0 {width, height = 1280, 800}+ for _, ev := range events {+ if ev.Type == "wheel" || ev.Type == "mouse" && ev.Action == "move" && ev.Buttons != 0 {+ v.markMotion(ctx)+ break+ }+ }for _, ev := range events {x, y := clampUnit(ev.X)*width, clampUnit(ev.Y)*heightvar err errordiff --git a/internal/runtime/core_cases_test.go b/internal/runtime/core_cases_test.goindex 84d21929..94df90ce 100644--- a/internal/runtime/core_cases_test.go+++ b/internal/runtime/core_cases_test.go@@ -125,7 +125,7 @@ func TestBrowserViewOperationsWithFakeCDP(t *testing.T) {if err := view.pageCall(ctx, "Page.reload", nil); err != nil {t.Fatal(err)}- if got := view.screencastParams(); got["maxWidth"] != 1600 {+ if got := view.screencastParams(); got["maxWidth"] != 2560 {t.Fatalf("params=%v", got)}view.frameWidth = 100