Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11247 render copilot code-seam repair as quiet progress, not repeated chat (#6746)
Andrew Neilson211de42ea35f
diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.pyindex a241a40ed..81a8994a0 100644--- a/skyvern/forge/sdk/copilot/narration.py+++ b/skyvern/forge/sdk/copilot/narration.py@@ -65,6 +65,12 @@ MAX_DESIGN_ACTIVITY_ENTRIES = 50# Mirror of the FE ACTIVITY_TOOL_DENYLIST in narrativeState.ts.ACTIVITY_TOOL_DENYLIST = frozenset({"list_credentials", "get_run_results", "get_browser_screenshot"})+# Shared classification for a code-authoring reject the streaming adapter renders+# as quiet de-duplicated progress. Tagged on the reject (workflow_update) and+# consumed by the SSE layer (streaming_adapter) — one source of truth for both.+CODE_REPAIR_PROGRESS_SURFACE_KIND = "code_repair_progress"+CODE_REPAIR_PROGRESS_TEXT = "Refining the workflow's code"+_TOOL_ACTIVITY_DISPLAY_LABELS = {# Mirror of the FE ACTIVITY_TOOL_DISPLAY_LABELS in narrativeState.ts."update_workflow": "Updating workflow",@@ -182,6 +188,8 @@ class NarratorState:block_activity: dict[str, list[NarrativeActivityEntry]] = field(default_factory=dict)design_activity: list[NarrativeActivityEntry] = field(default_factory=list)running_block_label: str | None = None+ # Per-turn (NarratorState lives one turn); collapses repeated code-repair progress to one entry.+ emitted_progress_texts: set[str] = field(default_factory=set)def record_activity(self, entry: NarrativeActivityEntry | None) -> None:if entry is None:diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex 946857adf..4c80d254a 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -14,8 +14,10 @@ import structlogfrom skyvern.forge.request_logging import redact_sensitive_fieldsfrom skyvern.forge.sdk.copilot.context import InFlightStreamToolCallfrom skyvern.forge.sdk.copilot.narration import (+ CODE_REPAIR_PROGRESS_SURFACE_KIND,NarratorState,TransitionKind,+ build_narration_activity,build_tool_call_activity,build_tool_result_activity,cancel_in_flight,@@ -30,6 +32,7 @@ from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user,from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotDesignEndUpdate,WorkflowCopilotDesignStartUpdate,+ WorkflowCopilotNarrationUpdate,WorkflowCopilotStreamMessageType,WorkflowCopilotToolCallUpdate,WorkflowCopilotToolResultUpdate,@@ -60,6 +63,17 @@ _OBSERVATION_TOOLS = {}+def _code_repair_progress_text(parsed: dict[str, Any]) -> str | None:+ """Return the classified code-repair progress text carried on a reject's ``data``, else None."""+ data = parsed.get("data")+ if not isinstance(data, dict):+ return None+ if data.get("surface_kind") != CODE_REPAIR_PROGRESS_SURFACE_KIND:+ return None+ progress_text = data.get("progress_text")+ return progress_text if isinstance(progress_text, str) and progress_text else None++async def stream_to_sse(result: RunResultStreaming,stream: EventSourceStream,@@ -188,48 +202,61 @@ async def stream_to_sse(output = getattr(event.item, "output", None)parsed = parse_tool_output(output)- blocker_signals = _tool_blocker_signal_candidates(ctx)- summary = format_tool_result_for_user(tool_name, parsed, blocker_signal=blocker_signals)- success = parsed.get("ok", True)- detail = summarize_tool_result_detail(parsed, tool_name=tool_name, blocker_signal=blocker_signals)- narrator_state.record_activity(- build_tool_result_activity(tool_name, summary, success, iteration, call_id)- )+ progress_text = _code_repair_progress_text(parsed)+ if progress_text is not None:+ # Presentation-only: render the reject as quiet de-duplicated progress;+ # enforcement and the turn-halt below still run.+ await _emit_code_repair_progress(+ narrator_state=narrator_state,+ stream=stream,+ progress_text=progress_text,+ iteration=iteration,+ client_gone=client_gone,+ )+ _update_enforcement_from_tool(ctx, tool_name, parsed)+ else:+ blocker_signals = _tool_blocker_signal_candidates(ctx)+ summary = format_tool_result_for_user(tool_name, parsed, blocker_signal=blocker_signals)+ success = parsed.get("ok", True)+ detail = summarize_tool_result_detail(parsed, tool_name=tool_name, blocker_signal=blocker_signals)+ narrator_state.record_activity(+ build_tool_result_activity(tool_name, summary, success, iteration, call_id)+ )- if not client_gone:- await stream.send(- WorkflowCopilotToolResultUpdate(- type=WorkflowCopilotStreamMessageType.TOOL_RESULT,+ if not client_gone:+ await stream.send(+ WorkflowCopilotToolResultUpdate(+ type=WorkflowCopilotStreamMessageType.TOOL_RESULT,+ tool_name=tool_name,+ success=success,+ summary=summary,+ iteration=iteration,+ tool_call_id=call_id,+ detail=detail,+ )+ )++ if narrator_enabled:+ ctx_before = snapshot_ctx(ctx)+ _update_enforcement_from_tool(ctx, tool_name, parsed)+ ctx_after = snapshot_ctx(ctx)++ prior_tool_name = (+ narrator_state.pending_activity[-1].tool_name if narrator_state.pending_activity else None+ )+ narrator_state.record_tool(tool_name=tool_name,- success=success,summary=summary,+ success=success,iteration=iteration,- tool_call_id=call_id,- detail=detail,+ details=extract_tool_details(tool_name, parsed),)- )-- if narrator_enabled:- ctx_before = snapshot_ctx(ctx)- _update_enforcement_from_tool(ctx, tool_name, parsed)- ctx_after = snapshot_ctx(ctx)-- prior_tool_name = (- narrator_state.pending_activity[-1].tool_name if narrator_state.pending_activity else None- )- narrator_state.record_tool(- tool_name=tool_name,- summary=summary,- success=success,- iteration=iteration,- details=extract_tool_details(tool_name, parsed),- )- for transition in detect_transitions(ctx_before, ctx_after, tool_name, prior_tool_name):- narrator_state.record_transition(transition)- narrator_state.current_iteration = iteration- schedule_narration(narrator_state, stream, iteration)- else:- _update_enforcement_from_tool(ctx, tool_name, parsed)+ for transition in detect_transitions(ctx_before, ctx_after, tool_name, prior_tool_name):+ narrator_state.record_transition(transition)+ narrator_state.current_iteration = iteration+ schedule_narration(narrator_state, stream, iteration)+ else:+ _update_enforcement_from_tool(ctx, tool_name, parsed)try:_maybe_raise_unrecoverable_tool_error(ctx, tool_name, parsed)@@ -262,6 +289,42 @@ async def stream_to_sse(await cancel_in_flight(narrator_state)+async def _emit_code_repair_progress(+ *,+ narrator_state: NarratorState,+ stream: EventSourceStream,+ progress_text: str,+ iteration: int,+ client_gone: bool,+) -> None:+ """Surface a code-authoring reject as one quiet progress entry per turn. The persisted activity+ record runs unconditionally (so a disconnected client gets it on rehydration); only the live+ frame is gated on the client."""+ # Advance the narrator iteration on every classified round-trip, including a de-duplicated one,+ # so readers (e.g. the run-outcome frame) don't see a stale value.+ narrator_state.current_iteration = iteration+ if progress_text in narrator_state.emitted_progress_texts:+ return+ narrator_state.emitted_progress_texts.add(progress_text)++ # An LLM tool-start narration scheduled at tool_called would otherwise double+ # up with this progress entry for the same iteration; drop it.+ await cancel_in_flight(narrator_state)++ narration_ts = datetime.now(timezone.utc)+ narrator_state.record_activity(build_narration_activity(progress_text, iteration, narration_ts))+ if client_gone:+ return+ await stream.send(+ WorkflowCopilotNarrationUpdate(+ type=WorkflowCopilotStreamMessageType.NARRATION,+ narration=progress_text,+ iteration=iteration,+ timestamp=narration_ts,+ )+ )++async def flush_goal_satisfied_tool_result(stream: EventSourceStream, ctx: CopilotContext) -> None:"""Emit the TOOL_RESULT frame for the goal-satisfying tool.diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex c7943f927..b1068291f 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -61,6 +61,7 @@ from skyvern.forge.sdk.copilot.enforcement import (credential_priority_authoring_churn_stop_signal,)from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx+from skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXTfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violationsfrom skyvern.forge.sdk.copilot.output_policy import (OutputPolicyReason,@@ -991,6 +992,14 @@ def _code_block_parameter_contract_error(workflow_yaml: str) -> str | None:return "\n".join(errors) if errors else None+def _code_repair_progress_data() -> dict[str, str]:+ """Tag a code-authoring reject so the streaming adapter renders it as quiet de-duplicated progress."""+ return {+ "surface_kind": CODE_REPAIR_PROGRESS_SURFACE_KIND,+ "progress_text": CODE_REPAIR_PROGRESS_TEXT,+ }++def _code_seam_rejection_user_summary(*, metadata_rejected: bool, code_rejected: bool) -> str:if metadata_rejected and code_rejected:return "I need to adjust the workflow's code and its verification details before testing."@@ -3142,6 +3151,7 @@ async def _update_workflow("ok": False,"error": raw_conflict_marker_error,"user_facing_summary": _compiled_authoring_user_summary(),+ "data": _code_repair_progress_data(),}# Tool wrappers run authority/loop guards before calling here. The composition# gate below consumes these refs, so they must be visible before validation.@@ -3155,6 +3165,7 @@ async def _update_workflow("ok": False,"error": "\n".join(imposition.violations),"user_facing_summary": _compiled_authoring_user_summary(),+ "data": _code_repair_progress_data(),}workflow_yaml = imposition.workflow_yamlstripped_sandbox_imports: list[str] = []@@ -3166,6 +3177,7 @@ async def _update_workflow("ok": False,"error": "\n".join(typed_default_violations),"user_facing_summary": _compiled_authoring_user_summary(),+ "data": _code_repair_progress_data(),}params["workflow_yaml"] = workflow_yamlparameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)@@ -3174,6 +3186,7 @@ async def _update_workflow("ok": False,"error": parameter_contract_error,"user_facing_summary": _compiled_authoring_user_summary(),+ "data": _code_repair_progress_data(),}missing_metadata_error = _missing_code_artifact_metadata_error(workflow_yaml,@@ -3185,6 +3198,7 @@ async def _update_workflow("ok": False,"error": missing_metadata_error,"user_facing_summary": _compiled_authoring_user_summary(),+ "data": _code_repair_progress_data(),}scout_trajectory = getattr(ctx, "scout_trajectory", None)normalization = _normalize_code_artifact_metadata_detailed(@@ -3243,6 +3257,7 @@ async def _update_workflow(metadata_rejected=code_artifact_metadata_error is not None,code_rejected=bool(code_safety_errors),),+ "data": _code_repair_progress_data(),}if credential_scout_errors:_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex f4948cb11..fd7a7fc54 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -676,6 +676,54 @@ class TestCodeSafetySeam:assert ctx.code_artifact_metadata == {}+class TestCodeRepairProgressClassification:+ @pytest.mark.asyncio+ async def test_code_safety_seam_reject_carries_progress_surface_kind(self) -> None:+ ctx = _code_only_ctx()+ result = await _update_workflow({"workflow_yaml": _REQUESTS_IMPORT_CODE_YAML}, ctx)+ assert result["ok"] is False+ assert result["data"]["surface_kind"] == "code_repair_progress"+ assert result["data"]["progress_text"]+ # The substantive copy is unchanged; the progress text is a separate carrier.+ assert result["user_facing_summary"] == (+ "I need to adjust the workflow's code so it can run safely before testing."+ )+ assert result["data"]["progress_text"] != result["user_facing_summary"]++ @pytest.mark.asyncio+ async def test_raw_conflict_marker_reject_carries_progress_surface_kind(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ result = await _update_workflow({"workflow_yaml": f"<<<<<<< HEAD\n{_SAFE_CODE_YAML}"}, ctx)+ assert result["ok"] is False+ assert result["data"]["surface_kind"] == "code_repair_progress"+ assert result["data"]["progress_text"]++ @pytest.mark.asyncio+ async def test_credential_scout_reject_is_not_classified_as_progress(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = []+ result = await _update_workflow(+ {+ "workflow_yaml": _credential_code_yaml(+ code="""+ await page.locator("#email").fill(login_credential.username)+ await page.locator("input[type='password']").fill(login_credential.password)+ await page.locator("#totpmfa").fill(login_credential.totp)+ await page.locator("input[type='submit']").click()+ await page.wait_for_load_state("load")+ """+ )+ },+ ctx,+ )+ assert result["ok"] is False+ assert result["data"]["failure_type"] == "missing_credential_or_init"+ assert result["data"].get("surface_kind") is None++class TestCodeBlockParameterPersistSeam:@pytest.mark.asyncioasync def test_undeclared_parameter_key_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/unit/test_copilot_narrative_activity.py b/tests/unit/test_copilot_narrative_activity.pyindex eb70e8f6b..6f7a49447 100644--- a/tests/unit/test_copilot_narrative_activity.py+++ b/tests/unit/test_copilot_narrative_activity.py@@ -75,6 +75,15 @@ def test_narration_activity_shape() -> None:assert "toolName" not in entry+def test_emitted_progress_texts_is_a_fresh_per_state_set() -> None:+ # NarratorState is born and dies with one turn's CopilotContext, so the+ # set is per-turn by construction (no cross-turn leakage between states).+ first = NarratorState()+ first.emitted_progress_texts.add("Refining the workflow's code")+ second = NarratorState()+ assert second.emitted_progress_texts == set()++def test_record_activity_routes_to_design_when_no_block_running() -> None:state = NarratorState()state.record_activity(build_tool_call_activity("update_workflow", 0, "c1"))diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.pyindex b5f010d3b..9ad9473f1 100644--- a/tests/unit/test_copilot_streaming_adapter.py+++ b/tests/unit/test_copilot_streaming_adapter.py@@ -7,8 +7,12 @@ from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest+from agents.items import RunItem+from agents.stream_events import RunItemStreamEvent+from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.streaming_adapter import _sanitize_input, stream_to_sse+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageTypedef test_strips_workflow_yaml() -> None:@@ -784,3 +788,195 @@ class TestFlushGoalSatisfiedToolResult:assert sent == []assert narrator_state.design_activityassert narrator_state.design_activity[-1]["kind"] == "tool_result"+++_PROGRESS_TEXT = "Refining the workflow's code"+++def _code_repair_reject_payload() -> list[dict[str, str]]:+ body = json.dumps(+ {+ "ok": False,+ "error": "Insecure code detected: Not allowed to import modules.",+ "user_facing_summary": "I need to adjust the workflow's code so it can run safely before testing.",+ "data": {"surface_kind": "code_repair_progress", "progress_text": _PROGRESS_TEXT},+ }+ )+ return [{"type": "text", "text": body}]+++def _round_trip_events(call_id: str, tool_name: str, output_payload: Any) -> list[Any]:+ call_item = MagicMock(spec=RunItem)+ call_item.raw_item = {"call_id": call_id, "name": tool_name, "arguments": "{}"}+ out_item = MagicMock(spec=RunItem)+ out_item.raw_item = {"call_id": call_id, "name": tool_name}+ out_item.output = output_payload+ return [+ RunItemStreamEvent(name="tool_called", item=call_item),+ RunItemStreamEvent(name="tool_output", item=out_item),+ ]+++def _sink_stream(sent: list[Any], *, disconnected: bool = False) -> MagicMock:+ async def _send(payload: Any) -> bool:+ sent.append(payload)+ return True++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=disconnected)+ stream.send = _send+ return stream+++def _copilot_ctx() -> Any:+ return CopilotContext(+ organization_id="org_test",+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ workflow_yaml="",+ browser_session_id=None,+ stream=None, # type: ignore[arg-type]+ api_key=None,+ user_message="",+ )+++class TestCodeRepairProgressStreaming:+ @pytest.mark.asyncio+ async def test_repeated_classified_rejects_collapse_to_one_progress_frame(self) -> None:+ """Repeated identical classified rejects emit one progress narration and no failure rows."""+ events: list[Any] = []+ for idx in range(3):+ events.extend(_round_trip_events(f"c{idx}", "update_and_run_blocks", _code_repair_reject_payload()))++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(*events)+ result.cancel = MagicMock()++ sent: list[Any] = []+ ctx = _copilot_ctx()++ await stream_to_sse(result, _sink_stream(sent), ctx)++ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]+ assert tool_results == []++ narrations = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.NARRATION]+ assert len(narrations) == 1+ assert narrations[0].narration == _PROGRESS_TEXT++ narration_entries = [e for e in ctx.narrator_state.design_activity if e["kind"] == "narration"]+ assert len(narration_entries) == 1+ assert narration_entries[0]["text"] == _PROGRESS_TEXT+ assert all(e["kind"] != "tool_result" for e in ctx.narrator_state.design_activity)+ # De-duplicated rejects still advance the narrator iteration (read by the run-outcome frame).+ assert ctx.narrator_state.current_iteration == 2++ @pytest.mark.asyncio+ async def test_classified_progress_text_not_doubled_with_narrator_enabled(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """With the narrator handler enabled, the progress text still emits at most once."""++ async def _handler(prompt: str, prompt_name: str, **kwargs: object) -> str:+ return "a generic tool-start narration sentence"++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.streaming_adapter.resolve_narrator_handler",+ AsyncMock(return_value=_handler),+ )++ events: list[Any] = []+ for idx in range(3):+ events.extend(_round_trip_events(f"c{idx}", "update_and_run_blocks", _code_repair_reject_payload()))++ async def _slow_events() -> Any:+ for event in events:+ yield event+ await asyncio.sleep(0)+ for _ in range(10):+ await asyncio.sleep(0)++ result = MagicMock()+ result.stream_events = _slow_events+ result.cancel = MagicMock()++ sent: list[Any] = []+ ctx = _copilot_ctx()++ await stream_to_sse(result, _sink_stream(sent), ctx)++ progress_frames = [+ p+ for p in sent+ if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.NARRATION and p.narration == _PROGRESS_TEXT+ ]+ assert len(progress_frames) == 1++ progress_entries = [+ e for e in ctx.narrator_state.design_activity if e["kind"] == "narration" and e["text"] == _PROGRESS_TEXT+ ]+ assert len(progress_entries) == 1++ @pytest.mark.asyncio+ async def test_classified_reject_persists_one_entry_on_disconnect(self) -> None:+ """A disconnected client still gets the persisted progress entry; no live frame is sent."""+ events = _round_trip_events("c1", "update_and_run_blocks", _code_repair_reject_payload())++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(*events)+ result.cancel = MagicMock()++ sent: list[Any] = []+ ctx = _copilot_ctx()++ await stream_to_sse(result, _sink_stream(sent, disconnected=True), ctx)++ assert sent == []+ narration_entries = [e for e in ctx.narrator_state.design_activity if e["kind"] == "narration"]+ assert len(narration_entries) == 1+ assert narration_entries[0]["text"] == _PROGRESS_TEXT++ @pytest.mark.asyncio+ async def test_classified_reject_still_runs_enforcement_and_increments_iteration(self) -> None:+ """A classified reject still runs the enforcement update and advances the iteration."""+ first = _round_trip_events("c1", "update_and_run_blocks", _code_repair_reject_payload())+ second_call_item = MagicMock()+ second_call_item.raw_item = {"call_id": "c2", "name": "navigate_browser", "arguments": "{}"}+ second_call = RunItemStreamEvent(name="tool_called", item=second_call_item)++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(*first, second_call)+ result.cancel = MagicMock()++ sent: list[Any] = []+ ctx = _copilot_ctx()++ await stream_to_sse(result, _sink_stream(sent), ctx)++ # update_and_run_blocks flips test_after_update_done even on an ok:False reject; skipping+ # enforcement on the classified path would change agent behavior.+ assert ctx.test_after_update_done is True+ tool_calls = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_CALL]+ assert tool_calls[-1].iteration == 1++ @pytest.mark.asyncio+ async def test_unclassified_reject_still_renders_failure_tool_result(self) -> None:+ """An unclassified ok:False reject still emits a success=false TOOL_RESULT and activity row."""+ payload = [{"type": "text", "text": json.dumps({"ok": False, "error": "plain failure"})}]+ events = _round_trip_events("c1", "update_workflow", payload)++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(*events)+ result.cancel = MagicMock()++ sent: list[Any] = []+ ctx = _copilot_ctx()++ await stream_to_sse(result, _sink_stream(sent), ctx)++ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]+ assert len(tool_results) == 1+ assert tool_results[0].success is False+ narrations = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.NARRATION]+ assert narrations == []