Skip to content

Automate browser based workflows with AI

feat(SKY-12187): stream real codegen progress while copilot writes code (#7277)

Celal Zamanoğlu4961777eb781
diff --git a/skyvern/config.py b/skyvern/config.py
index 7a73310f4..171ef52da 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -206,6 +206,9 @@ class Settings(BaseSettings):
# Off = standard block authoring. On = prefer code blocks for browser work.
WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = False
WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY: bool = False
+ # Kill switch for the live codegen-progress SSE frame (drafted block labels while an authoring
+ # tool call streams). Off restores exact pre-change behavior; old frontends drop the frame either way.
+ WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED: bool = True
# Default code_only for MCP block/workflow tools. Off = permissive.
MCP_CODE_ONLY_MODE: bool = False
# Default for the bounded code-block self-heal; off by default.
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index a0710607f..bf4b62602 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -4,11 +4,15 @@ from __future__ import annotations
import asyncio
import json
+import re
+import time
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any
import structlog
+from skyvern.config import settings
+
# Reuse the HTTP-logging redactor so SSE tool inputs and request-body logs
# share one exact-match sensitive-key policy.
from skyvern.forge.request_logging import redact_sensitive_fields
@@ -30,6 +34,7 @@ from skyvern.forge.sdk.copilot.narration import (
)
from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user, summarize_tool_result_detail
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotCodegenProgressUpdate,
WorkflowCopilotDesignEndUpdate,
WorkflowCopilotDesignStartUpdate,
WorkflowCopilotNarrationUpdate,
@@ -62,6 +67,156 @@ _OBSERVATION_TOOLS = {
"press_key",
}
+_AUTHORING_TOOL_NAMES = frozenset({"update_and_run_blocks", "update_workflow"})
+# Pure substring heuristic over raw (unparsed) JSON text: a free-text field (e.g. navigation_goal)
+# that happens to contain the literal "label:" would also match. Accepted trade-off of not
+# json.loads-ing the partial buffer; worst case is a spurious drafted-block entry.
+_CODEGEN_LABEL_RE = re.compile(r"label:\s*\\?\"?([A-Za-z0-9_][A-Za-z0-9_ \-]{0,79})")
+_CODEGEN_MIN_GAP_SECONDS = 2.0
+# Keep enough trailing context that a label split across two argument deltas still matches.
+_CODEGEN_TAIL_OVERLAP = 96
+_CODEGEN_MAX_LABELS = 50
+
+
+class _CodegenCallState:
+ __slots__ = (
+ "tool_name",
+ "labels",
+ "_seen_labels",
+ "chars",
+ "n_deltas",
+ "tail",
+ "last_emit_monotonic",
+ "started_monotonic",
+ )
+
+ def __init__(self, tool_name: str) -> None:
+ self.tool_name = tool_name
+ self.labels: list[str] = []
+ self._seen_labels: set[str] = set()
+ self.chars = 0
+ self.n_deltas = 0
+ self.tail = ""
+ self.last_emit_monotonic = 0.0
+ self.started_monotonic = time.monotonic()
+
+ def add_labels(self, text: str) -> bool:
+ found_new = False
+ for match in _CODEGEN_LABEL_RE.finditer(text):
+ if match.end() == len(text):
+ # Match runs up to the edge of the currently scanned text: the label may still
+ # be mid-stream, split across a delta boundary. Wait for a delimiter to confirm
+ # it (re-scanned via the tail overlap on the next delta) instead of committing
+ # a truncated fragment permanently.
+ continue
+ if len(self.labels) >= _CODEGEN_MAX_LABELS:
+ break
+ label = match.group(1)
+ if label in self._seen_labels:
+ continue
+ self._seen_labels.add(label)
+ self.labels.append(label)
+ found_new = True
+ return found_new
+
+
+class _CodegenProgressTracker:
+ """Tracks streamed authoring-tool-call arguments (keyed by output_index, per the
+ FAKE_RESPONSES_ID caveat) and emits throttled CODEGEN_PROGRESS frames."""
+
+ def __init__(self) -> None:
+ self._calls: dict[int, _CodegenCallState] = {}
+
+ async def on_raw_event(
+ self,
+ data: Any,
+ stream: EventSourceStream,
+ ctx: CopilotContext,
+ iteration: int,
+ ) -> None:
+ event_type = getattr(data, "type", "")
+
+ if event_type == "response.created":
+ self._calls.clear()
+ return
+
+ if event_type == "response.output_item.added":
+ item = getattr(data, "item", None)
+ if getattr(item, "type", None) != "function_call":
+ return
+ name = getattr(item, "name", None)
+ if name not in _AUTHORING_TOOL_NAMES:
+ return
+ output_index = getattr(data, "output_index", None)
+ if output_index is None:
+ return
+ state = _CodegenCallState(name)
+ self._calls[output_index] = state
+ await self._emit(stream, ctx, iteration, state)
+ return
+
+ if event_type == "response.function_call_arguments.delta":
+ output_index = getattr(data, "output_index", None)
+ tracked_state = self._calls.get(output_index) if output_index is not None else None
+ if tracked_state is None:
+ return
+ delta = getattr(data, "delta", "") or ""
+ tracked_state.n_deltas += 1
+ tracked_state.chars += len(delta)
+ found_new = tracked_state.add_labels(tracked_state.tail + delta)
+ tracked_state.tail = (tracked_state.tail + delta)[-_CODEGEN_TAIL_OVERLAP:]
+ gap_elapsed = (time.monotonic() - tracked_state.last_emit_monotonic) >= _CODEGEN_MIN_GAP_SECONDS
+ if found_new or gap_elapsed:
+ await self._emit(stream, ctx, iteration, tracked_state)
+ return
+
+ if event_type in ("response.output_item.done", "response.function_call_arguments.done"):
+ # Both events fire for one completed call; pop() makes the second a no-op.
+ output_index = getattr(data, "output_index", None)
+ finished_state = self._calls.pop(output_index, None) if output_index is not None else None
+ if finished_state is not None:
+ LOG.info(
+ "copilot_codegen_progress_summary",
+ tool_name=finished_state.tool_name,
+ n_deltas=finished_state.n_deltas,
+ chars=finished_state.chars,
+ n_labels=len(finished_state.labels),
+ elapsed=time.monotonic() - finished_state.started_monotonic,
+ )
+ return
+
+ async def _emit(
+ self,
+ stream: EventSourceStream,
+ ctx: CopilotContext,
+ iteration: int,
+ state: _CodegenCallState,
+ ) -> None:
+ try:
+ await maybe_emit_design_start(stream, ctx)
+ except Exception as emit_err:
+ LOG.warning("copilot_narrative_design_start_emit_failed", error=str(emit_err))
+
+ # Advance the throttle gate regardless of connection outcome: otherwise a
+ # permanently disconnected client leaves last_emit_monotonic frozen and every
+ # subsequent delta re-attempts is_disconnected(), the "never per delta" cost
+ # this throttle exists to bound.
+ state.last_emit_monotonic = time.monotonic()
+ if await stream.is_disconnected():
+ return
+ try:
+ await stream.send(
+ WorkflowCopilotCodegenProgressUpdate(
+ tool_name=state.tool_name,
+ blocks_drafted=list(state.labels),
+ chars_streamed=state.chars,
+ iteration=iteration,
+ timestamp=datetime.now(timezone.utc),
+ )
+ )
+ except Exception as emit_err:
+ LOG.warning("copilot_codegen_progress_emit_failed", error=str(emit_err))
+
def _code_repair_progress_text(parsed: dict[str, Any]) -> str | None:
"""Return the classified code-repair progress text carried on a reject's ``data``, else None."""
@@ -96,7 +251,7 @@ async def stream_to_sse(
reasons unrelated to a dropped client) is re-raised unchanged so
asyncio's cancellation machinery still runs normally.
"""
- from agents.stream_events import RunItemStreamEvent
+ from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent
from skyvern.forge.sdk.copilot.enforcement import (
CopilotUnrecoverableToolError,
@@ -115,6 +270,8 @@ async def stream_to_sse(
# carry the same iteration value; it advances after the matching result.
iteration = 0
+ codegen_tracker = _CodegenProgressTracker() if settings.WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED else None
+
# Narrator state persists across enforcement iterations via ctx so
# cadence (last_emitted_at, min-gap) survives run_with_enforcement retries.
# Resolve the handler once (PostHog override → env-driven fallback) so
@@ -134,6 +291,9 @@ async def stream_to_sse(
try:
async for event in result.stream_events():
+ if codegen_tracker is not None and isinstance(event, RawResponsesStreamEvent):
+ await codegen_tracker.on_raw_event(event.data, stream, ctx, iteration)
+ continue
if not isinstance(event, RunItemStreamEvent):
continue
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index fac5cc05d..e4c7dc170 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -192,6 +192,7 @@ class WorkflowCopilotStreamMessageType(StrEnum):
DESIGN_START = "design_start"
DESIGN_END = "design_end"
WORKFLOW_DRAFT = "workflow_draft"
+ CODEGEN_PROGRESS = "codegen_progress"
class WorkflowCopilotProcessingUpdate(BaseModel):
@@ -398,6 +399,22 @@ class WorkflowCopilotWorkflowDraftUpdate(BaseModel):
)
+class WorkflowCopilotCodegenProgressUpdate(BaseModel):
+ """Live-only drafting progress while the LLM streams authoring-tool arguments. Not persisted; the
+ workflow_draft frame supersedes it."""
+
+ type: WorkflowCopilotStreamMessageType = Field(
+ WorkflowCopilotStreamMessageType.CODEGEN_PROGRESS, description="Message type"
+ )
+ tool_name: str = Field(..., description="Authoring tool whose arguments are being streamed")
+ blocks_drafted: list[str] = Field(
+ default_factory=list, description="Cumulative ordered unique block labels seen so far in this call"
+ )
+ chars_streamed: int = Field(..., description="Cumulative argument characters streamed so far in this call")
+ iteration: int = Field(..., description="Agent loop iteration number; matches the TOOL_CALL frame that follows")
+ timestamp: datetime = Field(..., description="Server timestamp")
+
+
class WorkflowYAMLConversionRequest(BaseModel):
workflow_definition_yaml: str = Field(..., description="Workflow definition YAML to convert to blocks")
workflow_id: str = Field(..., description="Workflow ID")
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index 1022ae435..a8c6f2dfd 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -8,8 +8,18 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from agents.items import RunItem
-from agents.stream_events import RunItemStreamEvent
-
+from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent
+from openai.types.responses import (
+ ResponseCreatedEvent,
+ ResponseFunctionCallArgumentsDeltaEvent,
+ ResponseFunctionCallArgumentsDoneEvent,
+ ResponseOutputItemAddedEvent,
+ ResponseOutputItemDoneEvent,
+)
+from openai.types.responses.response import Response
+from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
+
+from skyvern.forge.sdk.copilot import streaming_adapter as streaming_adapter_module
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.streaming_adapter import _sanitize_input, _update_enforcement_from_tool, stream_to_sse
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType
@@ -88,6 +98,91 @@ async def _stream_events_from(*events: Any) -> Any:
yield event
+def _minimal_response() -> Response:
+ return Response(
+ id="resp_1",
+ created_at=0.0,
+ model="gpt-4o",
+ object="response",
+ output=[],
+ parallel_tool_calls=True,
+ tool_choice="auto",
+ tools=[],
+ )
+
+
+def _response_created_event() -> RawResponsesStreamEvent:
+ return RawResponsesStreamEvent(
+ data=ResponseCreatedEvent(response=_minimal_response(), sequence_number=1, type="response.created")
+ )
+
+
+def _item_added_event(output_index: int, name: str, call_id: str = "__fake_responses_id__") -> RawResponsesStreamEvent:
+ item = ResponseFunctionToolCall(arguments="", call_id=call_id, name=name, type="function_call")
+ return RawResponsesStreamEvent(
+ data=ResponseOutputItemAddedEvent(
+ item=item, output_index=output_index, sequence_number=1, type="response.output_item.added"
+ )
+ )
+
+
+def _args_delta_event(output_index: int, delta: str, item_id: str = "__fake_responses_id__") -> RawResponsesStreamEvent:
+ return RawResponsesStreamEvent(
+ data=ResponseFunctionCallArgumentsDeltaEvent(
+ delta=delta,
+ item_id=item_id,
+ output_index=output_index,
+ sequence_number=1,
+ type="response.function_call_arguments.delta",
+ )
+ )
+
+
+def _args_done_event(
+ output_index: int, arguments: str, name: str, item_id: str = "__fake_responses_id__"
+) -> RawResponsesStreamEvent:
+ return RawResponsesStreamEvent(
+ data=ResponseFunctionCallArgumentsDoneEvent(
+ arguments=arguments,
+ item_id=item_id,
+ name=name,
+ output_index=output_index,
+ sequence_number=1,
+ type="response.function_call_arguments.done",
+ )
+ )
+
+
+def _item_done_event(output_index: int, name: str, arguments: str, item_id: str = "__fake_responses_id__") -> Any:
+ item = ResponseFunctionToolCall(arguments=arguments, call_id=item_id, name=name, type="function_call")
+ return RawResponsesStreamEvent(
+ data=ResponseOutputItemDoneEvent(
+ item=item, output_index=output_index, sequence_number=1, type="response.output_item.done"
+ )
+ )
+
+
+def _tool_called_event(call_id: str, name: str, arguments: str = "{}") -> RunItemStreamEvent:
+ call_item = MagicMock(spec=RunItem)
+ call_item.raw_item = {"call_id": call_id, "name": name, "arguments": arguments}
+ return RunItemStreamEvent(name="tool_called", item=call_item)
+
+
+def _tool_output_event(call_id: str, ok: bool = True, block_count: int = 2) -> RunItemStreamEvent:
+ out_item = MagicMock(spec=RunItem)
+ out_item.raw_item = {"call_id": call_id}
+ out_item.output = [{"type": "text", "text": json.dumps({"ok": ok, "data": {"block_count": block_count}})}]
+ return RunItemStreamEvent(name="tool_output", item=out_item)
+
+
+def _new_ctx() -> SimpleNamespace:
+ return SimpleNamespace(
+ last_artifact_health_blocker_reason=None,
+ completion_verification_result=None,
+ design_start_emitted=False,
+ )
+
+
@pytest.mark.asyncio
async def test_stream_to_sse_keeps_running_after_client_disconnect() -> None:
"""SKY-8986 regression: a dropped SSE client must NOT cancel the agent run.
@@ -1116,3 +1211,296 @@ async def test_stream_emits_no_detail_for_success() -> None:
)
assert payload.success is True
assert payload.detail is None
+
+
+def _codegen_payloads(sent: list[Any]) -> list[Any]:
+ return [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.CODEGEN_PROGRESS]
+
+
+async def test_codegen_progress_pins_incremental_label_extraction_on_raw_deltas() -> None:
+ """Regression pin: fails on old code because RawResponsesStreamEvent is dropped at the
+ ``isinstance(event, RunItemStreamEvent)`` skip, so zero CODEGEN_PROGRESS frames are ever sent."""
+ delta1 = '{"workflow_yaml": "blocks:\\n- block_type: navigation\\n label: open_page\\n'
+ delta2 = " navigation_goal: Navigate to the page\\n- block_type: task\\n label: fill_form\\n"
+ delta3 = ' navigation_goal: Fill out the form\\n"'
+ delta4 = ', "block_labels": ["open_page", "fill_form"]}'
+ full_args = delta1 + delta2 + delta3 + delta4
+
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ _args_delta_event(0, delta1),
+ _args_delta_event(0, delta2),
+ _args_delta_event(0, delta3),
+ _args_delta_event(0, delta4),
+ _args_done_event(0, full_args, "update_and_run_blocks"),
+ _item_done_event(0, "update_and_run_blocks", full_args),
+ _tool_called_event("c1", "update_and_run_blocks"),
+ _tool_output_event("c1"),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ codegen_payloads = _codegen_payloads(sent)
+ assert codegen_payloads, "expected at least one CODEGEN_PROGRESS frame"
+ assert any(p.blocks_drafted == ["open_page", "fill_form"] for p in codegen_payloads)
+ assert codegen_payloads[0].blocks_drafted == []
+ assert codegen_payloads[0].chars_streamed == 0
+
+
+async def test_codegen_progress_does_not_leak_truncated_label_split_across_deltas() -> None:
+ """A label that happens to end exactly at a delta boundary (real token-by-token
+ streaming can split anywhere) must never surface a truncated fragment like "open_pa"
+ in blocks_drafted -- only the completed "open_page" once the boundary resolves."""
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ _args_delta_event(0, '{"workflow_yaml": "blocks:\\n label: open_pa'),
+ _args_delta_event(0, 'ge\\n navigation_goal: x\\n"}'),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ codegen_payloads = _codegen_payloads(sent)
+ all_labels = {label for p in codegen_payloads for label in p.blocks_drafted}
+ assert "open_pa" not in all_labels
+ assert any(p.blocks_drafted == ["open_page"] for p in codegen_payloads)
+
+
+async def test_codegen_progress_throttles_no_new_label_deltas_then_emits_keepalive(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class _FakeClock:
+ def __init__(self) -> None:
+ self.value = 0.0
+
+ def monotonic(self) -> float:
+ return self.value
+
+ clock = _FakeClock()
+ monkeypatch.setattr(streaming_adapter_module, "time", clock)
+
+ async def _events() -> Any:
+ yield _item_added_event(0, "update_workflow")
+ yield _args_delta_event(0, "no label content here, ")
+ yield _args_delta_event(0, "still nothing new, ")
+ clock.value = 2.5
+ yield _args_delta_event(0, "and more with no label, ")
+
+ result = MagicMock()
+ result.stream_events = _events
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ codegen_payloads = _codegen_payloads(sent)
+ # ItemAdded emits immediately; the two in-gap deltas emit nothing; the
+ # post-gap delta emits a throttled keepalive with more chars, no new labels.
+ assert len(codegen_payloads) == 2
+ assert codegen_payloads[0].chars_streamed == 0
+ assert codegen_payloads[1].chars_streamed > codegen_payloads[0].chars_streamed
+ assert codegen_payloads[1].blocks_drafted == []
+
+
+async def test_codegen_progress_disabled_by_kill_switch(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(streaming_adapter_module.settings, "WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED", False)
+
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ _args_delta_event(0, '{"workflow_yaml": "blocks:\\n label: open_page\\n"}'),
+ _tool_called_event("c1", "update_and_run_blocks"),
+ _tool_output_event("c1"),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ assert _codegen_payloads(sent) == []
+ tool_call_payloads = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_CALL]
+ assert len(tool_call_payloads) == 1
+
+
+async def test_codegen_progress_ignores_non_authoring_tool() -> None:
+ events = [
+ _item_added_event(0, "evaluate"),
+ _args_delta_event(0, "label: should_not_appear\\n"),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ assert _codegen_payloads(sent) == []
+
+
+async def test_codegen_progress_resets_on_response_created() -> None:
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ _args_delta_event(0, '{"workflow_yaml": "blocks:\\n label: block_one\\n"}'),
+ _response_created_event(),
+ _item_added_event(0, "update_and_run_blocks"),
+ _args_delta_event(0, '{"workflow_yaml": "blocks:\\n label: block_two\\n"}'),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ await stream_to_sse(result, stream, _new_ctx())
+
+ codegen_payloads = _codegen_payloads(sent)
+ assert any(p.blocks_drafted == ["block_two"] for p in codegen_payloads)
+ assert not any("block_one" in p.blocks_drafted for p in codegen_payloads if p.blocks_drafted == ["block_two"])
+
+
+async def test_codegen_progress_first_frame_preceded_by_design_start() -> None:
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ ]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+
+ ctx = _new_ctx()
+ assert ctx.design_start_emitted is False
+
+ await stream_to_sse(result, stream, ctx)
+
+ types_in_order = [getattr(p, "type", None) for p in sent]
+ assert WorkflowCopilotStreamMessageType.DESIGN_START in types_in_order
+ design_start_index = types_in_order.index(WorkflowCopilotStreamMessageType.DESIGN_START)
+ codegen_index = types_in_order.index(WorkflowCopilotStreamMessageType.CODEGEN_PROGRESS)
+ assert design_start_index < codegen_index
+ assert ctx.design_start_emitted is True
+
+
+async def test_codegen_progress_is_disconnected_bounded_when_client_gone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """SKY-8986-adjacent: many argument deltas with a disconnected client must not call
+ is_disconnected() once per delta -- only at throttled emit attempts. Clock is pinned
+ (not real wall-clock) so the 2s gap deterministically never elapses within the 20 deltas,
+ proving the throttle -- not fast test execution -- bounds the call count."""
+
+ class _FakeClock:
+ def __init__(self) -> None:
+ self.value = 0.0
+
+ def monotonic(self) -> float:
+ return self.value
+
+ monkeypatch.setattr(streaming_adapter_module, "time", _FakeClock())
+
+ events = [
+ _item_added_event(0, "update_and_run_blocks"),
+ ] + [_args_delta_event(0, "repeated_label_chunk ") for _ in range(20)]
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=True)
+ stream.send = AsyncMock(return_value=True)
+
+ # Pre-mark design_start emitted so the unconditional (not disconnect-gated)
+ # design_start send doesn't muddy the codegen-frame-specific assertion below.
+ ctx = _new_ctx()
+ ctx.design_start_emitted = True
+
+ await stream_to_sse(result, stream, ctx)
+
+ stream.send.assert_not_called()
+ # 20 deltas were pumped with a clock that never advances (gap never elapses) and no
+ # new labels; is_disconnected must be checked only at the initial ItemAdded emit.
+ assert stream.is_disconnected.call_count == 1