Skip to content

Automate browser based workflows with AI

Report Copilot test-run failures honestly, and remember ones no later run retried (SKY-13206) (#7873)

Andrew Neilson479da0687833
diff --git a/pyproject.toml b/pyproject.toml
index f65e7fe32..a19aa9d7e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,7 +33,7 @@ local = [
"aiohttp>=3.13.4,<4",
"python-multipart>=0.0.30,<1",
"toml>=0.10.2,<0.11",
- "jinja2>=3.1.2,<4",
+ "jinja2>=3.1.6,<4",
"litellm>=1.84.0",
"playwright>1.46.0 ; python_version >= '3.12' and python_version < '3.14'",
"playwright==1.46.0 ; python_version >= '3.11' and python_version < '3.12'",
@@ -104,7 +104,7 @@ server = [
"aiohttp>=3.13.4,<4",
"python-multipart>=0.0.30,<1",
"toml>=0.10.2,<0.11",
- "jinja2>=3.1.2,<4",
+ "jinja2>=3.1.6,<4",
"uvicorn[standard]>=0.44.0",
"litellm>=1.84.0",
"playwright>1.46.0 ; python_version >= '3.12' and python_version < '3.14'",
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 939344a0b..908d56de4 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -2,6 +2,7 @@ import asyncio
import copy
import dataclasses
import json
+import re
import time
import warnings
from asyncio import CancelledError
@@ -11,7 +12,6 @@ import litellm
import structlog
from anthropic import NOT_GIVEN
from anthropic.types.beta.beta_message import BetaMessage as AnthropicMessage
-from jinja2 import Template
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.router import AllowedFailsPolicy
from litellm.utils import CustomStreamWrapper, ModelResponse
@@ -70,6 +70,12 @@ from skyvern.utils.url_validators import validate_fetch_url
configure_litellm_transport()
LOG = structlog.get_logger()
+_HASHED_HREF_PLACEHOLDER = re.compile(r"\{\{\s*(_[0-9a-f]{64})\s*\}\}")
+
+
+def _render_hashed_href_map(llm_content: str, hashed_href_map: dict[str, str]) -> str:
+ return _HASHED_HREF_PLACEHOLDER.sub(lambda match: hashed_href_map.get(match.group(1), ""), llm_content)
+
# Transient upstream faults litellm maps to these subclass openai.APIError, NOT
# litellm.exceptions.APIError, so a bare `except litellm.exceptions.APIError` misses them.
@@ -1856,7 +1862,7 @@ class LLMAPIHandlerFactory:
rendered_response_json = None
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
- rendered_content = Template(llm_content).render(context.hashed_href_map)
+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)
parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
@@ -2414,7 +2420,7 @@ class LLMAPIHandlerFactory:
rendered_response_json = None
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
- rendered_content = Template(llm_content).render(context.hashed_href_map)
+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)
parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
@@ -3017,7 +3023,7 @@ class LLMCaller:
rendered_response_json = None
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
- rendered_content = Template(llm_content).render(context.hashed_href_map)
+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)
parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 5a5970c70..d901d26d0 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
observed_value_extraction_scaffold_lines,
+ unresolved_runtime_block_failure,
)
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.code_block_synthesis import (
@@ -220,7 +221,7 @@ from skyvern.forge.sdk.copilot.turn_outcome import (
with_copilot_code_mode_diagnostics,
)
from skyvern.forge.sdk.copilot.turn_ownership import blocker_signal_render_allowed
-from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome, UnresolvedRuntimeFailure
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -1838,6 +1839,16 @@ def _assemble_terminal_envelope_safe(
return payload
+def _with_unresolved_runtime_failure_note(user_response: str, failure: UnresolvedRuntimeFailure) -> str:
+ label = failure.block_label or "an earlier step"
+ note = (
+ f"One thing to flag: an earlier test run ({failure.workflow_run_id}) failed at "
+ f'"{label}", the failing call is still in the draft, and no later run '
+ "verifiably re-exercised it — so that step is still unproven."
+ )
+ return f"{user_response.rstrip()}\n\n{note}" if user_response.strip() else note
+
+
def _make_agent_result(
ctx: CopilotContext | None,
*,
@@ -1898,6 +1909,26 @@ def _make_agent_result(
payload_updates["outcomeAdjudication"] = adjudication
if payload_updates or len(payload_base) != len(narrative_payload):
kwargs["narrative_payload"] = {**payload_base, **payload_updates}
+ if ctx is not None and turn_outcome is not None and turn_outcome.response_kind == ResponseKind.BUILD:
+ unresolved_failure = unresolved_runtime_block_failure(ctx)
+ if unresolved_failure is not None:
+ turn_outcome = turn_outcome.model_copy(update={"unresolved_runtime_failure": unresolved_failure})
+ kwargs["user_response"] = _with_unresolved_runtime_failure_note(
+ str(kwargs.get("user_response") or ""), unresolved_failure
+ )
+ # A reloaded chat renders the narrative card, and hydration prefers narrativeSummary over
+ # terminalMessage, so the qualification has to ride every surface or it survives the
+ # turn and disappears on refresh.
+ narrative = kwargs.get("narrative_payload")
+ if isinstance(narrative, dict):
+ kwargs["narrative_payload"] = {
+ **narrative,
+ **{
+ key: _with_unresolved_runtime_failure_note(narrative[key], unresolved_failure)
+ for key in ("terminalMessage", "narrativeSummary")
+ if isinstance(narrative.get(key), str) and narrative[key].strip()
+ },
+ }
terminal_envelope: dict[str, Any] | None = None
if ctx is not None:
blocker_reason, halt_kind = _terminal_halt_fields(ctx)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index e734bb77b..77947283d 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import ast
import hashlib
import json
import re
@@ -24,6 +25,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
)
from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, workflow_target_url
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+from skyvern.forge.sdk.copilot.failure_tracking import selector_identities_in_text, selector_identity_from_failure
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.result_evidence import (
LoadedResultCompositionEvidence,
@@ -31,6 +33,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (
scout_observation_contract_valid,
)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, RunOutcomeReasonCode
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import UnresolvedRuntimeFailure
LOG = structlog.get_logger()
@@ -123,6 +126,7 @@ class RecordedBuildTestOutcome(BaseModel):
attempted_tool: str = ""
attempted_target: str = ""
attempted_block_label: str = ""
+ attempted_call_ref: str = ""
verdict: BuildTestOutcomeVerdict
reason_code: BuildTestOutcomeReasonCode
observed_evidence_summary: str = ""
@@ -363,6 +367,10 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
"is_authoritative": outcome.is_authoritative,
"workflow_run_id": outcome.workflow_run_id,
"authored_structure_signature": outcome.authored_structure_signature,
+ "block_labels": list(outcome.block_labels),
+ "attempted_block_label": outcome.attempted_block_label,
+ "attempted_block_signature": _attempted_block_signature(ctx, outcome),
+ "attempted_call_ref": outcome.attempted_call_ref,
}
)
del history[:-_HISTORY_LIMIT]
@@ -381,6 +389,135 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
)
+def _attempted_block_signature(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome) -> str:
+ """YAML-derived so it stays comparable later; ``block_shape_hashes`` is built from block models
+ and never hashes equal to a YAML-derived signature."""
+ if outcome.reason_code != "runtime_block_failure" or not outcome.attempted_block_label:
+ return ""
+ return authored_block_signatures_from_workflow(ctx.workflow_yaml).get(outcome.attempted_block_label, "")
+
+
+def _code_blocks_by_label(workflow_yaml: str | None) -> dict[str, str]:
+ """Code text for every code block in the draft, including blocks nested in containers or loops."""
+ code_by_label: dict[str, str] = {}
+ if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
+ return code_by_label
+ parsed = _parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, Mapping):
+ return code_by_label
+
+ def walk(value: object) -> None:
+ for block in _mapping_list(value):
+ label = _safe_str(block.get("label"))
+ code = block.get("code")
+ if _safe_str(block.get("block_type")).lower() == "code" and label and isinstance(code, str):
+ code_by_label[label] = code
+ walk(block.get("blocks"))
+ walk(block.get("loop_blocks"))
+
+ walk(_dict(parsed.get("workflow_definition")).get("blocks"))
+ return code_by_label
+
+
+# Every construct that can reach the end of a block without running something inside it, including
+# expression-level ones: a zero-length comprehension skips its body exactly as an unentered loop does.
+# Over-detection only costs a redundant note; a miss silently drops a real failure.
+_SELECTOR_CALL_ATTRS = frozenset({"locator", "get_by_role", "get_by_text", "get_by_label", "get_by_placeholder"})
+
+_CALL_SKIPPING_NODES = (
+ ast.If,
+ ast.IfExp,
+ ast.BoolOp,
+ ast.Try,
+ ast.TryStar,
+ ast.While,
+ ast.For,
+ ast.AsyncFor,
+ ast.comprehension,
+ ast.Match,
+ ast.Lambda,
+ ast.FunctionDef,
+ ast.AsyncFunctionDef,
+)
+
+
+def _parse_block_code(code: str) -> ast.AsyncFunctionDef | None:
+ """Block code is an async body, so it only parses inside a wrapper; None means unparseable."""
+ try:
+ wrapper = ast.parse("async def _block():\n" + textwrap.indent(code, " ")).body[0]
+ except Exception:
+ # Deeply nested generated code raises MemoryError from the parser rather than SyntaxError,
+ # and no caller may propagate a parse failure out of turn assembly.
+ return None
+ return wrapper if isinstance(wrapper, ast.AsyncFunctionDef) else None
+
+
+def _code_can_skip_a_call(code: str) -> bool:
+ """Unparseable code counts as branching so an unrecognized shape never credits re-exercise."""
+ wrapper = _parse_block_code(code)
+ if wrapper is None:
+ return True
+ return any(isinstance(node, _CALL_SKIPPING_NODES) for stmt in wrapper.body for node in ast.walk(stmt))
+
+
+def _selector_removal_is_provable(code: str) -> bool:
+ """Whether every selector in the block is a literal, so a scan that misses one proves removal.
+
+ A selector built at runtime (an f-string, a variable, a concatenation) is invisible to the
+ literal scan, so its absence would otherwise read as the failing call having been edited away.
+ """
+ wrapper = _parse_block_code(code)
+ if wrapper is None:
+ return False
+ for node in ast.walk(wrapper):
+ if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):
+ continue
+ if node.func.attr not in _SELECTOR_CALL_ATTRS:
+ continue
+ selector_args = [*node.args, *(kw.value for kw in node.keywords if kw.arg == "name")]
+ if any(not isinstance(arg, ast.Constant) or not isinstance(arg.value, str) for arg in selector_args):
+ return False
+ return True
+
+
+def unresolved_runtime_block_failure(ctx: _RecordedBuildTestOutcomeContext) -> UnresolvedRuntimeFailure | None:
+ """The newest runtime block failure that no later run verifiably re-exercised or edited away."""
+ history = ctx.recorded_build_test_outcome_history
+ for index in range(len(history) - 1, -1, -1):
+ entry = history[index]
+ if entry.get("reason_code") != "runtime_block_failure":
+ continue
+ label = _safe_str(entry.get("attempted_block_label"))
+ signature = _safe_str(entry.get("attempted_block_signature"))
+ run_id = _safe_str(entry.get("workflow_run_id"))
+ call_ref = _safe_str(entry.get("attempted_call_ref"))
+ # Scout evaluations and author-time rejects share this history, so a later *run* has to be
+ # identified by phase and a different run id -- otherwise author-time work after the failure
+ # would read as a later run that skipped the block.
+ later_runs = [
+ entry_after
+ for entry_after in history[index + 1 :]
+ if entry_after.get("phase") == "persisted_block_run"
+ and _safe_str(entry_after.get("workflow_run_id")) not in ("", run_id)
+ ]
+ code = _code_blocks_by_label(ctx.workflow_yaml).get(label)
+ if not (label and run_id and later_runs and code):
+ return None
+ # Executing the block re-exercises the failing call only when the block runs straight
+ # through; a branch inside it can reach the end without reaching the call.
+ if not _code_can_skip_a_call(code):
+ for later_run in later_runs:
+ if label in _string_list(later_run.get("block_labels")):
+ return None
+ if call_ref:
+ if call_ref not in selector_identities_in_text(code) and _selector_removal_is_provable(code):
+ return None
+ elif signature and authored_block_signatures_from_workflow(ctx.workflow_yaml).get(label) != signature:
+ return None
+ return UnresolvedRuntimeFailure(workflow_run_id=run_id, block_label=label)
+ return None
+
+
def bind_post_run_page_path_failure(
ctx: _RecordedBuildTestOutcomeContext,
page_evidence: Mapping[str, object],
@@ -1261,6 +1398,9 @@ def recorded_outcome_from_run_blocks_result(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",
+ attempted_call_ref=selector_identity_from_failure(
+ _safe_str(failed_block.get("failure_reason")) if failed_block is not None else ""
+ ),
verdict=verdict,
reason_code=reason_code,
workflow_run_id=workflow_run_id or None,
diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.py
index 59959db65..47dac121f 100644
--- a/skyvern/forge/sdk/copilot/failure_tracking.py
+++ b/skyvern/forge/sdk/copilot/failure_tracking.py
@@ -167,6 +167,28 @@ def _selector_from_text(text: str) -> tuple[str, str]:
return "", ""
+_CSS_SELECTOR_KIND = "locator"
+
+
+def selector_identity_from_failure(text: str) -> str:
+ kind, value = _selector_from_text(text or "")
+ # "selector" and "locator" are the same CSS string under two spellings; the code side only ever
+ # spells it "locator", so a failure that named it "selector" must normalize or never match.
+ return f"{_CSS_SELECTOR_KIND if kind == 'selector' else kind}:{value}" if kind else ""
+
+
+def selector_identities_in_text(text: str) -> set[str]:
+ """Every locator identity in ``text``, normalized to match ``selector_identity_from_failure``."""
+ identities: set[str] = set()
+ for match in _LOCATOR_RE.finditer(text or ""):
+ identities.add(f"{_CSS_SELECTOR_KIND}:{_normalize_signature_text(match.group('selector'))}")
+ for match in _ROLE_RE.finditer(text or ""):
+ role = _normalize_signature_text(match.group("role"))
+ name = _normalize_signature_text(match.group("name") or "")
+ identities.add(f"role:{role}:{name}" if name else f"role:{role}")
+ return identities
+
+
def _error_class_from_text(text: str) -> str:
code_block_imposition = _code_block_imposition_error_class(text)
if code_block_imposition:
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 3831e6645..0bf40dfab 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -1914,7 +1914,16 @@ def _tool_visible_result_after_completion_verification(
return result
copied_data = dict(data) if isinstance(data, dict) else {}
- copied_data["failure_reason"] = outcome_unverified_reason
+ # Every outcome-unverified reason asserts a run that completed, so substituting it for a raised
+ # block's own reason told the agent a failed run had completed and sent repair at the extraction
+ # instead of at the failing call. Append it as subordinate context instead.
+ run_failure_reason = next(iter_failure_reasons(result), "")
+ agent_facing_reason = (
+ f"{run_failure_reason.rstrip('. ')}. Completion verification also reported: {outcome_unverified_reason}"
+ if run_failure_reason
+ else outcome_unverified_reason
+ )
+ copied_data["failure_reason"] = agent_facing_reason
copied_data["completion_verification"] = (
completion_verification.to_trace_data() if completion_verification is not None else None
)
@@ -1932,7 +1941,7 @@ def _tool_visible_result_after_completion_verification(
return {
**result,
"ok": False,
- "error": outcome_unverified_reason,
+ "error": agent_facing_reason,
"data": copied_data,
}
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 44ee5d34d..affe23e98 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -2642,7 +2642,11 @@ def _record_run_blocks_result(
copilot_ctx.last_artifact_health_blocker_labels = []
copilot_ctx.last_artifact_health_failure_classes = []
if completion_verification is not None and completion_verification.status == "evaluated":
- copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ # Every unverified-outcome reason asserts a run that completed, so a run that
+ # raised has no outcome-gate reason and must report its own failure instead.
+ copilot_ctx.last_outcome_gate_reason = (
+ _outcome_unverified_reason(copilot_ctx, completion_verification) if run_ok else None
+ )
copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_id
copilot_ctx.last_test_suspicious_success = False
if prior_committed_outcome is None:
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index af3d7dab9..be0f667f2 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -7,10 +7,11 @@ in any ``copilot/`` business logic — derivation lives in
from __future__ import annotations
+import re
from enum import StrEnum
from typing import Any, Literal
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
class ResponseKind(StrEnum):
@@ -21,6 +22,26 @@ class ResponseKind(StrEnum):
RECOVER = "recover"
+_UNSAFE_IDENTIFIER_RE = re.compile(r"[^A-Za-z0-9_\- ]")
+
+
+class UnresolvedRuntimeFailure(BaseModel):
+ """A failure a successful turn could not clear, recorded so the outcome is gradeable after the
+ fact; the reply text derives independently, and nothing keys success or verification on this."""
+
+ model_config = ConfigDict(extra="ignore", frozen=True)
+
+ workflow_run_id: str
+ block_label: str
+
+ @field_validator("workflow_run_id", "block_label")
+ @classmethod
+ def _bare_identifier(cls, value: str) -> str:
+ # Both fields are model-authored and reach the chat reply and the history API, so they are
+ # reduced to bounded identifiers here rather than at each surface that renders them.
+ return _UNSAFE_IDENTIFIER_RE.sub("", value)[:80].strip()
+
+
class TurnOutcome(BaseModel):
# extra="ignore" so a rolling deploy that adds a new TurnOutcome field
# does not make older readers silently treat freshly-written rows as None.
@@ -39,3 +60,4 @@ class TurnOutcome(BaseModel):
copilot_last_code_build_failed: bool = False
copilot_pending_capability: str | None = None
copilot_turn_id: str | None = None
+ unresolved_runtime_failure: UnresolvedRuntimeFailure | None = None
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 43cd4198a..01724b7fe 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -10,6 +10,7 @@ from unittest.mock import MagicMock
import pytest
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisInput,
@@ -209,3 +210,63 @@ def make_completion_criterion(
requested_output_floor_rekeyed=requested_output_floor_rekeyed,
floor_rekeyed_from_path=floor_rekeyed_from_path,
)
+
+
+def two_page_login_yaml(*, submit_selector: str = "Login") -> str:
+ """The shape copilot emits in code-block mode: branch bodies are code inside one always-executed
+ block, so a passing run can traverse it without reaching the guarded call."""
+ return f"""
+ title: Sign in and read the metric
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: sign_in_and_read
+ code: |
+ await page.fill("#user", "demo")
+ await page.click("#submit")
+ if await page.locator("#token").count():
+ await page.get_by_role("button", name="{submit_selector}", exact=True).click()
+ return {{"visitors": "9.42K"}}
+ """
+
+
+def straight_line_login_yaml() -> str:
+ """One always-executed code block with no branching: executing it reaches every call in it."""
+ return """
+ title: Sign in and read the metric
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: sign_in_and_read
+ code: |
+ await page.fill("#user", "demo")
+ await page.get_by_role("button", name="Login", exact=True).click()
+ return {"visitors": "9.42K"}
+ """
+
+
+def failed_second_factor_run(run_id: str) -> RecordedBuildTestOutcome:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ attempted_block_label="sign_in_and_read",
+ attempted_call_ref="role:button:Login",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id=run_id,
+ block_labels=["sign_in_and_read"],
+ structural_failure_identity="locator-timeout-identity",
+ )
+
+
+def passing_run(run_id: str, block_labels: list[str]) -> RecordedBuildTestOutcome:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="progress_observed",
+ reason_code="run_completed_unevaluated",
+ workflow_run_id=run_id,
+ block_labels=block_labels,
+ structural_failure_identity="",
+ evidence_refs=["rows:1"],
+ )
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index b5b6419ab..b07817cc0 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from asyncio import CancelledError
from datetime import datetime
+from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -27,6 +28,94 @@ def _custom_llm_config(model_name: str, api_base: str = "https://llm.example.tes
return LLMConfig(model_name, [], False, False, {"api_key": "test-key", "api_base": api_base})
+def test_render_hashed_href_map_blocks_lipsum_globals_command(tmp_path: Path) -> None:
+ marker = tmp_path / "ssti_canary"
+ payload = "{{ lipsum.__globals__['os'].popen('touch " + str(marker) + "').read() }}"
+
+ result = api_handler_factory._render_hashed_href_map(payload, {})
+
+ assert not marker.exists()
+ assert result == payload
+
+
+def test_render_hashed_href_map_blocks_string_class_subclasses_gadget(tmp_path: Path) -> None:
+ marker = tmp_path / "ssti_canary"
+ payload = (
+ "{% for cls in ''.__class__.__mro__[1].__subclasses__() %}"
+ "{% if cls.__name__ == 'catch_warnings' %}"
+ f"{{{{ cls()._module.__builtins__['__import__']('os').system('touch {marker}') }}}}"
+ "{% endif %}{% endfor %}"
+ )
+
+ result = api_handler_factory._render_hashed_href_map(payload, {})
+
+ assert not marker.exists()
+ assert result == payload
+
+
+def test_render_hashed_href_map_blocks_cycler_globals_gadget(tmp_path: Path) -> None:
+ marker = tmp_path / "ssti_canary"
+ payload = f"{{{{ cycler.__init__.__globals__['os'].popen('touch {marker}').read() }}}}"
+
+ result = api_handler_factory._render_hashed_href_map(payload, {})
+
+ assert not marker.exists()
+ assert result == payload
+
+
+def test_render_hashed_href_map_replaces_only_hashed_url_placeholders_with_optional_spaces() -> None:
+ first_key = f"_{'a' * 64}"
+ second_key = f"_{'b' * 64}"
+ first_url = "https://example.test/a/very/long/path?with=query&and=more-query-parameters"
+ second_url = "https://example.test/another/path"
+ hashed_href_map = {first_key: first_url, second_key: second_url}
+ payload = (
+ '{"message":"Use the links without changing ordinary prose.",'
+ f'"links":["{{{{{first_key}}}}}","{{{{ {second_key} }}}}","{{{{{first_key}}}}}"],'
+ '"template":"{{ unknown_key }}"}'
+ )
+ expected = (
+ '{"message":"Use the links without changing ordinary prose.",'
+ f'"links":["{first_url}","{second_url}","{first_url}"],'
+ '"template":"{{ unknown_key }}"}'
+ )
+
+ assert api_handler_factory._render_hashed_href_map(f"{{{{{first_key}}}}}", hashed_href_map) == first_url
+ assert api_handler_factory._render_hashed_href_map(payload, hashed_href_map) == expected
+
+
+def test_render_hashed_href_map_removes_unknown_hash_and_preserves_other_placeholder() -> None:
+ unknown_hash = f"_{'c' * 64}"
+ other_placeholder = "ordinary prose with {{ unknown_key }} left intact"
+
+ assert api_handler_factory._render_hashed_href_map(f"{{{{{unknown_hash}}}}}", {}) == ""
+ assert api_handler_factory._render_hashed_href_map(other_placeholder, {}) == other_placeholder
+
+
+def test_render_hashed_href_map_returns_malformed_template_unchanged() -> None:
+ payload = "response with stray {{"
+
+ result = api_handler_factory._render_hashed_href_map(payload, {})
+
+ assert result == payload
+
+
+def test_render_hashed_href_map_returns_self_calling_macro_unchanged() -> None:
+ payload = "{% macro m() %}{{ m() }}{% endmacro %}{{ m() }}"
+
+ result = api_handler_factory._render_hashed_href_map(payload, {})
+
+ assert result == payload
+
+
+def test_render_hashed_href_map_returns_expensive_templates_unchanged() -> None:
+ range_payload = "{% for i in range(10000000) %}x{% endfor %}"
+ multiplication_payload = "{{ 'a' * 1000000000 }}"
+
+ assert api_handler_factory._render_hashed_href_map(range_payload, {}) == range_payload
+ assert api_handler_factory._render_hashed_href_map(multiplication_payload, {}) == multiplication_payload
+
+
@pytest.mark.parametrize(
("model_name", "http_client_attribute"),
[("openai/example-model", "_client"), ("ollama_chat/example-model", "client")],
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 21433c602..f341949e4 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -22,12 +22,20 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
recorded_outcome_from_run_blocks_result,
recorded_outcome_from_scout_act_observe_hollow,
run_backed_repair_evidence_exists,
+ unresolved_runtime_block_failure,
)
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+from skyvern.forge.sdk.copilot.failure_tracking import selector_identity_from_failure
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, LoadedResultCompositionTarget
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from tests.unit.copilot_test_helpers import (
+ failed_second_factor_run,
+ passing_run,
+ straight_line_login_yaml,
+ two_page_login_yaml,
+)
def _metadata_reject_ladder_input(*, structural_key: str = "reject-key") -> MetadataRejectLadderInput:
@@ -1512,3 +1520,230 @@ def test_clearing_the_grounding_requirement_also_drops_the_binding_constraint()
assert ctx.recorded_outcome_grounding_requirement is None
assert ctx.recorded_outcome_binding_constraint is None
+
+
+def _run_history_ctx(workflow_yaml: str) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_yaml=workflow_yaml,
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
+
+
+def test_failure_stays_open_when_a_later_run_passes_without_taking_the_failed_branch() -> None:
+ """The live shape: the block runs to completion while its internal branch skips the failing call."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.workflow_run_id == "wr_1"
+ assert open_failure.block_label == "sign_in_and_read"
+
+
+def test_a_straight_line_block_a_later_run_executed_is_re_exercised() -> None:
+ """No branch can skip a call, so executing the block proves the failing call ran again."""
+ ctx = _run_history_ctx(straight_line_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ assert unresolved_runtime_block_failure(ctx) is None
+
+
+def test_failure_retires_once_the_implicated_call_leaves_the_draft() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+ ctx.workflow_yaml = two_page_login_yaml(submit_selector="Continue")
+
+ assert unresolved_runtime_block_failure(ctx) is None
+
+
+def test_an_unrelated_edit_to_the_block_does_not_retire_the_failing_call() -> None:
+ """The call-level check earns its place here: the block changed, the failing call did not."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+ ctx.workflow_yaml = two_page_login_yaml().replace(
+ 'return {"visitors": "9.42K"}', 'return {"visitors": "9.42K", "extra": 1}'
+ )
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.block_label == "sign_in_and_read"
+
+
+def test_a_failure_with_no_later_run_is_the_turns_own_headline_not_an_unresolved_note() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+
+ assert unresolved_runtime_block_failure(ctx) is None
+
+
+def test_another_record_for_the_same_run_id_is_not_a_later_run() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_1", ["sign_in_and_read"]))
+
+ assert unresolved_runtime_block_failure(ctx) is None
+
+
+def test_author_time_work_after_the_failure_is_not_a_later_run() -> None:
+ """Scout evaluations and author-time rejects share this history but execute nothing."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="synthesized_parameter_binding_ambiguous",
+ structural_failure_identity="author-time-identity",
+ ),
+ )
+
+ assert unresolved_runtime_block_failure(ctx) is None
+
+
+def test_a_later_run_still_arms_the_note_when_author_time_work_follows_it() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["read_metric"]))
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="scout_evaluate",
+ attempted_tool="evaluate",
+ verdict="progress_observed",
+ reason_code="run_completed_unevaluated",
+ structural_failure_identity="scout-identity",
+ ),
+ )
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.workflow_run_id == "wr_1"
+
+
+def test_a_code_block_nested_in_a_loop_is_still_found() -> None:
+ """A failure inside a loop body must stay reportable, not vanish because the walk missed it."""
+ ctx = _run_history_ctx(
+ """
+ title: Loop over rows
+ workflow_definition:
+ blocks:
+ - block_type: for_loop
+ label: each_row
+ loop_blocks:
+ - block_type: code
+ label: sign_in_and_read
+ code: |
+ if await page.locator("#token").count():
+ await page.get_by_role("button", name="Login", exact=True).click()
+ """
+ )
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["each_row"]))
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.block_label == "sign_in_and_read"
+
+
+def test_a_css_selector_failure_matches_the_locator_call_in_the_draft() -> None:
+ """A failure that spells it "selector:" must still match code that spells it locator()."""
+ ctx = _run_history_ctx(
+ """
+ title: Read the metric
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: sign_in_and_read
+ code: |
+ if await page.locator("#token").count():
+ await page.locator("#submit-btn").click()
+ """
+ )
+ # Derived, not hardcoded: the point of the test is that the two spellings normalize together.
+ call_ref = selector_identity_from_failure('TimeoutError: resolved selector: "#submit-btn" never appeared')
+ assert call_ref == "locator:#submit-btn"
+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": call_ref})
+ record_build_test_outcome(ctx, outcome)
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.block_label == "sign_in_and_read"
+
+
+def test_run_blocks_outcome_extracts_the_implicated_call_from_the_failure_text() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "failure_type": "runtime_error",
+ "blocks": [
+ {
+ "label": "get_visitors",
+ "block_type": "code",
+ "status": "failed",
+ "failure_reason": (
+ "code block failed. failure reason: Failed to execute code block. Reason: "
+ "TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for "
+ 'get_by_role("button", name="Continue", exact=True)'
+ ),
+ },
+ ],
+ },
+ },
+ )
+
+ assert outcome is not None
+ assert outcome.attempted_call_ref == "role:button:Continue"
+
+
+def _templated_selector_yaml(*, templated: bool = True) -> str:
+ call = 'f"#submit-{kind}"' if templated else '"#submit-btn"'
+ return f"""
+ title: Read the metric
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: sign_in_and_read
+ code: |
+ if await page.locator("#token").count():
+ await page.locator({call}).click()
+ """
+
+
+def test_a_templated_selector_does_not_read_as_the_call_being_removed() -> None:
+ """A runtime-built selector is invisible to the literal scan, so it cannot prove removal."""
+ ctx = _run_history_ctx(_templated_selector_yaml())
+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": "locator:#submit-btn"})
+ record_build_test_outcome(ctx, outcome)
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ open_failure = unresolved_runtime_block_failure(ctx)
+
+ assert open_failure is not None
+ assert open_failure.block_label == "sign_in_and_read"
+
+
+def test_an_all_literal_block_still_retires_when_the_call_is_gone() -> None:
+ """The pass path: with every selector a literal, absence really is proof of removal."""
+ ctx = _run_history_ctx(_templated_selector_yaml(templated=False))
+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": "locator:#gone-btn"})
+ record_build_test_outcome(ctx, outcome)
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ assert unresolved_runtime_block_failure(ctx) is None
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 1fafbcd01..d98570ff6 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -21,6 +21,10 @@ from skyvern.forge.sdk.copilot.agent import (
_rewrite_failed_test_response,
_verified_workflow_or_none,
)
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ compose_loop_blocker_user_facing_reason,
+ loop_blocker_evidence_from_ctx,
+)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSet,
StoredCriteriaSnapshot,
@@ -3289,6 +3293,25 @@ def test_tool_visible_result_fails_when_confirmation_block_outcome_unmet() -> No
assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"
+def test_tool_visible_result_leads_with_the_raised_blocks_own_reason() -> None:
+ """Live shape: the block raised, but the agent was told the run completed, so it repaired the
+ extraction instead of the failing call."""
+ ctx = _ctx_with_blocks("extraction")
+ result = _failed_code_block_result()
+ result["data"]["failure_reason"] = (
+ "code block failed. failure reason: Failed to execute code block. "
+ "Reason: Exception: Could not parse visitor total from the page"
+ )
+
+ visible = _tool_visible_result_after_completion_verification(ctx, result, _evaluated(("c0", False)))
+
+ assert not visible["error"].startswith("The run completed")
+ assert "Could not parse visitor total" in visible["error"]
+ assert "Could not parse visitor total" in visible["data"]["failure_reason"]
+ assert "item in cart" in visible["error"]
+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"
+
+
def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:
ctx = _ctx_with_blocks("goto_url", "navigation")
@@ -4724,12 +4747,26 @@ def _failed_code_block_result() -> dict:
}
-def test_failed_run_records_gate_reason_separately_from_raw_block_failure() -> None:
+def test_failed_run_leaves_no_outcome_unverified_reason_and_reports_the_block_failure() -> None:
ctx = _ctx_with_blocks("extraction")
_record_run_blocks_result(ctx, _failed_code_block_result(), completion_verification=_evaluated(("c0", False)))
- assert "item in cart" in (ctx.last_outcome_gate_reason or "")
- assert "TimeoutError" not in (ctx.last_outcome_gate_reason or "")
+ assert ctx.last_outcome_gate_reason is None
+ assert ctx.last_outcome_gate_workflow_run_id == "wr_x"
assert "TimeoutError" in (ctx.last_test_failure_reason or "")
+ composed, _ = compose_loop_blocker_user_facing_reason("loop_detected_generic", loop_blocker_evidence_from_ctx(ctx))
+ assert "The run completed" not in composed
+
+
+def test_ok_run_still_records_the_outcome_unverified_reason() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
+ assert "item in cart" in (ctx.last_outcome_gate_reason or "")
+ assert ctx.last_outcome_gate_workflow_run_id == "wr_x"
+ composed, tiers = compose_loop_blocker_user_facing_reason(
+ "loop_detected_generic", loop_blocker_evidence_from_ctx(ctx)
+ )
+ assert "The run completed" in composed
+ assert "verdict" in tiers
def test_gate_reason_survives_a_later_run_without_verification() -> None:
diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py
index 7f6c1a00d..4e2c14eac 100644
--- a/tests/unit/test_copilot_make_agent_result_backfill.py
+++ b/tests/unit/test_copilot_make_agent_result_backfill.py
@@ -10,6 +10,7 @@ import pytest
from skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_result
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.build_test_outcome import record_build_test_outcome
from skyvern.forge.sdk.copilot.context import (
AgentResult,
CopilotContext,
@@ -17,8 +18,9 @@ from skyvern.forge.sdk.copilot.context import (
)
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+from tests.unit.copilot_test_helpers import failed_second_factor_run
from tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx
-from tests.unit.copilot_test_helpers import make_verified_goal_contract
+from tests.unit.copilot_test_helpers import make_verified_goal_contract, passing_run, two_page_login_yaml
def _verified_goal_ctx() -> CopilotContext:
@@ -44,7 +46,7 @@ def _payload(**overrides: object) -> dict:
"blocks": [],
"terminal": "response",
"terminalMessage": "done",
- "narrativeSummary": None,
+ "narrativeSummary": "Built it.",
"priorBlockCount": None,
"designActivity": [],
"startedAt": None,
@@ -175,3 +177,96 @@ def test_make_agent_result_records_resolved_credentials_as_durable_approval() ->
approved = StructuredContext.from_json_str(result.global_llm_context).approved_credentials
assert [record.credential_id for record in approved] == ["cred_portal"]
+
+
+def _ctx_with_open_second_factor_failure(
+ *, later_run_labels: list[str], final_selector: str = "Login"
+) -> CopilotContext:
+ ctx = _ctx(workflow_yaml=two_page_login_yaml())
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", later_run_labels))
+ ctx.workflow_yaml = two_page_login_yaml(submit_selector=final_selector)
+ return ctx
+
+
+def test_build_turn_reports_the_failure_no_later_run_re_exercised() -> None:
+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])
+
+ result = _result(
+ ctx,
+ user_response="Built it. The workflow reads the visitor count.",
+ turn_outcome=_outcome(ResponseKind.BUILD),
+ narrative_payload=_payload(),
+ )
+
+ assert result.turn_outcome is not None
+ unresolved = result.turn_outcome.unresolved_runtime_failure
+ assert unresolved is not None
+ assert unresolved.workflow_run_id == "wr_1"
+ assert unresolved.block_label == "sign_in_and_read"
+ assert "wr_1" in result.user_response
+ assert "sign_in_and_read" in result.user_response
+ assert "Built it. The workflow reads the visitor count." in result.user_response
+
+
+def test_the_qualified_turn_is_still_a_success() -> None:
+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])
+
+ result = _result(
+ ctx,
+ user_response="Built it.",
+ turn_outcome=_outcome(ResponseKind.BUILD),
+ narrative_payload=_payload(),
+ )
+
+ assert result.turn_outcome is not None
+ assert result.turn_outcome.unresolved_runtime_failure is not None
+ assert result.turn_outcome.response_kind == ResponseKind.BUILD
+ assert result.turn_outcome.terminal_reason is None
+
+
+def test_a_re_exercised_failure_leaves_the_reply_and_outcome_unqualified() -> None:
+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["sign_in_and_read"], final_selector="Continue")
+
+ result = _result(
+ ctx,
+ user_response="Built it.",
+ turn_outcome=_outcome(ResponseKind.BUILD),
+ narrative_payload=_payload(),
+ )
+
+ assert result.turn_outcome is not None
+ assert result.turn_outcome.unresolved_runtime_failure is None
+ assert result.user_response == "Built it."
+
+
+def test_a_clarifying_turn_is_never_qualified() -> None:
+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])
+
+ result = _result(
+ ctx,
+ user_response="Which metric did you want?",
+ turn_outcome=_outcome(ResponseKind.CLARIFY),
+ narrative_payload=_payload(),
+ )
+
+ assert result.turn_outcome is not None
+ assert result.turn_outcome.unresolved_runtime_failure is None
+ assert result.user_response == "Which metric did you want?"
+
+
+def test_the_qualification_also_rides_the_narrative_terminal_message() -> None:
+ """The chat panel renders the narrative card, not the raw reply, so both carry the note."""
+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])
+
+ result = _result(
+ ctx,
+ user_response="Built it.",
+ turn_outcome=_outcome(ResponseKind.BUILD),
+ narrative_payload=_payload(terminalMessage="All done.", narrativeSummary="All done."),
+ )
+
+ assert result.narrative_payload is not None
+ for key in ("terminalMessage", "narrativeSummary"):
+ assert "wr_1" in result.narrative_payload[key], key
+ assert "sign_in_and_read" in result.narrative_payload[key], key
diff --git a/uv.lock b/uv.lock
index 452f7e14f..05b35dbe9 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6151,8 +6151,8 @@ requires-dist = [
{ name = "greenlet", marker = "python_full_version >= '3.12' and python_full_version < '3.14' and extra == 'local'", specifier = ">3.0.3" },
{ name = "greenlet", marker = "python_full_version >= '3.12' and python_full_version < '3.14' and extra == 'server'", specifier = ">3.0.3" },
{ name = "httpx", extras = ["socks"], specifier = ">=0.27.0" },
- { name = "jinja2", marker = "extra == 'local'", specifier = ">=3.1.2,<4" },
- { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.2,<4" },
+ { name = "jinja2", marker = "extra == 'local'", specifier = ">=3.1.6,<4" },
+ { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" },
{ name = "json-repair", marker = "extra == 'local'", specifier = ">=0.60.1,<0.61" },
{ name = "json-repair", marker = "extra == 'server'", specifier = ">=0.60.1,<0.61" },
{ name = "json5", marker = "extra == 'local'", specifier = ">=0.13.0,<1" },