Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Report Copilot test-run failures honestly, and remember ones no later run retried (SKY-13206) (#7873)
Andrew Neilson479da0687833
diff --git a/pyproject.toml b/pyproject.tomlindex f65e7fe32..a19aa9d7e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -33,7 +33,7 @@ local = ["aiohttp>=3.13.4,<4","python-multipart>=0.0.30,<1","toml>=0.10.2,<0.11",- "jinja2>=3.1.2,<4",+ "jinja2>=3.1.6,<4","litellm>=1.84.0","playwright>1.46.0 ; python_version >= '3.12' and python_version < '3.14'","playwright==1.46.0 ; python_version >= '3.11' and python_version < '3.12'",@@ -104,7 +104,7 @@ server = ["aiohttp>=3.13.4,<4","python-multipart>=0.0.30,<1","toml>=0.10.2,<0.11",- "jinja2>=3.1.2,<4",+ "jinja2>=3.1.6,<4","uvicorn[standard]>=0.44.0","litellm>=1.84.0","playwright>1.46.0 ; python_version >= '3.12' and python_version < '3.14'",diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 939344a0b..908d56de4 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -2,6 +2,7 @@ import asyncioimport copyimport dataclassesimport json+import reimport timeimport warningsfrom asyncio import CancelledError@@ -11,7 +12,6 @@ import litellmimport structlogfrom anthropic import NOT_GIVENfrom anthropic.types.beta.beta_message import BetaMessage as AnthropicMessage-from jinja2 import Templatefrom litellm.llms.custom_httpx.http_handler import AsyncHTTPHandlerfrom litellm.types.router import AllowedFailsPolicyfrom litellm.utils import CustomStreamWrapper, ModelResponse@@ -70,6 +70,12 @@ from skyvern.utils.url_validators import validate_fetch_urlconfigure_litellm_transport()LOG = structlog.get_logger()+_HASHED_HREF_PLACEHOLDER = re.compile(r"\{\{\s*(_[0-9a-f]{64})\s*\}\}")+++def _render_hashed_href_map(llm_content: str, hashed_href_map: dict[str, str]) -> str:+ return _HASHED_HREF_PLACEHOLDER.sub(lambda match: hashed_href_map.get(match.group(1), ""), llm_content)+# Transient upstream faults litellm maps to these subclass openai.APIError, NOT# litellm.exceptions.APIError, so a bare `except litellm.exceptions.APIError` misses them.@@ -1856,7 +1862,7 @@ class LLMAPIHandlerFactory:rendered_response_json = Noneif context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)- rendered_content = Template(llm_content).render(context.hashed_href_map)+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:@@ -2414,7 +2420,7 @@ class LLMAPIHandlerFactory:rendered_response_json = Noneif context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)- rendered_content = Template(llm_content).render(context.hashed_href_map)+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:@@ -3017,7 +3023,7 @@ class LLMCaller:rendered_response_json = Noneif context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)- rendered_content = Template(llm_content).render(context.hashed_href_map)+ rendered_content = _render_hashed_href_map(llm_content, context.hashed_href_map)parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 5a5970c70..d901d26d0 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (RecordedOutcomeBindingConstraint,RecordedOutcomeGroundingRequirement,observed_value_extraction_scaffold_lines,+ unresolved_runtime_block_failure,)from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.code_block_synthesis import (@@ -220,7 +221,7 @@ from skyvern.forge.sdk.copilot.turn_outcome import (with_copilot_code_mode_diagnostics,)from skyvern.forge.sdk.copilot.turn_ownership import blocker_signal_render_allowed-from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome, UnresolvedRuntimeFailurefrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_statusfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatHistoryMessage,@@ -1838,6 +1839,16 @@ def _assemble_terminal_envelope_safe(return payload+def _with_unresolved_runtime_failure_note(user_response: str, failure: UnresolvedRuntimeFailure) -> str:+ label = failure.block_label or "an earlier step"+ note = (+ f"One thing to flag: an earlier test run ({failure.workflow_run_id}) failed at "+ f'"{label}", the failing call is still in the draft, and no later run '+ "verifiably re-exercised it — so that step is still unproven."+ )+ return f"{user_response.rstrip()}\n\n{note}" if user_response.strip() else note++def _make_agent_result(ctx: CopilotContext | None,*,@@ -1898,6 +1909,26 @@ def _make_agent_result(payload_updates["outcomeAdjudication"] = adjudicationif payload_updates or len(payload_base) != len(narrative_payload):kwargs["narrative_payload"] = {**payload_base, **payload_updates}+ if ctx is not None and turn_outcome is not None and turn_outcome.response_kind == ResponseKind.BUILD:+ unresolved_failure = unresolved_runtime_block_failure(ctx)+ if unresolved_failure is not None:+ turn_outcome = turn_outcome.model_copy(update={"unresolved_runtime_failure": unresolved_failure})+ kwargs["user_response"] = _with_unresolved_runtime_failure_note(+ str(kwargs.get("user_response") or ""), unresolved_failure+ )+ # A reloaded chat renders the narrative card, and hydration prefers narrativeSummary over+ # terminalMessage, so the qualification has to ride every surface or it survives the+ # turn and disappears on refresh.+ narrative = kwargs.get("narrative_payload")+ if isinstance(narrative, dict):+ kwargs["narrative_payload"] = {+ **narrative,+ **{+ key: _with_unresolved_runtime_failure_note(narrative[key], unresolved_failure)+ for key in ("terminalMessage", "narrativeSummary")+ if isinstance(narrative.get(key), str) and narrative[key].strip()+ },+ }terminal_envelope: dict[str, Any] | None = Noneif ctx is not None:blocker_reason, halt_kind = _terminal_halt_fields(ctx)diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex e734bb77b..77947283d 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -1,5 +1,6 @@from __future__ import annotations+import astimport hashlibimport jsonimport re@@ -24,6 +25,7 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, workflow_target_urlfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.failure_tracking import selector_identities_in_text, selector_identity_from_failurefrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.result_evidence import (LoadedResultCompositionEvidence,@@ -31,6 +33,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (scout_observation_contract_valid,)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, RunOutcomeReasonCode+from skyvern.forge.sdk.schemas.copilot_turn_outcome import UnresolvedRuntimeFailureLOG = structlog.get_logger()@@ -123,6 +126,7 @@ class RecordedBuildTestOutcome(BaseModel):attempted_tool: str = ""attempted_target: str = ""attempted_block_label: str = ""+ attempted_call_ref: str = ""verdict: BuildTestOutcomeVerdictreason_code: BuildTestOutcomeReasonCodeobserved_evidence_summary: str = ""@@ -363,6 +367,10 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re"is_authoritative": outcome.is_authoritative,"workflow_run_id": outcome.workflow_run_id,"authored_structure_signature": outcome.authored_structure_signature,+ "block_labels": list(outcome.block_labels),+ "attempted_block_label": outcome.attempted_block_label,+ "attempted_block_signature": _attempted_block_signature(ctx, outcome),+ "attempted_call_ref": outcome.attempted_call_ref,})del history[:-_HISTORY_LIMIT]@@ -381,6 +389,135 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re)+def _attempted_block_signature(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome) -> str:+ """YAML-derived so it stays comparable later; ``block_shape_hashes`` is built from block models+ and never hashes equal to a YAML-derived signature."""+ if outcome.reason_code != "runtime_block_failure" or not outcome.attempted_block_label:+ return ""+ return authored_block_signatures_from_workflow(ctx.workflow_yaml).get(outcome.attempted_block_label, "")+++def _code_blocks_by_label(workflow_yaml: str | None) -> dict[str, str]:+ """Code text for every code block in the draft, including blocks nested in containers or loops."""+ code_by_label: dict[str, str] = {}+ if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():+ return code_by_label+ parsed = _parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, Mapping):+ return code_by_label++ def walk(value: object) -> None:+ for block in _mapping_list(value):+ label = _safe_str(block.get("label"))+ code = block.get("code")+ if _safe_str(block.get("block_type")).lower() == "code" and label and isinstance(code, str):+ code_by_label[label] = code+ walk(block.get("blocks"))+ walk(block.get("loop_blocks"))++ walk(_dict(parsed.get("workflow_definition")).get("blocks"))+ return code_by_label+++# Every construct that can reach the end of a block without running something inside it, including+# expression-level ones: a zero-length comprehension skips its body exactly as an unentered loop does.+# Over-detection only costs a redundant note; a miss silently drops a real failure.+_SELECTOR_CALL_ATTRS = frozenset({"locator", "get_by_role", "get_by_text", "get_by_label", "get_by_placeholder"})++_CALL_SKIPPING_NODES = (+ ast.If,+ ast.IfExp,+ ast.BoolOp,+ ast.Try,+ ast.TryStar,+ ast.While,+ ast.For,+ ast.AsyncFor,+ ast.comprehension,+ ast.Match,+ ast.Lambda,+ ast.FunctionDef,+ ast.AsyncFunctionDef,+)+++def _parse_block_code(code: str) -> ast.AsyncFunctionDef | None:+ """Block code is an async body, so it only parses inside a wrapper; None means unparseable."""+ try:+ wrapper = ast.parse("async def _block():\n" + textwrap.indent(code, " ")).body[0]+ except Exception:+ # Deeply nested generated code raises MemoryError from the parser rather than SyntaxError,+ # and no caller may propagate a parse failure out of turn assembly.+ return None+ return wrapper if isinstance(wrapper, ast.AsyncFunctionDef) else None+++def _code_can_skip_a_call(code: str) -> bool:+ """Unparseable code counts as branching so an unrecognized shape never credits re-exercise."""+ wrapper = _parse_block_code(code)+ if wrapper is None:+ return True+ return any(isinstance(node, _CALL_SKIPPING_NODES) for stmt in wrapper.body for node in ast.walk(stmt))+++def _selector_removal_is_provable(code: str) -> bool:+ """Whether every selector in the block is a literal, so a scan that misses one proves removal.++ A selector built at runtime (an f-string, a variable, a concatenation) is invisible to the+ literal scan, so its absence would otherwise read as the failing call having been edited away.+ """+ wrapper = _parse_block_code(code)+ if wrapper is None:+ return False+ for node in ast.walk(wrapper):+ if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):+ continue+ if node.func.attr not in _SELECTOR_CALL_ATTRS:+ continue+ selector_args = [*node.args, *(kw.value for kw in node.keywords if kw.arg == "name")]+ if any(not isinstance(arg, ast.Constant) or not isinstance(arg.value, str) for arg in selector_args):+ return False+ return True+++def unresolved_runtime_block_failure(ctx: _RecordedBuildTestOutcomeContext) -> UnresolvedRuntimeFailure | None:+ """The newest runtime block failure that no later run verifiably re-exercised or edited away."""+ history = ctx.recorded_build_test_outcome_history+ for index in range(len(history) - 1, -1, -1):+ entry = history[index]+ if entry.get("reason_code") != "runtime_block_failure":+ continue+ label = _safe_str(entry.get("attempted_block_label"))+ signature = _safe_str(entry.get("attempted_block_signature"))+ run_id = _safe_str(entry.get("workflow_run_id"))+ call_ref = _safe_str(entry.get("attempted_call_ref"))+ # Scout evaluations and author-time rejects share this history, so a later *run* has to be+ # identified by phase and a different run id -- otherwise author-time work after the failure+ # would read as a later run that skipped the block.+ later_runs = [+ entry_after+ for entry_after in history[index + 1 :]+ if entry_after.get("phase") == "persisted_block_run"+ and _safe_str(entry_after.get("workflow_run_id")) not in ("", run_id)+ ]+ code = _code_blocks_by_label(ctx.workflow_yaml).get(label)+ if not (label and run_id and later_runs and code):+ return None+ # Executing the block re-exercises the failing call only when the block runs straight+ # through; a branch inside it can reach the end without reaching the call.+ if not _code_can_skip_a_call(code):+ for later_run in later_runs:+ if label in _string_list(later_run.get("block_labels")):+ return None+ if call_ref:+ if call_ref not in selector_identities_in_text(code) and _selector_removal_is_provable(code):+ return None+ elif signature and authored_block_signatures_from_workflow(ctx.workflow_yaml).get(label) != signature:+ return None+ return UnresolvedRuntimeFailure(workflow_run_id=run_id, block_label=label)+ return None++def bind_post_run_page_path_failure(ctx: _RecordedBuildTestOutcomeContext,page_evidence: Mapping[str, object],@@ -1261,6 +1398,9 @@ def recorded_outcome_from_run_blocks_result(phase="persisted_block_run",attempted_tool="update_and_run_blocks",attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",+ attempted_call_ref=selector_identity_from_failure(+ _safe_str(failed_block.get("failure_reason")) if failed_block is not None else ""+ ),verdict=verdict,reason_code=reason_code,workflow_run_id=workflow_run_id or None,diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex 59959db65..47dac121f 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -167,6 +167,28 @@ def _selector_from_text(text: str) -> tuple[str, str]:return "", ""+_CSS_SELECTOR_KIND = "locator"+++def selector_identity_from_failure(text: str) -> str:+ kind, value = _selector_from_text(text or "")+ # "selector" and "locator" are the same CSS string under two spellings; the code side only ever+ # spells it "locator", so a failure that named it "selector" must normalize or never match.+ return f"{_CSS_SELECTOR_KIND if kind == 'selector' else kind}:{value}" if kind else ""+++def selector_identities_in_text(text: str) -> set[str]:+ """Every locator identity in ``text``, normalized to match ``selector_identity_from_failure``."""+ identities: set[str] = set()+ for match in _LOCATOR_RE.finditer(text or ""):+ identities.add(f"{_CSS_SELECTOR_KIND}:{_normalize_signature_text(match.group('selector'))}")+ for match in _ROLE_RE.finditer(text or ""):+ role = _normalize_signature_text(match.group("role"))+ name = _normalize_signature_text(match.group("name") or "")+ identities.add(f"role:{role}:{name}" if name else f"role:{role}")+ return identities++def _error_class_from_text(text: str) -> str:code_block_imposition = _code_block_imposition_error_class(text)if code_block_imposition:diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 3831e6645..0bf40dfab 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -1914,7 +1914,16 @@ def _tool_visible_result_after_completion_verification(return resultcopied_data = dict(data) if isinstance(data, dict) else {}- copied_data["failure_reason"] = outcome_unverified_reason+ # Every outcome-unverified reason asserts a run that completed, so substituting it for a raised+ # block's own reason told the agent a failed run had completed and sent repair at the extraction+ # instead of at the failing call. Append it as subordinate context instead.+ run_failure_reason = next(iter_failure_reasons(result), "")+ agent_facing_reason = (+ f"{run_failure_reason.rstrip('. ')}. Completion verification also reported: {outcome_unverified_reason}"+ if run_failure_reason+ else outcome_unverified_reason+ )+ copied_data["failure_reason"] = agent_facing_reasoncopied_data["completion_verification"] = (completion_verification.to_trace_data() if completion_verification is not None else None)@@ -1932,7 +1941,7 @@ def _tool_visible_result_after_completion_verification(return {**result,"ok": False,- "error": outcome_unverified_reason,+ "error": agent_facing_reason,"data": copied_data,}diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 44ee5d34d..affe23e98 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -2642,7 +2642,11 @@ def _record_run_blocks_result(copilot_ctx.last_artifact_health_blocker_labels = []copilot_ctx.last_artifact_health_failure_classes = []if completion_verification is not None and completion_verification.status == "evaluated":- copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)+ # Every unverified-outcome reason asserts a run that completed, so a run that+ # raised has no outcome-gate reason and must report its own failure instead.+ copilot_ctx.last_outcome_gate_reason = (+ _outcome_unverified_reason(copilot_ctx, completion_verification) if run_ok else None+ )copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_idcopilot_ctx.last_test_suspicious_success = Falseif prior_committed_outcome is None:diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.pyindex af3d7dab9..be0f667f2 100644--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py@@ -7,10 +7,11 @@ in any ``copilot/`` business logic — derivation lives infrom __future__ import annotations+import refrom enum import StrEnumfrom typing import Any, Literal-from pydantic import BaseModel, ConfigDict, Field+from pydantic import BaseModel, ConfigDict, Field, field_validatorclass ResponseKind(StrEnum):@@ -21,6 +22,26 @@ class ResponseKind(StrEnum):RECOVER = "recover"+_UNSAFE_IDENTIFIER_RE = re.compile(r"[^A-Za-z0-9_\- ]")+++class UnresolvedRuntimeFailure(BaseModel):+ """A failure a successful turn could not clear, recorded so the outcome is gradeable after the+ fact; the reply text derives independently, and nothing keys success or verification on this."""++ model_config = ConfigDict(extra="ignore", frozen=True)++ workflow_run_id: str+ block_label: str++ @field_validator("workflow_run_id", "block_label")+ @classmethod+ def _bare_identifier(cls, value: str) -> str:+ # Both fields are model-authored and reach the chat reply and the history API, so they are+ # reduced to bounded identifiers here rather than at each surface that renders them.+ return _UNSAFE_IDENTIFIER_RE.sub("", value)[:80].strip()++class TurnOutcome(BaseModel):# extra="ignore" so a rolling deploy that adds a new TurnOutcome field# does not make older readers silently treat freshly-written rows as None.@@ -39,3 +60,4 @@ class TurnOutcome(BaseModel):copilot_last_code_build_failed: bool = Falsecopilot_pending_capability: str | None = Nonecopilot_turn_id: str | None = None+ unresolved_runtime_failure: UnresolvedRuntimeFailure | None = Nonediff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.pyindex 43cd4198a..01724b7fe 100644--- a/tests/unit/copilot_test_helpers.py+++ b/tests/unit/copilot_test_helpers.py@@ -10,6 +10,7 @@ from unittest.mock import MagicMockimport pytestfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisInput,@@ -209,3 +210,63 @@ def make_completion_criterion(requested_output_floor_rekeyed=requested_output_floor_rekeyed,floor_rekeyed_from_path=floor_rekeyed_from_path,)+++def two_page_login_yaml(*, submit_selector: str = "Login") -> str:+ """The shape copilot emits in code-block mode: branch bodies are code inside one always-executed+ block, so a passing run can traverse it without reaching the guarded call."""+ return f"""+ title: Sign in and read the metric+ workflow_definition:+ blocks:+ - block_type: code+ label: sign_in_and_read+ code: |+ await page.fill("#user", "demo")+ await page.click("#submit")+ if await page.locator("#token").count():+ await page.get_by_role("button", name="{submit_selector}", exact=True).click()+ return {{"visitors": "9.42K"}}+ """+++def straight_line_login_yaml() -> str:+ """One always-executed code block with no branching: executing it reaches every call in it."""+ return """+ title: Sign in and read the metric+ workflow_definition:+ blocks:+ - block_type: code+ label: sign_in_and_read+ code: |+ await page.fill("#user", "demo")+ await page.get_by_role("button", name="Login", exact=True).click()+ return {"visitors": "9.42K"}+ """+++def failed_second_factor_run(run_id: str) -> RecordedBuildTestOutcome:+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ attempted_block_label="sign_in_and_read",+ attempted_call_ref="role:button:Login",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id=run_id,+ block_labels=["sign_in_and_read"],+ structural_failure_identity="locator-timeout-identity",+ )+++def passing_run(run_id: str, block_labels: list[str]) -> RecordedBuildTestOutcome:+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="progress_observed",+ reason_code="run_completed_unevaluated",+ workflow_run_id=run_id,+ block_labels=block_labels,+ structural_failure_identity="",+ evidence_refs=["rows:1"],+ )diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex b5b6419ab..b07817cc0 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom asyncio import CancelledErrorfrom datetime import datetime+from pathlib import Pathfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -27,6 +28,94 @@ def _custom_llm_config(model_name: str, api_base: str = "https://llm.example.tesreturn LLMConfig(model_name, [], False, False, {"api_key": "test-key", "api_base": api_base})+def test_render_hashed_href_map_blocks_lipsum_globals_command(tmp_path: Path) -> None:+ marker = tmp_path / "ssti_canary"+ payload = "{{ lipsum.__globals__['os'].popen('touch " + str(marker) + "').read() }}"++ result = api_handler_factory._render_hashed_href_map(payload, {})++ assert not marker.exists()+ assert result == payload+++def test_render_hashed_href_map_blocks_string_class_subclasses_gadget(tmp_path: Path) -> None:+ marker = tmp_path / "ssti_canary"+ payload = (+ "{% for cls in ''.__class__.__mro__[1].__subclasses__() %}"+ "{% if cls.__name__ == 'catch_warnings' %}"+ f"{{{{ cls()._module.__builtins__['__import__']('os').system('touch {marker}') }}}}"+ "{% endif %}{% endfor %}"+ )++ result = api_handler_factory._render_hashed_href_map(payload, {})++ assert not marker.exists()+ assert result == payload+++def test_render_hashed_href_map_blocks_cycler_globals_gadget(tmp_path: Path) -> None:+ marker = tmp_path / "ssti_canary"+ payload = f"{{{{ cycler.__init__.__globals__['os'].popen('touch {marker}').read() }}}}"++ result = api_handler_factory._render_hashed_href_map(payload, {})++ assert not marker.exists()+ assert result == payload+++def test_render_hashed_href_map_replaces_only_hashed_url_placeholders_with_optional_spaces() -> None:+ first_key = f"_{'a' * 64}"+ second_key = f"_{'b' * 64}"+ first_url = "https://example.test/a/very/long/path?with=query&and=more-query-parameters"+ second_url = "https://example.test/another/path"+ hashed_href_map = {first_key: first_url, second_key: second_url}+ payload = (+ '{"message":"Use the links without changing ordinary prose.",'+ f'"links":["{{{{{first_key}}}}}","{{{{ {second_key} }}}}","{{{{{first_key}}}}}"],'+ '"template":"{{ unknown_key }}"}'+ )+ expected = (+ '{"message":"Use the links without changing ordinary prose.",'+ f'"links":["{first_url}","{second_url}","{first_url}"],'+ '"template":"{{ unknown_key }}"}'+ )++ assert api_handler_factory._render_hashed_href_map(f"{{{{{first_key}}}}}", hashed_href_map) == first_url+ assert api_handler_factory._render_hashed_href_map(payload, hashed_href_map) == expected+++def test_render_hashed_href_map_removes_unknown_hash_and_preserves_other_placeholder() -> None:+ unknown_hash = f"_{'c' * 64}"+ other_placeholder = "ordinary prose with {{ unknown_key }} left intact"++ assert api_handler_factory._render_hashed_href_map(f"{{{{{unknown_hash}}}}}", {}) == ""+ assert api_handler_factory._render_hashed_href_map(other_placeholder, {}) == other_placeholder+++def test_render_hashed_href_map_returns_malformed_template_unchanged() -> None:+ payload = "response with stray {{"++ result = api_handler_factory._render_hashed_href_map(payload, {})++ assert result == payload+++def test_render_hashed_href_map_returns_self_calling_macro_unchanged() -> None:+ payload = "{% macro m() %}{{ m() }}{% endmacro %}{{ m() }}"++ result = api_handler_factory._render_hashed_href_map(payload, {})++ assert result == payload+++def test_render_hashed_href_map_returns_expensive_templates_unchanged() -> None:+ range_payload = "{% for i in range(10000000) %}x{% endfor %}"+ multiplication_payload = "{{ 'a' * 1000000000 }}"++ assert api_handler_factory._render_hashed_href_map(range_payload, {}) == range_payload+ assert api_handler_factory._render_hashed_href_map(multiplication_payload, {}) == multiplication_payload++@pytest.mark.parametrize(("model_name", "http_client_attribute"),[("openai/example-model", "_client"), ("ollama_chat/example-model", "client")],diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex 21433c602..f341949e4 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -22,12 +22,20 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (recorded_outcome_from_run_blocks_result,recorded_outcome_from_scout_act_observe_hollow,run_backed_repair_evidence_exists,+ unresolved_runtime_block_failure,)from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.failure_tracking import selector_identity_from_failurefrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, LoadedResultCompositionTargetfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+from tests.unit.copilot_test_helpers import (+ failed_second_factor_run,+ passing_run,+ straight_line_login_yaml,+ two_page_login_yaml,+)def _metadata_reject_ladder_input(*, structural_key: str = "reject-key") -> MetadataRejectLadderInput:@@ -1512,3 +1520,230 @@ def test_clearing_the_grounding_requirement_also_drops_the_binding_constraint()assert ctx.recorded_outcome_grounding_requirement is Noneassert ctx.recorded_outcome_binding_constraint is None+++def _run_history_ctx(workflow_yaml: str) -> SimpleNamespace:+ return SimpleNamespace(+ workflow_yaml=workflow_yaml,+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )+++def test_failure_stays_open_when_a_later_run_passes_without_taking_the_failed_branch() -> None:+ """The live shape: the block runs to completion while its internal branch skips the failing call."""+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.workflow_run_id == "wr_1"+ assert open_failure.block_label == "sign_in_and_read"+++def test_a_straight_line_block_a_later_run_executed_is_re_exercised() -> None:+ """No branch can skip a call, so executing the block proves the failing call ran again."""+ ctx = _run_history_ctx(straight_line_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))++ assert unresolved_runtime_block_failure(ctx) is None+++def test_failure_retires_once_the_implicated_call_leaves_the_draft() -> None:+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))+ ctx.workflow_yaml = two_page_login_yaml(submit_selector="Continue")++ assert unresolved_runtime_block_failure(ctx) is None+++def test_an_unrelated_edit_to_the_block_does_not_retire_the_failing_call() -> None:+ """The call-level check earns its place here: the block changed, the failing call did not."""+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))+ ctx.workflow_yaml = two_page_login_yaml().replace(+ 'return {"visitors": "9.42K"}', 'return {"visitors": "9.42K", "extra": 1}'+ )++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.block_label == "sign_in_and_read"+++def test_a_failure_with_no_later_run_is_the_turns_own_headline_not_an_unresolved_note() -> None:+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))++ assert unresolved_runtime_block_failure(ctx) is None+++def test_another_record_for_the_same_run_id_is_not_a_later_run() -> None:+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_1", ["sign_in_and_read"]))++ assert unresolved_runtime_block_failure(ctx) is None+++def test_author_time_work_after_the_failure_is_not_a_later_run() -> None:+ """Scout evaluations and author-time rejects share this history but execute nothing."""+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="author_time_reject",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="synthesized_parameter_binding_ambiguous",+ structural_failure_identity="author-time-identity",+ ),+ )++ assert unresolved_runtime_block_failure(ctx) is None+++def test_a_later_run_still_arms_the_note_when_author_time_work_follows_it() -> None:+ ctx = _run_history_ctx(two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["read_metric"]))+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="scout_evaluate",+ attempted_tool="evaluate",+ verdict="progress_observed",+ reason_code="run_completed_unevaluated",+ structural_failure_identity="scout-identity",+ ),+ )++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.workflow_run_id == "wr_1"+++def test_a_code_block_nested_in_a_loop_is_still_found() -> None:+ """A failure inside a loop body must stay reportable, not vanish because the walk missed it."""+ ctx = _run_history_ctx(+ """+ title: Loop over rows+ workflow_definition:+ blocks:+ - block_type: for_loop+ label: each_row+ loop_blocks:+ - block_type: code+ label: sign_in_and_read+ code: |+ if await page.locator("#token").count():+ await page.get_by_role("button", name="Login", exact=True).click()+ """+ )+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["each_row"]))++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.block_label == "sign_in_and_read"+++def test_a_css_selector_failure_matches_the_locator_call_in_the_draft() -> None:+ """A failure that spells it "selector:" must still match code that spells it locator()."""+ ctx = _run_history_ctx(+ """+ title: Read the metric+ workflow_definition:+ blocks:+ - block_type: code+ label: sign_in_and_read+ code: |+ if await page.locator("#token").count():+ await page.locator("#submit-btn").click()+ """+ )+ # Derived, not hardcoded: the point of the test is that the two spellings normalize together.+ call_ref = selector_identity_from_failure('TimeoutError: resolved selector: "#submit-btn" never appeared')+ assert call_ref == "locator:#submit-btn"+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": call_ref})+ record_build_test_outcome(ctx, outcome)+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.block_label == "sign_in_and_read"+++def test_run_blocks_outcome_extracts_the_implicated_call_from_the_failure_text() -> None:+ outcome = recorded_outcome_from_run_blocks_result(+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "failure_type": "runtime_error",+ "blocks": [+ {+ "label": "get_visitors",+ "block_type": "code",+ "status": "failed",+ "failure_reason": (+ "code block failed. failure reason: Failed to execute code block. Reason: "+ "TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for "+ 'get_by_role("button", name="Continue", exact=True)'+ ),+ },+ ],+ },+ },+ )++ assert outcome is not None+ assert outcome.attempted_call_ref == "role:button:Continue"+++def _templated_selector_yaml(*, templated: bool = True) -> str:+ call = 'f"#submit-{kind}"' if templated else '"#submit-btn"'+ return f"""+ title: Read the metric+ workflow_definition:+ blocks:+ - block_type: code+ label: sign_in_and_read+ code: |+ if await page.locator("#token").count():+ await page.locator({call}).click()+ """+++def test_a_templated_selector_does_not_read_as_the_call_being_removed() -> None:+ """A runtime-built selector is invisible to the literal scan, so it cannot prove removal."""+ ctx = _run_history_ctx(_templated_selector_yaml())+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": "locator:#submit-btn"})+ record_build_test_outcome(ctx, outcome)+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))++ open_failure = unresolved_runtime_block_failure(ctx)++ assert open_failure is not None+ assert open_failure.block_label == "sign_in_and_read"+++def test_an_all_literal_block_still_retires_when_the_call_is_gone() -> None:+ """The pass path: with every selector a literal, absence really is proof of removal."""+ ctx = _run_history_ctx(_templated_selector_yaml(templated=False))+ outcome = failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": "locator:#gone-btn"})+ record_build_test_outcome(ctx, outcome)+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))++ assert unresolved_runtime_block_failure(ctx) is Nonediff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 1fafbcd01..d98570ff6 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -21,6 +21,10 @@ from skyvern.forge.sdk.copilot.agent import (_rewrite_failed_test_response,_verified_workflow_or_none,)+from skyvern.forge.sdk.copilot.blocker_signal import (+ compose_loop_blocker_user_facing_reason,+ loop_blocker_evidence_from_ctx,+)from skyvern.forge.sdk.copilot.completion_criteria_store import (StoredCriteriaSet,StoredCriteriaSnapshot,@@ -3289,6 +3293,25 @@ def test_tool_visible_result_fails_when_confirmation_block_outcome_unmet() -> Noassert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"+def test_tool_visible_result_leads_with_the_raised_blocks_own_reason() -> None:+ """Live shape: the block raised, but the agent was told the run completed, so it repaired the+ extraction instead of the failing call."""+ ctx = _ctx_with_blocks("extraction")+ result = _failed_code_block_result()+ result["data"]["failure_reason"] = (+ "code block failed. failure reason: Failed to execute code block. "+ "Reason: Exception: Could not parse visitor total from the page"+ )++ visible = _tool_visible_result_after_completion_verification(ctx, result, _evaluated(("c0", False)))++ assert not visible["error"].startswith("The run completed")+ assert "Could not parse visitor total" in visible["error"]+ assert "Could not parse visitor total" in visible["data"]["failure_reason"]+ assert "item in cart" in visible["error"]+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"++def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:ctx = _ctx_with_blocks("goto_url", "navigation")@@ -4724,12 +4747,26 @@ def _failed_code_block_result() -> dict:}-def test_failed_run_records_gate_reason_separately_from_raw_block_failure() -> None:+def test_failed_run_leaves_no_outcome_unverified_reason_and_reports_the_block_failure() -> None:ctx = _ctx_with_blocks("extraction")_record_run_blocks_result(ctx, _failed_code_block_result(), completion_verification=_evaluated(("c0", False)))- assert "item in cart" in (ctx.last_outcome_gate_reason or "")- assert "TimeoutError" not in (ctx.last_outcome_gate_reason or "")+ assert ctx.last_outcome_gate_reason is None+ assert ctx.last_outcome_gate_workflow_run_id == "wr_x"assert "TimeoutError" in (ctx.last_test_failure_reason or "")+ composed, _ = compose_loop_blocker_user_facing_reason("loop_detected_generic", loop_blocker_evidence_from_ctx(ctx))+ assert "The run completed" not in composed+++def test_ok_run_still_records_the_outcome_unverified_reason() -> None:+ ctx = _ctx_with_blocks("extraction")+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))+ assert "item in cart" in (ctx.last_outcome_gate_reason or "")+ assert ctx.last_outcome_gate_workflow_run_id == "wr_x"+ composed, tiers = compose_loop_blocker_user_facing_reason(+ "loop_detected_generic", loop_blocker_evidence_from_ctx(ctx)+ )+ assert "The run completed" in composed+ assert "verdict" in tiersdef test_gate_reason_survives_a_later_run_without_verification() -> None:diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.pyindex 7f6c1a00d..4e2c14eac 100644--- a/tests/unit/test_copilot_make_agent_result_backfill.py+++ b/tests/unit/test_copilot_make_agent_result_backfill.py@@ -10,6 +10,7 @@ import pytestfrom skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_resultfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+from skyvern.forge.sdk.copilot.build_test_outcome import record_build_test_outcomefrom skyvern.forge.sdk.copilot.context import (AgentResult,CopilotContext,@@ -17,8 +18,9 @@ from skyvern.forge.sdk.copilot.context import ()from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome+from tests.unit.copilot_test_helpers import failed_second_factor_runfrom tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx-from tests.unit.copilot_test_helpers import make_verified_goal_contract+from tests.unit.copilot_test_helpers import make_verified_goal_contract, passing_run, two_page_login_yamldef _verified_goal_ctx() -> CopilotContext:@@ -44,7 +46,7 @@ def _payload(**overrides: object) -> dict:"blocks": [],"terminal": "response","terminalMessage": "done",- "narrativeSummary": None,+ "narrativeSummary": "Built it.","priorBlockCount": None,"designActivity": [],"startedAt": None,@@ -175,3 +177,96 @@ def test_make_agent_result_records_resolved_credentials_as_durable_approval() ->approved = StructuredContext.from_json_str(result.global_llm_context).approved_credentialsassert [record.credential_id for record in approved] == ["cred_portal"]+++def _ctx_with_open_second_factor_failure(+ *, later_run_labels: list[str], final_selector: str = "Login"+) -> CopilotContext:+ ctx = _ctx(workflow_yaml=two_page_login_yaml())+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", later_run_labels))+ ctx.workflow_yaml = two_page_login_yaml(submit_selector=final_selector)+ return ctx+++def test_build_turn_reports_the_failure_no_later_run_re_exercised() -> None:+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])++ result = _result(+ ctx,+ user_response="Built it. The workflow reads the visitor count.",+ turn_outcome=_outcome(ResponseKind.BUILD),+ narrative_payload=_payload(),+ )++ assert result.turn_outcome is not None+ unresolved = result.turn_outcome.unresolved_runtime_failure+ assert unresolved is not None+ assert unresolved.workflow_run_id == "wr_1"+ assert unresolved.block_label == "sign_in_and_read"+ assert "wr_1" in result.user_response+ assert "sign_in_and_read" in result.user_response+ assert "Built it. The workflow reads the visitor count." in result.user_response+++def test_the_qualified_turn_is_still_a_success() -> None:+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])++ result = _result(+ ctx,+ user_response="Built it.",+ turn_outcome=_outcome(ResponseKind.BUILD),+ narrative_payload=_payload(),+ )++ assert result.turn_outcome is not None+ assert result.turn_outcome.unresolved_runtime_failure is not None+ assert result.turn_outcome.response_kind == ResponseKind.BUILD+ assert result.turn_outcome.terminal_reason is None+++def test_a_re_exercised_failure_leaves_the_reply_and_outcome_unqualified() -> None:+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["sign_in_and_read"], final_selector="Continue")++ result = _result(+ ctx,+ user_response="Built it.",+ turn_outcome=_outcome(ResponseKind.BUILD),+ narrative_payload=_payload(),+ )++ assert result.turn_outcome is not None+ assert result.turn_outcome.unresolved_runtime_failure is None+ assert result.user_response == "Built it."+++def test_a_clarifying_turn_is_never_qualified() -> None:+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])++ result = _result(+ ctx,+ user_response="Which metric did you want?",+ turn_outcome=_outcome(ResponseKind.CLARIFY),+ narrative_payload=_payload(),+ )++ assert result.turn_outcome is not None+ assert result.turn_outcome.unresolved_runtime_failure is None+ assert result.user_response == "Which metric did you want?"+++def test_the_qualification_also_rides_the_narrative_terminal_message() -> None:+ """The chat panel renders the narrative card, not the raw reply, so both carry the note."""+ ctx = _ctx_with_open_second_factor_failure(later_run_labels=["read_metric"])++ result = _result(+ ctx,+ user_response="Built it.",+ turn_outcome=_outcome(ResponseKind.BUILD),+ narrative_payload=_payload(terminalMessage="All done.", narrativeSummary="All done."),+ )++ assert result.narrative_payload is not None+ for key in ("terminalMessage", "narrativeSummary"):+ assert "wr_1" in result.narrative_payload[key], key+ assert "sign_in_and_read" in result.narrative_payload[key], keydiff --git a/uv.lock b/uv.lockindex 452f7e14f..05b35dbe9 100644--- a/uv.lock+++ b/uv.lock@@ -6151,8 +6151,8 @@ requires-dist = [{ name = "greenlet", marker = "python_full_version >= '3.12' and python_full_version < '3.14' and extra == 'local'", specifier = ">3.0.3" },{ name = "greenlet", marker = "python_full_version >= '3.12' and python_full_version < '3.14' and extra == 'server'", specifier = ">3.0.3" },{ name = "httpx", extras = ["socks"], specifier = ">=0.27.0" },- { name = "jinja2", marker = "extra == 'local'", specifier = ">=3.1.2,<4" },- { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.2,<4" },+ { name = "jinja2", marker = "extra == 'local'", specifier = ">=3.1.6,<4" },+ { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" },{ name = "json-repair", marker = "extra == 'local'", specifier = ">=0.60.1,<0.61" },{ name = "json-repair", marker = "extra == 'server'", specifier = ">=0.60.1,<0.61" },{ name = "json5", marker = "extra == 'local'", specifier = ">=0.13.0,<1" },