Skip to content

Automate browser based workflows with AI

Copilot: stop discarding saved drafts for reasons the save path only steers on (SKY-13125) (#7794)

Andrew Neilson42e208a7bac1
diff --git a/skyvern/cli/mcp_tools/_element_state.py b/skyvern/cli/mcp_tools/_element_state.py
index b57a4ea0e..a0f4d5187 100644
--- a/skyvern/cli/mcp_tools/_element_state.py
+++ b/skyvern/cli/mcp_tools/_element_state.py
@@ -11,7 +11,10 @@ DEFAULT_DIRECT_ACTION_TIMEOUT_MS = 5000
DIRECT_ACTION_TIMEOUT_ENV = "SKYVERN_MCP_DIRECT_TIMEOUT_MS"
MIN_ACTION_TIMEOUT_MS = 1000
MAX_ACTION_TIMEOUT_MS = 60000
-ELEMENT_STATE_PROBE_TIMEOUT_MS = 1000
+# The probe reports why an action failed, and every timeout here degrades to the useless "unknown"
+# state. It only runs after an action already failed, so a wider budget costs nothing on the happy
+# path and buys an accurate reason on the path that needs one.
+ELEMENT_STATE_PROBE_TIMEOUT_MS = 3000
ACTION_TIMEOUT_DESCRIPTION = (
"Max time to wait for the element in ms. "
@@ -24,7 +27,9 @@ _STATE_ERRORS: dict[ElementState, tuple[str, str, str]] = {
"not_found": (
ErrorCode.SELECTOR_NOT_FOUND,
"Selector did not match any element before the direct action timeout",
- "Verify the selector matches an element on the page, or wait for the element to render before retrying.",
+ "The page may have re-rendered since it was inspected, which invalidates a selector captured "
+ "earlier. Inspect the page again and act on a freshly reported selector rather than retrying "
+ "this one.",
),
"hidden": (
ErrorCode.ACTION_FAILED,
@@ -44,7 +49,8 @@ _STATE_ERRORS: dict[ElementState, tuple[str, str, str]] = {
"unknown": (
ErrorCode.ACTION_FAILED,
"Direct action failed before the element became actionable",
- "Re-check the selector and page state, or use an intent-based action if the target is dynamic.",
+ "The element may not have settled, or the page may have re-rendered since it was inspected. "
+ "Inspect the page again and act on a freshly reported selector.",
),
}
@@ -75,6 +81,21 @@ def is_pointer_interception_error(exc: Exception) -> bool:
return "intercepts pointer events" in message or "intercepted by another element" in message
+async def _click_point_is_covered(locator: Any) -> bool:
+ """True when a different element sits at the target's click point, i.e. the click cannot land."""
+ try:
+ return bool(
+ await locator.evaluate(
+ "el => { const r = el.getBoundingClientRect();"
+ " if (!r.width || !r.height) return false;"
+ " const top = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2);"
+ " return !!top && top !== el && !el.contains(top); }"
+ )
+ )
+ except Exception:
+ return False
+
+
async def classify_element_state(page: Any, selector: str, *, pointer_intercepted: bool = False) -> ElementState:
# Frame-aware actions resolve against SkyvernPage._locator_scope (working iframe if set);
# the probe must query the same root or an iframe failure misclassifies as not_found.
@@ -93,6 +114,10 @@ async def classify_element_state(page: Any, selector: str, *, pointer_intercepte
return "disabled"
if pointer_intercepted:
return "occluded"
+ # Playwright reports an occluded target as a plain action timeout, so the caller cannot
+ # flag interception. Ask the page what actually sits at the click point instead.
+ if await _click_point_is_covered(first):
+ return "occluded"
return "unknown"
try:
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index 1cd058b48..cf46aaeab 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -265,6 +265,22 @@ def _must_reject_localhost_url(ctx: Any, url: str | None) -> bool:
return bool(url and is_localhost_url(url) and getattr(ctx, "can_access_localhost", None) is False)
+def _log_direct_failure_diagnostics(action: str, page: Any, selector: str, exc: Exception, error: dict) -> None:
+ """Record why a direct action failed, pairing the classified element state with the page it was
+ attempted on. Adds no protocol round-trips: the state comes from the error the caller just
+ built, and page.url is a cached property that survives a page whose protocol calls time out.
+ """
+ target = page.page if hasattr(page, "page") else page
+ LOG.info(
+ "direct_action_failure_diagnostics",
+ action=action,
+ selector=selector[:160],
+ page_url=str(getattr(target, "url", None))[:200],
+ element_state=(error.get("details") or {}).get("element_state"),
+ playwright_error=str(exc)[:300],
+ )
+
+
async def _direct_failure_result(
action: str,
ctx: Any,
@@ -278,6 +294,8 @@ async def _direct_failure_result(
value: str | None = None,
key: str | None = None,
) -> dict[str, Any]:
+ error = await make_direct_action_error(page, selector, exc, timeout_ms=timeout_ms)
+ _log_direct_failure_diagnostics(action, page, selector, exc, error)
return _action_result_factory(
ctx=ctx,
page=page,
@@ -290,7 +308,7 @@ async def _direct_failure_result(
ok=False,
browser_context=ctx,
timing_ms=timer.timing_ms,
- error=await make_direct_action_error(page, selector, exc, timeout_ms=timeout_ms),
+ error=error,
)
diff --git a/skyvern/config.py b/skyvern/config.py
index bfad34221..5f9e3d410 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -215,7 +215,6 @@ class Settings(BaseSettings):
# Off = standard block authoring. On = prefer code blocks for browser work.
WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = False
WORKFLOW_COPILOT_TERMINAL_ENVELOPE_RENDER: bool = False
- WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY: bool = False
WORKFLOW_COPILOT_QA_TOKEN_BUDGET: int | None = Field(default=None, gt=0)
# Pause a BUILD turn in place on a typed mid-loop credential ask instead of ending it;
# the FE resumes the same turn via a credential-connect card. Off = today's turn-terminal behavior.
diff --git a/skyvern/constants.py b/skyvern/constants.py
index d129d05c5..bf63cdf16 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -44,8 +44,8 @@ SKIP_INNER_NAV_RETRY_ERRORS = PERMANENT_NAV_ERRORS + PROXY_SENSITIVE_NAV_ERRORS
AUTO_COMPLETION_POTENTIAL_VALUES_COUNT = 3
DROPDOWN_MENU_MAX_DISTANCE = 100
BROWSER_DOWNLOADING_SUFFIX = ".crdownload"
-MAX_UPLOAD_FILE_COUNT = 50
-AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT = 50
+MAX_UPLOAD_FILE_COUNT = 300
+AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT = 300
CUSTOMER_STORAGE_UPLOAD_MAX_BYTES = 1 * 1024 * 1024 * 1024 # 1 GB per file (FileUploadBlock)
DEFAULT_MAX_SCREENSHOT_SCROLLS = 3
# Sanity ceiling for a user-supplied scroll count. Bounds the value well below the
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index ad8690968..7aeef7c1c 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -878,14 +878,6 @@ class AgentFunction:
) -> bool:
return request_override if request_override is not None else settings.MCP_CODE_ONLY_MODE
- async def resolve_copilot_author_time_gate_log_only_ids(
- self,
- *,
- turn_id: str,
- organization_id: str,
- ) -> frozenset[str]:
- return frozenset()
-
async def should_use_codeblock_runner(
self,
*,
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 7383d795e..6200f30b2 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -63,7 +63,7 @@ Use the Skyvern MCP browser tools to gather ground-truth evidence in ANY phase o
- For form/search/result pages, inspect when you do not know the real controls or result layout. Use `inspect_page_for_composition(target_url="current_page")`, `evaluate`, `get_browser_screenshot`, or targeted interaction tools as needed, then compose normal workflow blocks from the observed page.
- Treat each successful run or browser observation as evidence for the current page state only. Do not author downstream row expansion or extraction that depends on a future page state until you have run the next page-changing block and inspected the reached page. If the visible results already contain the requested values, add extraction from that observed page; if an expansion is required first, add and run the expansion block before adding extraction.
- Add `validation` blocks only when the reusable workflow's task actually includes a durable check. Do not add validation or extra navigation blocks just to double-check the shape of a site before each action.
-- If page evidence shows anti-bot or challenge controls, account for that in the normal action block that reaches/submits the challenged state. Use `get_browser_screenshot` when DOM evidence identifies a challenge but does not expose the visible challenge affordance. Prefer explicit challenge handling available to Skyvern over blind waits; if the latest run shows the challenge cannot be solved, stop and report the observed blocker instead of retrying the same flow.
+- If page evidence shows anti-bot or challenge controls, account for that in the normal action block that reaches/submits the challenged state. Use `get_browser_screenshot` whenever seeing the rendered page answers the question faster than reading the DOM — a challenge whose affordance the DOM does not expose is one such case, but so is confirming what an action changed on screen. Prefer explicit challenge handling available to Skyvern over blind waits; if the latest run shows the challenge cannot be solved, stop and report the observed blocker instead of retrying the same flow.
- If bounded page evidence says `challenge_state.gates_submit_controls=true`, the submit/search control is unavailable until the challenge is resolved. If a tested workflow already attempted challenge resolution and current-page evidence still shows that submit/search control disabled with no visible result or no-results state, stop and report the anti-bot blocker rather than repeating the same solve/wait/submit sequence.
- If a state-application block hits a per-tool budget, do not rerun the same navigation label unchanged. First read the budgeted run with `get_run_results`. If it reports a current URL or the browser is still on an advanced page, inspect `current_page`; answer from visible result/no-result evidence when possible, and only split/replace/run a smaller UI block after inspection shows a real missing state change.
- When you already have a verified prefix or a live browser page that advanced beyond the workflow's entrypoint, continue from that evidence. Do not turn the observed current page into a new workflow entrypoint and rerun the whole workflow from the start. Preserve unchanged upstream labels and text, and pass `block_labels` for only the missing downstream frontier.
@@ -73,10 +73,10 @@ CRITICAL RULES WHEN USING DIRECT BROWSER TOOLS:
- NEVER answer questions about page content, link counts, link targets, or page structure from memory or training data. Use evaluate or get_browser_screenshot.
- Use evaluate for reading/inspection; do NOT use evaluate to click or type. Use click and type_text for interactions.
- Direct browser `evaluate` is for scouting only. Persisted workflow `code` blocks must not use `page.evaluate`, `page.evaluate_handle`, `page.request`, or `page.context`; use `page.locator(...)`, `page.get_by_role(...)`, and locator DOM-reading methods such as `inner_text`, `text_content`, `get_attribute`, `count`, and `is_visible`.
-- CSS SELECTORS vs AI INTENT (cost-aware): a CSS-selector-ONLY click/type is an instant deterministic action. Passing an `intent` (alone, or alongside a selector) routes the action through a slower full-page AI scan that scrolls and screenshots the whole page to locate the element — even when the element is already on screen. So once `evaluate`/inspection evidence has identified the target, act by a CSS selector ALONE; reserve `intent` only for when you genuinely cannot derive a selector. When a shared class matches many elements (e.g. one Add-to-Cart per result row), scope the selector to the specific item (its container, a unique attribute, or `:nth-of-type`) rather than passing an `intent` to disambiguate. Do the "which element" reasoning in `evaluate` (read the DOM), not in a click `intent`. jQuery pseudo-selectors (:contains(), :eq(), :first, :visible, :has()) are NOT valid — use standard selectors like 'button.classname', 'a[href*="keyword"]', '#id', 'tr:nth-of-type(2) td a'.
-- SCROLL: Use scroll to reveal content below the fold. Provide direction (up/down/left/right) and optionally amount in pixels, a selector for the scrollable container, or intent to scroll a specific element into view.
-- SELECT_OPTION: Use select_option for <select> dropdowns. Provide the value and use selector or intent to target the element. For free-text inputs, use type_text instead.
-- PRESS_KEY: Use press_key for keyboard actions — Enter to submit forms, Tab to advance focus, Escape to close dialogs. Optionally provide selector or intent to focus an element first.
+- CSS SELECTORS: interactions target elements by CSS selector, which is instant and deterministic. The selectors reported by page inspection are verified to match exactly one element, so prefer copying one of those over composing your own. When a shared class matches many elements (e.g. one Add-to-Cart per result row), scope the selector to the specific item (its container, a unique attribute, or `:nth-of-type`). If a selector does not resolve or the action fails, observe the page again — with `evaluate`, `inspect_page_for_composition`, or `get_browser_screenshot`, whichever answers the question — and derive a better selector. jQuery pseudo-selectors (:contains(), :eq(), :first, :visible, :has()) are NOT valid — use standard selectors like 'button.classname', 'a[href*="keyword"]', '#id', 'tr:nth-of-type(2) td a'.
+- SCROLL: Use scroll to reveal content below the fold. Provide direction (up/down/left/right) and optionally amount in pixels, a selector for the scrollable container, or an intent to scroll a specific element into view.
+- SELECT_OPTION: Use select_option for <select> dropdowns. Provide the value and a selector to target the element. For free-text inputs, use type_text instead.
+- PRESS_KEY: Use press_key for keyboard actions — Enter to submit forms, Tab to advance focus, Escape to close dialogs. Optionally provide a selector to focus an element first.
- CONSOLE_MESSAGES: Use console_messages(level='error') to check for JavaScript errors when diagnosing page issues. This is read-only.
- A FAILED SCOUT CLICK IS NOT A DEAD END: "element is hidden, disabled, or intercepted by another element" usually means an overlay/modal (cookie banner, marketing popup) is covering it — and the click itself often dismisses that overlay. Do NOT abandon the route, navigate away, or redesign the workflow because one scout click failed: re-inspect the current page and retry the same target; the overlay or transient state has typically cleared. (Retrying a click that did NOT act is fine — that is distinct from replaying an earlier add/submit that already succeeded, which would duplicate its effect; see the next rule.)
- A FAILED ACTION MAY HAVE LANDED: if a click/type/submit reports failure but could have changed the page (a navigation, a cart/count change, a new modal), inspect the current page before retrying. Re-firing a non-idempotent action (add to cart, submit) can duplicate its effect — confirm whether it already happened, then decide.
diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py
index 3d982f318..2735b984f 100644
--- a/skyvern/forge/sdk/api/aws.py
+++ b/skyvern/forge/sdk/api/aws.py
@@ -400,8 +400,10 @@ class AsyncAWSClient:
LOG.exception("S3 metadata retrieval failed", uri=uri)
return None
- async def create_presigned_urls(self, uris: list[str]) -> list[str] | None:
+ async def create_presigned_urls(self, uris: list[str], expires_in: int | None = None) -> list[str] | None:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/generate_presigned_url.html
+ expiration = settings.PRESIGNED_URL_EXPIRATION if expires_in is None else expires_in
+
async def _op() -> list[str]:
presigned_urls = []
async with self._s3_client() as client:
@@ -410,7 +412,7 @@ class AsyncAWSClient:
url = await client.generate_presigned_url(
"get_object",
Params={"Bucket": parsed_uri.bucket, "Key": parsed_uri.key},
- ExpiresIn=settings.PRESIGNED_URL_EXPIRATION,
+ ExpiresIn=expiration,
)
presigned_urls.append(url)
return presigned_urls
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py
index 8b2ebf81e..ecdb77b78 100644
--- a/skyvern/forge/sdk/artifact/manager.py
+++ b/skyvern/forge/sdk/artifact/manager.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.api.files import create_named_temporary_file
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.artifact.signing import (
ARTIFACT_URL_EXPIRY_SECONDS,
+ artifact_url_expiry_seconds_for_type,
effective_artifact_url_expiry_seconds,
parse_keyring,
sign_artifact_url,
@@ -1262,7 +1263,12 @@ class ArtifactManager:
storage presigned URL would download the wrong bytes — fall through to
the endpoint, which knows how to extract the member). Otherwise the
storage backend's presigned URL (S3 / Azure SAS / local URI).
+
+ Screenshot and recording URLs are capped to
+ ``SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS`` regardless of the caller's
+ value (SKY-12527).
"""
+ expiry_seconds = artifact_url_expiry_seconds_for_type(artifact.artifact_type, expiry_seconds)
if _bundling_enabled() or artifact.bundle_key:
# Legacy unsigned URLs expose ``artifact_name`` for download display.
# Bundled members carry the in-ZIP filename
@@ -1315,7 +1321,7 @@ class ArtifactManager:
artifact.artifact_id,
artifact_name=artifact.bundle_key,
artifact_type=artifact.artifact_type,
- expiry_seconds=expiry_seconds,
+ expiry_seconds=artifact_url_expiry_seconds_for_type(artifact.artifact_type, expiry_seconds),
)
for artifact in artifacts
]
@@ -1339,7 +1345,7 @@ class ArtifactManager:
a.artifact_id,
artifact_name=a.bundle_key,
artifact_type=a.artifact_type,
- expiry_seconds=expiry_seconds,
+ expiry_seconds=artifact_url_expiry_seconds_for_type(a.artifact_type, expiry_seconds),
)
LOG.debug(
diff --git a/skyvern/forge/sdk/artifact/signing.py b/skyvern/forge/sdk/artifact/signing.py
index cf0fe11e4..15f23e44f 100644
--- a/skyvern/forge/sdk/artifact/signing.py
+++ b/skyvern/forge/sdk/artifact/signing.py
@@ -31,6 +31,8 @@ from urllib.parse import urlencode
from pydantic import BaseModel, model_validator
+from skyvern.forge.sdk.artifact.models import ArtifactType
+
ARTIFACT_URL_EXPIRY_SECONDS = 12 * 60 * 60 # 12 hours — global default when no per-org override.
# TTL for URLs minted at the point of use via GET /artifacts/{id}/signed-url.
@@ -38,6 +40,26 @@ ARTIFACT_URL_EXPIRY_SECONDS = 12 * 60 * 60 # 12 hours — global default when n
# it, so the window only needs to cover one fetch (or one video-buffering burst).
ARTIFACT_URL_ON_DEMAND_EXPIRY_SECONDS = 5 * 60
+# Screenshots and recordings capture whatever the browser rendered — page content,
+# filled form values, logged-in views — so their capability URLs get a tighter
+# window than downloads, which API consumers routinely fetch on their own
+# schedule. 1 hour is the floor `ARTIFACT_URL_EXPIRY_SECONDS_MIN` already
+# guarantees webhook consumers; first-party viewers survive it by re-minting via
+# GET /v1/artifacts/{id}/signed-url.
+SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS = 60 * 60
+
+SENSITIVE_ARTIFACT_TYPES: frozenset[ArtifactType] = frozenset(
+ {
+ ArtifactType.RECORDING,
+ ArtifactType.SESSION_REPLAY,
+ ArtifactType.SCREENSHOT,
+ ArtifactType.SCREENSHOT_LLM,
+ ArtifactType.SCREENSHOT_ACTION,
+ ArtifactType.SCREENSHOT_FINAL,
+ ArtifactType.SCREENSHOT_PROXY,
+ }
+)
+
# Bounds for the per-org override. 1 hour minimum keeps URLs useful for webhook
# consumers that retry across short outages; 7 days maximum follows the AWS S3
# presigned-URL cap so customers used to S3 don't get surprised. The route
@@ -65,6 +87,22 @@ def effective_artifact_url_expiry_seconds(per_org_value: int | None) -> int:
)
+def artifact_url_expiry_seconds_for_type(
+ artifact_type: ArtifactType | None,
+ expiry_seconds: int | None,
+) -> int | None:
+ """Cap a sensitive artifact's URL TTL; pass every other type through unchanged.
+
+ Never lengthens: a caller already asking for a shorter window (the
+ on-demand mint endpoint) keeps it.
+ """
+ if artifact_type not in SENSITIVE_ARTIFACT_TYPES:
+ return expiry_seconds
+ if expiry_seconds is None:
+ return SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
+ return min(expiry_seconds, SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS)
+
+
class HmacKeyEntry(BaseModel):
secret: str
"""HMAC secret as a plain string."""
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.py
index 714e629ae..d4f00ea72 100644
--- a/skyvern/forge/sdk/artifact/storage/azure.py
+++ b/skyvern/forge/sdk/artifact/storage/azure.py
@@ -23,9 +23,11 @@ from skyvern.forge.sdk.api.real_azure import RealAsyncAzureStorageClient
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.artifact.storage.base import (
FILE_EXTENTSION_MAP,
+ SENSITIVE_SHARE_URL_EXPIRY_HOURS,
BaseStorage,
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
+ presign_with_sensitive_cap,
)
from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
from skyvern.forge.sdk.models import Step
@@ -149,11 +151,17 @@ class AzureStorage(BaseStorage):
return await self.async_client.download_file(artifact.uri)
async def get_share_link(self, artifact: Artifact) -> str | None:
- share_urls = await self.async_client.create_sas_urls([artifact.uri])
+ share_urls = await self.get_share_links([artifact])
return share_urls[0] if share_urls else None
async def get_share_links(self, artifacts: list[Artifact]) -> list[str] | None:
- return await self.async_client.create_sas_urls([artifact.uri for artifact in artifacts])
+ return await presign_with_sensitive_cap(
+ artifacts,
+ presign=self.async_client.create_sas_urls,
+ presign_sensitive=lambda uris: self.async_client.create_sas_urls(
+ uris, expiry_hours=SENSITIVE_SHARE_URL_EXPIRY_HOURS
+ ),
+ )
async def store_artifact_from_path(self, artifact: Artifact, path: str) -> None:
tier = await self._get_storage_tier_for_org(artifact.organization_id)
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
index f6246d325..52babd0d3 100644
--- a/skyvern/forge/sdk/artifact/storage/base.py
+++ b/skyvern/forge/sdk/artifact/storage/base.py
@@ -1,14 +1,46 @@
from abc import ABC, abstractmethod
-from typing import BinaryIO
+from collections.abc import Awaitable, Callable
+from typing import BinaryIO, cast
from skyvern.forge import app
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
+from skyvern.forge.sdk.artifact.signing import SENSITIVE_ARTIFACT_TYPES, SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thought
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
+# Hour-granularity equivalent of the sensitive cap for the GCS/Azure signing
+# clients, whose APIs take hours; max() keeps a future sub-hour cap from
+# truncating to zero.
+SENSITIVE_SHARE_URL_EXPIRY_HOURS = max(1, SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS // 3600)
+
+
+async def presign_with_sensitive_cap(
+ artifacts: list[Artifact],
+ presign: Callable[[list[str]], Awaitable[list[str] | None]],
+ presign_sensitive: Callable[[list[str]], Awaitable[list[str] | None]],
+) -> list[str] | None:
+ """Batch-presign artifact URIs, routing screenshots/recordings through the
+ TTL-capped variant (SKY-12527). Order-preserving; None when any underlying
+ call fails, matching the clients' all-or-nothing batch contract.
+ """
+ sensitive = [i for i, artifact in enumerate(artifacts) if artifact.artifact_type in SENSITIVE_ARTIFACT_TYPES]
+ if not sensitive:
+ return await presign([artifact.uri for artifact in artifacts])
+ other = [i for i, artifact in enumerate(artifacts) if artifact.artifact_type not in SENSITIVE_ARTIFACT_TYPES]
+ urls: list[str | None] = [None] * len(artifacts)
+ for indices, mint in ((sensitive, presign_sensitive), (other, presign)):
+ if not indices:
+ continue
+ minted = await mint([artifacts[i].uri for i in indices])
+ if minted is None:
+ return None
+ for index, url in zip(indices, minted, strict=True):
+ urls[index] = url
+ return cast(list[str], urls)
+
async def _file_infos_from_artifacts(artifacts: list[Artifact], *, artifact_type: ArtifactType) -> list[FileInfo]:
"""Build the API-shaped ``FileInfo`` list from a homogeneous batch of
diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.py
index 7b7a016f2..ad8a45520 100644
--- a/skyvern/forge/sdk/artifact/storage/gcs.py
+++ b/skyvern/forge/sdk/artifact/storage/gcs.py
@@ -24,9 +24,11 @@ from skyvern.forge.sdk.api.real_gcp import RealAsyncGcsStorageClient
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.artifact.storage.base import (
FILE_EXTENTSION_MAP,
+ SENSITIVE_SHARE_URL_EXPIRY_HOURS,
BaseStorage,
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
+ presign_with_sensitive_cap,
)
from skyvern.forge.sdk.artifact.storage.run_recording_clips import (
RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS,
@@ -153,11 +155,17 @@ class GcsStorage(BaseStorage):
return await self.async_client.download_file(artifact.uri)
async def get_share_link(self, artifact: Artifact) -> str | None:
- share_urls = await self.async_client.create_signed_urls([artifact.uri])
+ share_urls = await self.get_share_links([artifact])
return share_urls[0] if share_urls else None
async def get_share_links(self, artifacts: list[Artifact]) -> list[str] | None:
- return await self.async_client.create_signed_urls([artifact.uri for artifact in artifacts])
+ return await presign_with_sensitive_cap(
+ artifacts,
+ presign=self.async_client.create_signed_urls,
+ presign_sensitive=lambda uris: self.async_client.create_signed_urls(
+ uris, expiry_hours=SENSITIVE_SHARE_URL_EXPIRY_HOURS
+ ),
+ )
async def store_artifact_from_path(self, artifact: Artifact, path: str) -> None:
storage_class = await self._get_storage_class_for_org(artifact.organization_id)
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index b67a2ba42..176a6a965 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -25,11 +25,13 @@ from skyvern.forge.sdk.api.files import (
wait_for_pending_extension_rename,
)
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
+from skyvern.forge.sdk.artifact.signing import SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
from skyvern.forge.sdk.artifact.storage.base import (
FILE_EXTENTSION_MAP,
BaseStorage,
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
+ presign_with_sensitive_cap,
)
from skyvern.forge.sdk.artifact.storage.run_recording_clips import (
RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS,
@@ -224,11 +226,17 @@ class S3Storage(BaseStorage):
return dict(pairs)
async def get_share_link(self, artifact: Artifact) -> str | None:
- share_urls = await self.async_client.create_presigned_urls([artifact.uri])
+ share_urls = await self.get_share_links([artifact])
return share_urls[0] if share_urls else None
async def get_share_links(self, artifacts: list[Artifact]) -> list[str] | None:
- return await self.async_client.create_presigned_urls([artifact.uri for artifact in artifacts])
+ return await presign_with_sensitive_cap(
+ artifacts,
+ presign=self.async_client.create_presigned_urls,
+ presign_sensitive=lambda uris: self.async_client.create_presigned_urls(
+ uris, expires_in=min(settings.PRESIGNED_URL_EXPIRATION, SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS)
+ ),
+ )
async def store_artifact_from_path(self, artifact: Artifact, path: str) -> None:
sc = await self._get_storage_class_for_org(artifact.organization_id, self.bucket, os.path.getsize(path))
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index cafa3c00a..66f6cf4b0 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -156,6 +156,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyVerdict,
actuation_obligation_key,
build_output_policy_diagnostics,
+ demote_author_time_steer_reasons,
derive_output_kind,
evaluate_actuation_obligation,
evaluate_output_policy,
@@ -190,7 +191,6 @@ from skyvern.forge.sdk.copilot.request_policy import (
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason
from skyvern.forge.sdk.copilot.runtime import (
_browser_context_is_attachable,
- cache_copilot_author_time_gate_log_only_ids,
)
from skyvern.forge.sdk.copilot.secret_redaction import SECRET_KEYWORD_ASSIGNMENT_PATTERN
from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values, scrub_secrets_from_text
@@ -3896,8 +3896,8 @@ def _inline_replace_workflow_credential_verdict(
ctx: CopilotContext, action_data: dict[str, Any], resp_type: str, user_response: str
) -> tuple[str, OutputPolicyVerdict, OutputPolicyVerdict]:
"""Rebind scouted credential literals into ``action_data['workflow_yaml']`` and evaluate the inline
- REPLACE_WORKFLOW output policy, recording churn when the hard-block verdict rejects. Returns the
- effective YAML plus the raw and hard-block verdicts."""
+ REPLACE_WORKFLOW output policy, recording churn when the author-time verdict rejects. Returns the
+ effective YAML plus the raw and author-time verdicts."""
workflow_yaml = action_data.get("workflow_yaml", "")
inline_rebind = rebind_scouted_credential_literals(workflow_yaml, ctx.scout_trajectory)
if inline_rebind.changed:
@@ -3924,18 +3924,31 @@ def _inline_replace_workflow_credential_verdict(
has_workflow_proposal=True,
output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
)
- hard_block_verdict = hard_block_output_policy_verdict(raw_verdict)
- if inline_rebind.residual_selectors and hard_block_verdict.allowed:
+ # This surface persists a draft, so it is graded like the update_workflow tool body rather than
+ # like a final reply: only what a later test-run cannot undo refuses here, and every other reason
+ # steers the next authoring attempt.
+ author_time_verdict = OutputPolicyVerdict(
+ allowed=raw_verdict.allowed,
+ output_kind=raw_verdict.output_kind,
+ reason_codes=list(raw_verdict.reason_codes),
+ )
+ steered_reasons = demote_author_time_steer_reasons(author_time_verdict)
+ if steered_reasons:
+ LOG.info(
+ "copilot inline REPLACE_WORKFLOW output policy reasons demoted to steering",
+ steered_reason_codes=[reason.value for reason in steered_reasons],
+ )
+ if inline_rebind.residual_selectors and author_time_verdict.allowed:
# This path bypasses the update_workflow guardrail, and the output-policy scan does not catch a bare
# `page.fill(sel, ...)`, so an inline secret the rebind could not neutralize would slip through here.
LOG.info(
"copilot inline REPLACE_WORKFLOW credential residual raw fill fail-closed",
residual_raw_credential_fill_selectors=list(inline_rebind.residual_selectors),
)
- hard_block_verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)
- if not hard_block_verdict.allowed:
- _record_output_policy_guardrail_churn(ctx, "replace_workflow_inline", workflow_yaml, hard_block_verdict)
- return workflow_yaml, raw_verdict, hard_block_verdict
+ author_time_verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)
+ if not author_time_verdict.allowed:
+ _record_output_policy_guardrail_churn(ctx, "replace_workflow_inline", workflow_yaml, author_time_verdict)
+ return workflow_yaml, raw_verdict, author_time_verdict
async def _translate_to_agent_result(
@@ -4044,23 +4057,24 @@ async def _translate_to_agent_result(
# The final-output policy pass still runs below; leave last_workflow
# / last_workflow_yaml unchanged until this candidate survives the
# inline checks.
+ from skyvern.forge.sdk.copilot.author_time_block import (
+ BANNED_BLOCKS_BLOCK_ID,
+ CODE_SAFETY_BLOCK_ID,
+ AuthorTimeBlock,
+ )
from skyvern.forge.sdk.copilot.tools import (
_banned_block_reject_message,
+ _code_block_safety_errors,
_detect_new_banned_blocks,
_detect_stale_block_metadata,
_record_banned_block_reject_span,
_stale_block_metadata_message,
- _timing_only_challenge_wait_reject_message,
composition_page_evidence_error,
workflow_target_url,
)
from skyvern.forge.sdk.copilot.tools.banned_blocks import _copilot_banned_block_types
+ from skyvern.forge.sdk.copilot.tools.workflow_update import _human_facing_code_safety_errors
- wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
- if wait_block_error:
- user_response = _with_inline_reject_note(user_response, wait_block_error)
- ctx.last_test_ok = None
- workflow_yaml = ""
banned_items = _detect_new_banned_blocks(
workflow_yaml,
ctx.last_workflow_yaml,
@@ -4068,23 +4082,43 @@ async def _translate_to_agent_result(
)
if banned_items:
_record_banned_block_reject_span("replace_workflow_inline", banned_items)
- user_response = _with_inline_reject_note(user_response, _banned_block_reject_message(banned_items, ctx))
+ inline_banned_blocks_block = AuthorTimeBlock(
+ block_id=BANNED_BLOCKS_BLOCK_ID,
+ error=_banned_block_reject_message(banned_items, ctx),
+ )
+ user_response = _with_inline_reject_note(user_response, inline_banned_blocks_block.error)
+ workflow_yaml = ""
+ # This surface persists a draft without the update_workflow tool, so it needs the same
+ # code-safety block: unsafe in-page code on a page holding a filled credential is the
+ # one thing a later test-run cannot undo. Constructed as an AuthorTimeBlock rather than
+ # discarded on a bare error string, so this refusal is bound by the same three-identity
+ # check as the tool seam and a new inline gate cannot join it silently.
+ inline_code_safety_errors = _human_facing_code_safety_errors(
+ _code_block_safety_errors(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)
+ )
+ if inline_code_safety_errors:
+ inline_code_safety_block = AuthorTimeBlock(
+ block_id=CODE_SAFETY_BLOCK_ID,
+ error="\n".join(str(error) for error in inline_code_safety_errors),
+ )
+ user_response = _with_inline_reject_note(user_response, inline_code_safety_block.error)
workflow_yaml = ""
+ # Stale metadata and missing page evidence are both authoring quality, not disclosure: a
+ # later test-run is what settles whether the draft works. They surface as a finding and
+ # clear the test credit, so the draft survives but cannot be reported as verified.
stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)
if stale_metadata:
user_response = _with_inline_reject_note(user_response, _stale_block_metadata_message(stale_metadata))
ctx.last_test_ok = None
- workflow_yaml = ""
composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
if composition_evidence_error:
LOG.info(
- "copilot inline composition page evidence rejected workflow",
+ "copilot inline composition page evidence finding",
workflow_permanent_id=ctx.workflow_permanent_id,
target_url=workflow_target_url(workflow_yaml),
)
user_response = _with_inline_reject_note(user_response, composition_evidence_error)
ctx.last_test_ok = None
- workflow_yaml = ""
if workflow_yaml:
# Inline REPLACE_WORKFLOW bypasses the update_workflow tool, so apply the same default here.
workflow_yaml = default_data_write_continue_on_failure(
@@ -5233,29 +5267,6 @@ def _reconcile_turn_end_ownership(
return result
-async def _cache_copilot_author_time_gate_log_only_ids(ctx: CopilotContext) -> None:
- try:
- resolved_ids = await app.AGENT_FUNCTION.resolve_copilot_author_time_gate_log_only_ids(
- turn_id=ctx.turn_id,
- organization_id=ctx.organization_id,
- )
- except Exception:
- LOG.warning(
- "Failed to resolve Copilot author-time gate log-only registry",
- turn_id=ctx.turn_id,
- organization_id=ctx.organization_id,
- exc_info=True,
- )
- resolved_ids = frozenset()
- cache_copilot_author_time_gate_log_only_ids(ctx, resolved_ids)
- selected_gate_ids = sorted(ctx.author_time_gate_log_only_ids)
- LOG.info(
- "copilot_author_time_gate_log_only_registry_resolved",
- selected_gate_ids=selected_gate_ids,
- selected_gate_count=len(selected_gate_ids),
- )
-
-
async def run_copilot_agent(
stream: EventSourceStream,
organization_id: str,
@@ -5473,7 +5484,6 @@ async def _run_copilot_turn_impl(
raise RuntimeError(
f"CopilotContext.turn_id ({ctx.turn_id!r}) diverged from route-supplied turn_id ({turn_id!r})"
)
- await _cache_copilot_author_time_gate_log_only_ids(ctx)
if ctx_sink is not None:
ctx_sink.append(ctx)
policy_inputs = RequestPolicyGuardrailInputs(
diff --git a/skyvern/forge/sdk/copilot/author_time_block.py b/skyvern/forge/sdk/copilot/author_time_block.py
new file mode 100644
index 000000000..effcc1f60
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/author_time_block.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+# ADR-0011: unsafe code in a persisted block outlives the turn and runs with the org's browser.
+CODE_SAFETY_BLOCK_ID = "code_safety"
+# ADR-0002: a raw or broadened credential in a persisted draft is an irreversible disclosure.
+CREDENTIAL_SCOUT_BLOCK_ID = "credential_scout"
+# ADR-0010: a persisted block outside the code-native ladder escapes the migration it forces.
+BANNED_BLOCKS_BLOCK_ID = "banned_blocks"
+
+AUTHOR_TIME_HARD_BLOCKS: frozenset[str] = frozenset(
+ {
+ CODE_SAFETY_BLOCK_ID,
+ CREDENTIAL_SCOUT_BLOCK_ID,
+ BANNED_BLOCKS_BLOCK_ID,
+ }
+)
+
+
+@dataclass(frozen=True)
+class AuthorTimeBlock:
+ """The only value an author-time refusal sink accepts. Constructing one with an
+ identity outside ``AUTHOR_TIME_HARD_BLOCKS`` raises, so a new validator cannot
+ become a wall without editing that constant."""
+
+ block_id: str
+ error: str
+ user_facing_summary: str | None = None
+ data: dict[str, Any] = field(default_factory=dict)
+
+ def __post_init__(self) -> None:
+ if self.block_id not in AUTHOR_TIME_HARD_BLOCKS:
+ raise ValueError(
+ f"{self.block_id!r} is not an author-time hard block; "
+ f"validators return findings. Allowed: {sorted(AUTHOR_TIME_HARD_BLOCKS)}"
+ )
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 67212d0eb..8bf2d6624 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-import json
import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
@@ -39,7 +38,6 @@ RecoveryHint = Literal[
]
LOG = structlog.get_logger()
-METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE = "metadata_reject_same_key_terminal"
# Matched case-insensitively. Imperative variants are narrow ("do not run" etc.) so plain "do not worry" prose doesn't false-positive.
@@ -151,52 +149,6 @@ class CopilotToolBlockerSignal(BaseModel):
return dict(value)
-def build_metadata_reject_same_key_terminal_signal(
- *,
- structural_key: str,
- reject_family: Literal[
- "missing_code_artifact_metadata",
- "metadata_normalization",
- "recorded_outcome_output_candidate",
- "recorded_outcome_output_coverage",
- ],
- missing_fields_by_label: Mapping[str, list[str]],
-) -> CopilotToolBlockerSignal:
- exact_fields = {label: list(fields) for label, fields in missing_fields_by_label.items()}
- rendered_fields = json.dumps(exact_fields, sort_keys=True, separators=(",", ":"))
- user_facing_reason = (
- "I couldn't satisfy the code_artifact_metadata gate after repeated identical submissions. "
- f"missing_fields_by_label={rendered_fields}. I've kept the draft so those exact fields can be added."
- )
- try:
- assert_clean_user_facing_text(user_facing_reason, blocked_tool="update_and_run_blocks")
- except ValueError:
- user_facing_reason = (
- "I couldn't satisfy the required output metadata after repeated identical submissions. "
- "I've kept the draft so the missing fields can be added."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="loop_detected",
- agent_steering_text=(
- "The same structural metadata rejection reached its terminal rung. Preserve the draft and report "
- f"the code_artifact_metadata gate with missing_fields_by_label={rendered_fields}."
- ),
- user_facing_reason=user_facing_reason,
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code=METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
- blocked_tool="update_and_run_blocks",
- extra={
- "reject_family": reject_family,
- "structural_key": structural_key,
- "gate_id": "code_artifact_metadata",
- "missing_fields_by_label": exact_fields,
- },
- )
-
-
def build_output_source_unobservable_blocker_signal(
*,
reason_code: str,
@@ -215,33 +167,18 @@ def build_output_source_unobservable_blocker_signal(
"visits, so there is nothing for the workflow to read them from. I've kept the draft; "
"tell me where those values appear and I'll wire them up."
)
- elif reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:
- user_facing = (
- "I rewrote this workflow several times to return the value(s) you asked for"
- f"{f' ({path_text})' if path_text else ''}, but each version left those outputs "
- "undeclared or unreturned, so the workflow can't reliably hand them back. I've kept "
- "the draft; tell me where each of those values appears and I'll declare and return them."
- )
else:
user_facing = (
"I couldn't shape this workflow so it reliably returns the values you asked for"
f"{f' ({path_text})' if path_text else ''}. I've kept the current draft; let me know "
"where those values show up and I'll try a different structure."
)
- if reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:
- agent_steer = (
- "STOP: the output contract kept failing its declaration/return coverage across repeated "
- f"re-authored candidates for required path(s) [{path_text or '(unknown)'}]. This is not "
- "repairable by re-authoring the same draft. Report the missing output declarations to the "
- "user and ask where the values appear. The prior draft is preserved; do not rerun the blocks."
- )
- else:
- agent_steer = (
- "STOP: the requested output contract has no observable extraction source in the current "
- f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "
- "re-authoring the same draft. Report the missing source to the user and ask where the "
- "values appear. The prior draft is preserved; do not rerun the blocks."
- )
+ agent_steer = (
+ "STOP: the requested output contract has no observable extraction source in the current "
+ f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "
+ "re-authoring the same draft. Report the missing source to the user and ask where the "
+ "values appear. The prior draft is preserved; do not rerun the blocks."
+ )
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
agent_steering_text=agent_steer,
@@ -259,39 +196,6 @@ def build_output_source_unobservable_blocker_signal(
)
-def build_definition_contract_unsatisfied_blocker_signal(
- *, unresolved_parameter_keys: Iterable[str], grounding_unresolved: bool = False
-) -> CopilotToolBlockerSignal:
- parameter_keys = sorted({str(key).strip() for key in unresolved_parameter_keys if str(key).strip()})
- if parameter_keys:
- key_text = ", ".join(f"`{key}`" for key in parameter_keys)
- user_facing = (
- f"I kept the workflow draft, but I could not safely connect the current page fields to these reusable "
- f"inputs: {key_text}. No workflow run was started."
- if grounding_unresolved
- else f"I kept the workflow draft, but it cannot run because these reusable inputs are not used: "
- f"{key_text}. Connect each input to the workflow before trying again."
- )
- else:
- user_facing = (
- "I kept the workflow draft, but it does not yet satisfy the required workflow definition. "
- "Complete the missing definition requirements before trying again."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=(
- "The exact candidate failed its definition contract before execution. Preserve the draft, stop this "
- "turn, and explain the unresolved reusable inputs without dispatching the candidate."
- ),
- user_facing_reason=user_facing,
- recovery_hint="report_blocker_to_user",
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code=DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,
- extra={"unresolved_parameter_keys": parameter_keys, "grounding_unresolved": grounding_unresolved},
- )
-
-
def build_llm_tool_error_payload(signal: CopilotToolBlockerSignal) -> str:
return signal.agent_steering_text
@@ -444,17 +348,12 @@ _LOOP_PROGRESS_TOOLS = frozenset(
_ACTIVE_TERMINAL_REPLACEABLE_REASON_CODES = frozenset({"tool_error_per_tool_budget_rerun"})
_TERMINAL_CHALLENGE_REPLACEABLE_REASON_CODES = frozenset({"tool_error_post_budget_challenge_result_evidence"})
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE = "tool_error_synthesized_block_persistence_required"
-UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE = "tool_error_uncovered_output_rescout_steer"
-RECORDED_OUTCOME_GROUNDING_REASON_CODE = "recorded_outcome_grounding_required"
-DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE = "definition_contract_unsatisfied"
SCHEMA_INCOMPATIBILITY_REASON_CODE = "schema_incompatibility"
-OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE = "output_contract_reject_budget_exhausted"
DISCOVERY_EXHAUSTED_NO_ENTRY_URL_REASON_CODE = "loop_detected_discovery_exhausted_no_entry_url"
_OUTPUT_CONTRACT_TERMINAL_REASON_CODES = frozenset(
{
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
}
)
@@ -468,14 +367,10 @@ GENUINELY_TERMINAL_BLOCKER_REASON_CODES: frozenset[str] = frozenset(
"tool_error_post_budget_challenge_blocker",
"tool_error_challenge_gated_submit_disabled",
"probable_site_block_stop",
- SCHEMA_INCOMPATIBILITY_REASON_CODE,
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
"advisory_dispatch_stalled",
- DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,
"repair_ceiling_reached",
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
DISCOVERY_EXHAUSTED_NO_ENTRY_URL_REASON_CODE,
}
)
@@ -506,18 +401,6 @@ def _should_stash_over_existing(
and existing.internal_reason_code in _TERMINAL_CHALLENGE_REPLACEABLE_REASON_CODES
):
return True
- if (
- incoming.internal_reason_code == RECORDED_OUTCOME_GROUNDING_REASON_CODE
- and existing.blocker_kind == "tool_error"
- and not existing.renders_final_reply
- ):
- return True
- if (
- existing.internal_reason_code == RECORDED_OUTCOME_GROUNDING_REASON_CODE
- and not existing.renders_final_reply
- and incoming.renders_final_reply
- ):
- return True
if (
existing.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
and not existing.renders_final_reply
diff --git a/skyvern/forge/sdk/copilot/build_phase.py b/skyvern/forge/sdk/copilot/build_phase.py
index d79a86840..263999d9a 100644
--- a/skyvern/forge/sdk/copilot/build_phase.py
+++ b/skyvern/forge/sdk/copilot/build_phase.py
@@ -287,6 +287,29 @@ def advance_to_testing(ctx: CopilotContext) -> None:
_log_transition(ctx, prev=prev, new=ctx.build_phase, reason="update_workflow_succeeded")
+# The complete vocabulary the build-phase plane may refuse with. It exists for the same reason
+# AUTHOR_TIME_HARD_BLOCKS and LOOP_PLANE_REFUSAL_REASON_CODES do: a phase gate added here cannot
+# wall a turn without appearing in this set.
+BUILD_PHASE_REFUSAL_REASON_CODES: frozenset[str] = frozenset(
+ {
+ "build_phase_discovery_disallowed_post_compose",
+ "build_phase_composition_inspection_blocked_pre_compose",
+ "build_phase_browser_blocked_pre_compose",
+ "build_phase_mutation_blocked_pre_compose",
+ }
+)
+
+
+def _declared_phase_refusal(signal: CopilotToolBlockerSignal) -> CopilotToolBlockerSignal:
+ reason = signal.internal_reason_code
+ if reason not in BUILD_PHASE_REFUSAL_REASON_CODES:
+ raise ValueError(
+ f"{reason!r} is not a declared build-phase refusal. A phase gate that ends a turn must be "
+ f"declared; anything else steers. Declared: {sorted(BUILD_PHASE_REFUSAL_REASON_CODES)}"
+ )
+ return signal
+
+
def _phase_blocker_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal | None:
"""Phase-aware authority blocker, parallel to `_turn_intent_tool_error` / `_request_policy_tool_error`."""
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
@@ -299,63 +322,79 @@ def _phase_blocker_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal
in_mutation = phase in MUTATION_PERMITTED_PHASES
if tool_name in _DISCOVERY_TOOLS and in_mutation:
- return CopilotToolBlockerSignal(
- blocker_kind="phase_gated",
- agent_steering_text=(
- "discover_workflow_entrypoint is only available before the entrypoint is resolved. "
- "The workflow already has a target URL — scout it with the browser tools and author with update_workflow. "
- "safe_reason_code=build_phase_discovery_disallowed_post_compose."
- ),
- user_facing_reason="I kept the existing target for this workflow instead of starting over.",
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"update_workflow", "update_and_run_blocks"}),
- internal_reason_code="build_phase_discovery_disallowed_post_compose",
- blocked_tool=tool_name,
+ return _declared_phase_refusal(
+ CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "discover_workflow_entrypoint is only available before the entrypoint is resolved. "
+ "The workflow already has a target URL — scout it with the browser tools and author with update_workflow. "
+ "safe_reason_code=build_phase_discovery_disallowed_post_compose."
+ ),
+ user_facing_reason="I kept the existing target for this workflow instead of starting over.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_workflow", "update_and_run_blocks"}),
+ renders_final_reply=False,
+ internal_reason_code="build_phase_discovery_disallowed_post_compose",
+ blocked_tool=tool_name,
+ )
)
if tool_name in _COMPOSITION_CONTEXT_TOOLS and in_discovery:
- return CopilotToolBlockerSignal(
- blocker_kind="phase_gated",
- agent_steering_text=(
- "Page inspection for composition is only available after an entrypoint URL is known. "
- "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
- "safe_reason_code=build_phase_composition_inspection_blocked_pre_compose."
- ),
- user_facing_reason="I need to know what page to inspect before I can read its form controls.",
- recovery_hint="ask_user_clarifying",
- cleared_by_tools=frozenset({"discover_workflow_entrypoint"}),
- internal_reason_code="build_phase_composition_inspection_blocked_pre_compose",
- blocked_tool=tool_name,
+ return _declared_phase_refusal(
+ CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Page inspection for composition is only available after an entrypoint URL is known. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
+ "safe_reason_code=build_phase_composition_inspection_blocked_pre_compose."
+ ),
+ user_facing_reason="I need to know what page to inspect before I can read its form controls.",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset({"discover_workflow_entrypoint"}),
+ renders_final_reply=False,
+ internal_reason_code="build_phase_composition_inspection_blocked_pre_compose",
+ blocked_tool=tool_name,
+ )
)
if tool_name in _BROWSER_PRIMITIVE_TOOLS and in_discovery:
- return CopilotToolBlockerSignal(
- blocker_kind="phase_gated",
- agent_steering_text=(
- "Direct browser tools are not callable before composition. "
- "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "
- "safe_reason_code=build_phase_browser_blocked_pre_compose."
- ),
- user_facing_reason="I need to know what site to work on before I can browse there. What URL should I use?",
- recovery_hint="ask_user_clarifying",
- cleared_by_tools=frozenset({"discover_workflow_entrypoint", "update_workflow", "update_and_run_blocks"}),
- internal_reason_code="build_phase_browser_blocked_pre_compose",
- blocked_tool=tool_name,
+ return _declared_phase_refusal(
+ CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Direct browser tools are not callable before composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "
+ "safe_reason_code=build_phase_browser_blocked_pre_compose."
+ ),
+ user_facing_reason="I need to know what site to work on before I can browse there. What URL should I use?",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(
+ {"discover_workflow_entrypoint", "update_workflow", "update_and_run_blocks"}
+ ),
+ renders_final_reply=False,
+ internal_reason_code="build_phase_browser_blocked_pre_compose",
+ blocked_tool=tool_name,
+ )
)
if tool_name in _MUTATION_TOOLS and in_discovery:
- return CopilotToolBlockerSignal(
- blocker_kind="phase_gated",
- agent_steering_text=(
- "Workflow mutation is gated to composition. "
- "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
- "safe_reason_code=build_phase_mutation_blocked_pre_compose."
- ),
- user_facing_reason="I need to know what site to work on before I can build a workflow. What URL should I use?",
- recovery_hint="ask_user_clarifying",
- cleared_by_tools=frozenset({"discover_workflow_entrypoint", "update_workflow", "update_and_run_blocks"}),
- internal_reason_code="build_phase_mutation_blocked_pre_compose",
- blocked_tool=tool_name,
+ return _declared_phase_refusal(
+ CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Workflow mutation is gated to composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
+ "safe_reason_code=build_phase_mutation_blocked_pre_compose."
+ ),
+ user_facing_reason="I need to know what site to work on before I can build a workflow. What URL should I use?",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(
+ {"discover_workflow_entrypoint", "update_workflow", "update_and_run_blocks"}
+ ),
+ renders_final_reply=False,
+ internal_reason_code="build_phase_mutation_blocked_pre_compose",
+ blocked_tool=tool_name,
+ )
)
return None
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 19299d4dc..e734bb77b 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -596,24 +596,6 @@ def clear_recorded_outcome_grounding_requirement(ctx: object) -> None:
ctx.recorded_outcome_binding_constraint = None # type: ignore[attr-defined]
-def recorded_outcome_grounding_requires_current_page(ctx: object) -> bool:
- requirement = getattr(ctx, "recorded_outcome_grounding_requirement", None)
- if not isinstance(requirement, RecordedOutcomeGroundingRequirement) or requirement.satisfied:
- return False
- if requirement.phase == "author_time_reject":
- return True
- if isinstance(requirement.workflow_run_id, str) and requirement.workflow_run_id:
- return True
- evidence = getattr(ctx, "composition_page_evidence", None)
- if isinstance(evidence, dict) and _evidence_current_url(evidence):
- return True
- contract = getattr(ctx, "scout_observation_contract", None)
- if scout_observation_contract_valid(contract):
- return True
- observed_urls = getattr(ctx, "observed_browser_urls", None)
- return isinstance(observed_urls, list) and any(isinstance(url, str) and url.strip() for url in observed_urls)
-
-
def maybe_satisfy_recorded_outcome_grounding_requirement(ctx: object) -> bool:
requirement = getattr(ctx, "recorded_outcome_grounding_requirement", None)
if not isinstance(requirement, RecordedOutcomeGroundingRequirement):
@@ -926,19 +908,6 @@ def recorded_outcome_from_author_time_reject(
)
-def author_time_reject_missing_output_paths(latest: RecordedBuildTestOutcome | None) -> set[str]:
- if latest is None or latest.phase != "author_time_reject":
- return set()
- paths: set[str] = set()
- for fact in latest.missing_requested_output_facts:
- if not isinstance(fact, Mapping):
- continue
- output_path = fact.get("output_path")
- if isinstance(output_path, str) and output_path:
- paths.add(output_path)
- return paths
-
-
def recorded_outcome_from_loaded_result_evidence(
evidence: LoadedResultCompositionEvidence,
) -> RecordedBuildTestOutcome:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index b53b2310f..dff2adf0b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -146,7 +146,6 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
- from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentClassifierResult
@@ -811,7 +810,6 @@ class CopilotContext(AgentContext):
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
tool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)
turn_halt: TurnHalt | None = None
- latest_schema_incompatibility: SchemaIncompatibility | None = None
# ``None`` until usage is observed; ``0`` only when a provider explicitly
# reported zero. Distinct values let cost grading flag missing telemetry.
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index b42012cb0..13b1e5d98 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -24,7 +24,6 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.output_policy import url_origin
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name
-from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_FAILURE_TYPE
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
@@ -56,7 +55,6 @@ class DiagnosisFailureType(StrEnum):
REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"
ACTIVE_RUN_TERMINAL_EVIDENCE = "active_run_terminal_evidence"
UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"
- SCHEMA_INCOMPATIBILITY = "schema_incompatibility"
DELIVERED_UNVERIFIED = "delivered_unverified"
UNKNOWN = "unknown"
@@ -252,11 +250,6 @@ def build_diagnosis_repair_contract(
"Stop the current retry loop: the active run reached the requested browser state, "
"but the reusable workflow is not verified end-to-end."
)
- elif failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:
- decision_summary = (
- "Stop re-authoring: the edited extraction schema declares fields that map to no output the "
- "workflow produces, so the mismatch is not repairable without user input."
- )
elif failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED:
decision_summary = "No repair selected; the latest run returned requested output but it was not verified."
if (
@@ -533,8 +526,6 @@ def _failure_type(
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
if _safe_str(data.get("failure_type")) == "missing_credential_or_init":
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
- if _safe_str(data.get("failure_type")) == SCHEMA_INCOMPATIBILITY_FAILURE_TYPE:
- return DiagnosisFailureType.SCHEMA_INCOMPATIBILITY
if repair_context is not None:
return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
if outcome_verified:
@@ -572,8 +563,6 @@ def _next_action(
return RepairNextAction.STOP
if failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:
return RepairNextAction.STOP
- if failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:
- return RepairNextAction.STOP
if ctx.last_test_non_retriable_nav_error:
return RepairNextAction.STOP
if _last_test_anti_bot_is_terminal(ctx, data) and failure_type in {
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 98e248d5c..d434303c2 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -21,7 +21,6 @@ from skyvern.forge.sdk.copilot import streaming_adapter
from skyvern.forge.sdk.copilot.blocker_signal import (
RAW_SECRET_LEAK_REASON_CODE,
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
- UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
CopilotToolBlockerSignal,
clear_tool_blocker_signals_for_reason_codes,
compose_loop_blocker_user_facing_reason,
@@ -32,7 +31,6 @@ from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_FAILURE_STREAK_ESCAP
from skyvern.forge.sdk.copilot.build_test_outcome import (
PostRunPagePathFailure,
RecordedBuildTestOutcome,
- author_time_reject_missing_output_paths,
latest_recorded_build_test_outcome_repeated,
record_build_test_outcome,
recorded_outcome_from_authoring_repair_context,
@@ -49,9 +47,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_durable_fallback_entry_target,
is_generic_entry_opener_click,
is_optional_dismissal_only_trajectory,
- locator_selector_literals,
missing_rung_text,
- normalized_scout_selector,
obligation_finding_reason_code,
obligation_finding_selector,
render_missing_rung_call_sources,
@@ -147,9 +143,7 @@ from skyvern.forge.sdk.copilot.run_outcome import (
)
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
- AuthorTimeGateAblationPayload,
PostRunPagePathInteractionWindow,
- record_author_time_gate_ablation_event,
)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.terminal_predicates import (
@@ -2438,12 +2432,8 @@ def synthesized_persistence_reopened_after_failed_run(ctx: Any) -> bool:
def synthesized_persistence_reopened(ctx: AgentContext) -> bool:
- if ctx.synthesized_block_reopened_for_output_coverage:
- return True
if ctx.synthesized_block_reopened_for_credential_scout:
return True
- if getattr(ctx, "synthesized_block_reopened_for_capture_obligation", False):
- return True
if synthesized_goal_completion_landing_pending(ctx):
return True
return synthesized_persistence_reopened_after_failed_run(ctx)
@@ -3200,22 +3190,6 @@ def _ambiguous_bare_selector_rescout_signal_state(ctx: Any, tool_name: str) -> s
return "allow"
-def _uncovered_output_reject_rescout_key(canonical_paths: set[str], structural_failure_identity: str) -> str:
- return f"{structural_failure_identity}|{','.join(sorted(canonical_paths))}"
-
-
-def _active_uncovered_output_reject_paths(ctx: AgentContext) -> set[str]:
- canonical = {
- _canonical_output_path(path)
- for path in author_time_reject_missing_output_paths(ctx.latest_recorded_build_test_outcome)
- }
- return canonical & uncovered_requested_output_paths(ctx) if canonical else set()
-
-
-def _uncovered_output_reject_admits_evaluate(ctx: CopilotContext, tool_name: str) -> bool:
- return tool_name == "evaluate" and bool(_active_uncovered_output_reject_paths(ctx))
-
-
def _actuation_obligation_live_fill_delivery_required(ctx: CopilotContext) -> bool:
turn_intent = getattr(ctx, "turn_intent", None)
if (
@@ -3283,101 +3257,6 @@ def _credential_scout_reopen_admits_evaluate(ctx: CopilotContext, tool_name: str
return tool_name == "evaluate" and bool(ctx.synthesized_block_reopened_for_credential_scout)
-def _never_captured_obligation_admits_expected_tool(
- ctx: CopilotContext, tool_name: str, arguments: Mapping[str, Any] | None
-) -> bool:
- obligation = getattr(ctx, "never_captured_obligation", None)
- if (
- obligation is None
- or obligation.state != "armed"
- or obligation.turn_id != ctx.turn_id
- or tool_name != obligation.expected_tool_name
- or not isinstance(arguments, Mapping)
- ):
- return False
- selector = arguments.get("selector")
- if not isinstance(selector, str) or not selector.strip():
- return False
- expected_selectors = {
- normalized_scout_selector(candidate) for candidate in locator_selector_literals(obligation.normalized_receiver)
- }
- if normalized_scout_selector(selector.strip()) not in expected_selectors:
- return False
- expected_argument = obligation.expected_argument_literal
- if expected_argument is None:
- return True
- argument_key = {"press_key": "key", "select_option": "value", "type_text": "text"}.get(tool_name)
- return argument_key is not None and str(arguments.get(argument_key) or "") == expected_argument
-
-
-def consume_uncovered_output_reopen_event(ctx: CopilotContext) -> bool:
- """Arm a one-shot scout-window reopen for the first author-time reject citing an uncovered
- requested-output path. Returns True only on that first reject per structural identity; a
- repeat identical reject falls through so it counts normally toward the repair ceiling."""
- active = _active_uncovered_output_reject_paths(ctx)
- if not active:
- return False
- latest = ctx.latest_recorded_build_test_outcome
- if latest is None:
- return False
- key = _uncovered_output_reject_rescout_key(active, latest.structural_failure_identity)
- if ctx.uncovered_output_rescout_context_key == key:
- return False
- ctx.uncovered_output_rescout_context_key = key
- ctx.synthesized_block_reopened_for_output_coverage = True
- return True
-
-
-def uncovered_output_reject_scout_steer_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- if tool_name not in _SYNTHESIZED_BLOCK_REAUTHORING_TOOLS:
- return None
- if not ctx.synthesized_block_reopened_for_output_coverage:
- return None
- active = _active_uncovered_output_reject_paths(ctx)
- if not active:
- return None
- latest = ctx.latest_recorded_build_test_outcome
- if latest is None:
- return None
- key = _uncovered_output_reject_rescout_key(active, latest.structural_failure_identity)
- if ctx.uncovered_output_rescout_steer_key == key:
- return None
- payload: AuthorTimeGateAblationPayload = {
- "uncovered_output_paths": sorted(active),
- "structural_failure_identity": latest.structural_failure_identity,
- }
- if record_author_time_gate_ablation_event(
- ctx,
- gate_id="uncovered_output_rescout_steer",
- reason_code=UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
- fingerprint=key,
- blocked_tool=tool_name,
- payload=payload,
- ):
- return None
- # Commit-after-claim: a yielded steer must not burn the one-shot rescout key.
- if claim_turn(ctx, TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER) is ClaimOutcome.YIELDED:
- return None
- ctx.uncovered_output_rescout_steer_key = key
- named_paths = ", ".join(sorted(active))
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=(
- "The authored workflow still leaves these requested output paths unobserved: "
- f"{named_paths}. Do NOT re-author yet. Call evaluate to scout the page where those values "
- "appear until they are observed, then author a block that returns them."
- ),
- user_facing_reason="I need to view the page with the requested details before saving the workflow.",
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"evaluate"}),
- preserves_workflow_draft=True,
- renders_final_reply=False,
- internal_reason_code=UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
- blocked_tool=tool_name,
- extra={"uncovered_output_paths": sorted(active)},
- )
-
-
def _should_block_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:
if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS and tool_name != "update_workflow":
return False
@@ -3554,14 +3433,9 @@ def synthesized_block_persistence_signal(
ambiguous_selector_rescout_state = _ambiguous_bare_selector_rescout_signal_state(ctx, tool_name)
if ambiguous_selector_rescout_state == "allow":
return None
- if _uncovered_output_reject_admits_evaluate(ctx, tool_name):
- return None
if _credential_scout_reopen_admits_evaluate(ctx, tool_name):
claim_turn(ctx, TurnClaimant.CREDENTIAL_SCOUT_REOPEN)
return None
- if _never_captured_obligation_admits_expected_tool(ctx, tool_name, arguments):
- claim_turn(ctx, TurnClaimant.CAPTURE_OBLIGATION_REOPEN)
- return None
if _actuation_obligation_admits_required_fill_tool(ctx, tool_name):
claim_turn(ctx, TurnClaimant.ACTUATION_OBLIGATION_FILL)
return None
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 2388e2d15..98d45825d 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -70,8 +70,6 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"pending_browser_interaction_observation",
"scouted_interactions",
"scout_trajectory",
- "never_captured_obligation",
- "synthesized_block_reopened_for_capture_obligation",
"pending_scout_source_url",
"pending_scout_typed_value",
"pending_scout_role_name",
@@ -103,9 +101,6 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"scout_observed_terminal_criterion_ids",
"scout_observation_contract",
"requested_output_extraction_candidate",
- "synthesized_block_reopened_for_output_coverage",
- "uncovered_output_rescout_context_key",
- "uncovered_output_rescout_steer_key",
"consecutive_no_progress_interaction_count",
)
diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.py
index c84aea29e..11d45ab4c 100644
--- a/skyvern/forge/sdk/copilot/output_contracts.py
+++ b/skyvern/forge/sdk/copilot/output_contracts.py
@@ -23,8 +23,6 @@ class OutputContractBailFamily(StrEnum):
class OutputContractActuationKind(StrEnum):
IMPOSED = "imposed"
- CREDIT = "credit"
- IMPOSE_SCOUTED_BINDING = "impose_scouted_binding"
STRUCTURE_DIRECTIVE = "structure_directive"
ADVISORY_RUN = "advisory_run"
BLOCKED_TERMINAL = "blocked_terminal"
@@ -65,11 +63,6 @@ class OutputContractActuationEvidence:
consumed_run_bound_required_path: bool = False
consumed_run_carried_page_extraction: bool = False
loaded_result_source_producible: bool = False
- # Steering produced no structural movement: the canonical structural fingerprint did not
- # change across the consumed directives. Never a reject count — counts inflate.
- steering_exhausted: bool = False
- credit_available: bool = False
- scouted_binding_impose_available: bool = False
@dataclass(frozen=True)
@@ -97,17 +90,9 @@ def resolve_output_contract_actuation(
page-source extraction, and that it still bound no required path. A consumed run whose output was
observed but bound nothing without a page-source extraction on board is not exhaustion evidence —
it re-enters the ladder once so the stronger page-source imposition can bind the on-screen values a
- code static-return provably cannot key. Once steering is exhausted the server resolves the
- submission itself — crediting a draft the strict prover can only call uncertain, else imposing the
- binding the recording pins — so a satisfiable submission has a reachable pass path instead of only
- a terminal; both inherit the advisory lane's post-run adjudication."""
+ code static-return provably cannot key."""
if evidence.imposed_available:
return OutputContractActuation(OutputContractActuationKind.IMPOSED, family)
- if evidence.steering_exhausted:
- if evidence.credit_available:
- return OutputContractActuation(OutputContractActuationKind.CREDIT, family)
- if evidence.scouted_binding_impose_available:
- return OutputContractActuation(OutputContractActuationKind.IMPOSE_SCOUTED_BINDING, family)
grantable_source = (
evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED
)
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 74e821052..34ce9ad9b 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -291,6 +291,29 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
)
+# The authoring seam refuses only what the ``credential_scout`` hard block covers, because a raw or
+# broadened credential in a persisted draft is an irreversible disclosure; everything else steers.
+# Each member's surface and the reason it cannot be dropped are in
+# cloud_docs/workflow-copilot/architecture/output-policy-disposition.md.
+_AUTHOR_TIME_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
+ {
+ OutputPolicyReason.RAW_SECRET_LEAK,
+ OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
+ OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
+ }
+)
+
+
+def demote_author_time_steer_reasons(verdict: OutputPolicyVerdict) -> list[OutputPolicyReason]:
+ """Drop the reasons that only steer the next authoring attempt, flipping ``allowed``
+ back to True when nothing that outlives the turn remains. Returns the demoted reasons
+ so the caller can trace them."""
+ steered = [reason for reason in verdict.reason_codes if reason not in _AUTHOR_TIME_HARD_BLOCK_REASONS]
+ for reason in steered:
+ verdict.remove(reason)
+ return steered
+
+
def coerce_cannot_act_reason(value: str | None) -> CannotActReason | None:
if value is None:
return None
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 4d1317e6f..00351da2c 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -4,10 +4,9 @@ from __future__ import annotations
import asyncio
import inspect
-from collections.abc import Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Literal, NotRequired, TypeAlias, TypedDict, cast
+from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, NotRequired, TypeAlias, TypedDict, cast
import structlog
@@ -63,7 +62,6 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, ScoutObservationContract
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
- from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
from skyvern.forge.sdk.copilot.turn_ownership import GatePrecedenceConflictEvent, TurnClaimant, TurnOwnership
@@ -78,42 +76,10 @@ _SESSION_CLEANUP_TIMEOUT_SECONDS = 5.0
_BROWSER_BOOT_WAIT_SECONDS = 30.0
_BROWSER_BOOT_POLL_INTERVAL_SECONDS = 0.25
_FINAL_BROWSER_SESSION_STATUSES: frozenset[str] = frozenset({"completed", "failed", "timeout"})
-DEFINITION_CONTRACT_UNSATISFIED_GATE_ID = "definition_contract_unsatisfied"
-RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID = "recorded_outcome_grounding_binder_ceiling"
-SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID = "synthesized_parameter_binding_ambiguous"
-OUTPUT_CONTRACT_ACTUATION_GATE_ID = "output_contract_actuation"
-METADATA_RUN_PREFLIGHT_REJECT_GATE_ID = "metadata_run_preflight_reject"
-UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID = "uncovered_output_rescout_steer"
-RECORDED_OUTCOME_GROUNDING_GATE_ID = "recorded_outcome_grounding"
-# Every author-time gate that has a suppression seam, and therefore everything the flag
-# payload and the local blanket can disable. Security lanes are absent by construction, not
-# by omission: they never call record_author_time_gate_ablation_event, so no payload reaches
-# them. A gate that gains a seam belongs here, or the flag cannot address it.
-AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(
- {
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
- SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- OUTPUT_CONTRACT_ACTUATION_GATE_ID,
- METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
- UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID,
- RECORDED_OUTCOME_GROUNDING_GATE_ID,
- }
-)
CodeArtifactMetadataValue: TypeAlias = (
str | int | float | bool | None | list["CodeArtifactMetadataValue"] | dict[str, "CodeArtifactMetadataValue"]
)
CodeArtifactMetadataPayload: TypeAlias = dict[str, CodeArtifactMetadataValue]
-AuthorTimeGateAblationPayloadValue: TypeAlias = (
- str
- | int
- | float
- | bool
- | None
- | Sequence["AuthorTimeGateAblationPayloadValue"]
- | dict[str, "AuthorTimeGateAblationPayloadValue"]
-)
-AuthorTimeGateAblationPayload: TypeAlias = dict[str, AuthorTimeGateAblationPayloadValue]
SdkActionWorkflowRunCacheKey: TypeAlias = tuple[str, str]
@@ -180,11 +146,6 @@ class PreRunPageReference:
workflow_run_id: str
-@dataclass(frozen=True)
-class RejectedCodeArtifactMetadataCapture:
- payload: Any
-
-
@dataclass(frozen=True)
class RegisteredArtifactEntry:
artifact_id: str
@@ -198,16 +159,6 @@ class RegisteredArtifactEvidence:
workflow_run_id: str
-@dataclass(frozen=True)
-class AuthorTimeGateAblationEvent:
- gate_id: str
- reason_code: str
- fingerprint: str
- log_only: bool
- blocked_tool: str | None = None
- payload: AuthorTimeGateAblationPayload = field(default_factory=dict)
-
-
class ScoutedEquivalentInput(TypedDict):
input_key: str
parameter_value: str
@@ -288,39 +239,6 @@ class ScoutedInteraction(TypedDict):
challenge_state: NotRequired[dict[str, Any]]
-NeverCapturedObligationState: TypeAlias = Literal["armed", "captured", "consumed"]
-
-
-@dataclass(frozen=True)
-class NeverCapturedReplayPayload:
- """Turn-ephemeral inputs required to retry the exact rejected authoring call."""
-
- params: dict[str, Any]
- allow_missing_credentials: bool | None = None
- allow_static_output_uncertainty: bool = False
- formation_prepared: bool = False
-
-
-@dataclass(frozen=True)
-class NeverCapturedObligation:
- """Turn-ephemeral authority to re-scout one exact authored browser mutation."""
-
- identity_digest: str
- turn_id: str
- draft_fingerprint: str
- block_label: str
- site: str
- method: str
- normalized_receiver: str
- call_shape_digest: str
- expected_tool_name: str
- armed_after_trajectory_index: int
- expected_argument_literal: str | None = None
- captured_trajectory_index: int | None = None
- state: NeverCapturedObligationState = "armed"
- replay_payload: NeverCapturedReplayPayload | None = None
-
-
@dataclass(frozen=True)
class PostRunPagePathInteractionWindow:
structural_key: str
@@ -479,7 +397,6 @@ class AgentContext:
code_artifact_metadata: dict[str, CodeArtifactMetadataPayload] = field(default_factory=dict)
raw_code_artifact_metadata: object | None = None
submitted_code_artifact_metadata_snapshot: Any = None
- rejected_code_artifact_metadata_captures: list[RejectedCodeArtifactMetadataCapture] = field(default_factory=list)
# Hydrated at turn start from StructuredContext.observed_acted_pages; lets the
# composition gate credit a page observed on a prior turn when this turn's
# flow_evidence does not cover it (closes the spent-inspection-budget
@@ -513,10 +430,6 @@ class AgentContext:
# Solve attempts already spent per challenge. A challenge that never passes is precisely the
# one re-observed on every later capture, so cost otherwise grows with how stuck the turn is.
challenge_solve_attempts: dict[str, int] = field(default_factory=dict)
- # One exact `never_captured` mutation may reopen scouting within this turn. The obligation is
- # completed only by a later generator-emitted canonical interaction, never by selector text alone.
- never_captured_obligation: NeverCapturedObligation | None = None
- never_captured_obligation_identity_history: set[str] = field(default_factory=set)
# Latest typed reached-download target from the scout steer; the synthesizer compiles the terminal
# expect_download step from it. Selector is the observed download link, not necessarily a trajectory click.
reached_download_target: ReachedDownloadTarget | None = None
@@ -528,13 +441,8 @@ class AgentContext:
scouted_spine_repeated_identical_missing_steps: bool = False
# Author-time output-contract cross-turn state, keyed by the contract signature; set lazily by workflow_update.
output_contract_pinned_block_label_by_signature: dict[str, str] = field(default_factory=dict)
- output_contract_reject_count_by_signature: dict[str, int] = field(default_factory=dict)
- output_contract_deferral_count_by_signature: dict[str, int] = field(default_factory=dict)
runtime_output_repair_attempt_by_signature: dict[str, bool] = field(default_factory=dict)
- # Progress-gated reset ledger: the last rejected draft's structural fingerprint and
- # whether an imposition landed since, so a genuinely-changed re-attempt resets the
- # steering-reject streak instead of counting cosmetic churn toward the cap.
- output_contract_last_reject_fingerprint_by_signature: dict[str, str] = field(default_factory=dict)
+ # Whether an imposition landed for this contract since the last attempt.
output_contract_imposed_since_last_reject_by_signature: dict[str, bool] = field(default_factory=dict)
# Structural fingerprint captured when a structure directive was armed; a re-entry whose
# fingerprint still matches means the directive went unconsumed (cosmetic churn), which
@@ -595,9 +503,7 @@ class AgentContext:
# returns. The freehand persist-seam surface leg exempts exactly this label and gates its siblings.
spine_imposition_carrier_label: str | None = None
synthesized_block_reopened_after_failed_run: bool = False
- synthesized_block_reopened_for_output_coverage: bool = False
synthesized_block_reopened_for_credential_scout: bool = False
- synthesized_block_reopened_for_capture_obligation: bool = False
# Business inputs proven required by an earlier synthesized-draft rejection stay required for the
# rest of the turn. A later retry cannot evade the floor by deleting those parameters from its YAML.
synthesized_business_required_parameter_keys: set[str] = field(default_factory=set)
@@ -606,8 +512,6 @@ class AgentContext:
# login prefix; releases the is_goal_complete terminal-action gate mirroring reached_download_target.
scout_observed_terminal_criterion_ids: set[str] = field(default_factory=set)
scout_observation_contract: ScoutObservationContract | None = None
- uncovered_output_rescout_context_key: str | None = None
- uncovered_output_rescout_steer_key: str | None = None
credential_scout_rescout_context_key: str | None = None
# Which requires-live-scout fields (username/password, non-empty) each scouted credential
# carries; recorded at credential resolve time and rehydrated from FillCarry across turns.
@@ -616,9 +520,6 @@ class AgentContext:
# showed a password-type control; orders page evidence against post-fill submits across evictions.
last_scout_observation_trajectory_index: int | None = None
last_scout_observation_has_password_control: bool = False
- # Count of times the scout-act download gate rejected a download-intent block this turn. Bounds
- # the author->scout->re-author cycle so a genuinely un-scoutable affordance halts honestly.
- download_scout_required_rejections: int = 0
# Required parameter keys the build-test resolution seam could not bind from a user param,
# a non-empty default, or a scout value. Reset per run; read when composing the run outcome.
unbound_required_parameter_keys: list[str] = field(default_factory=list)
@@ -660,12 +561,6 @@ class AgentContext:
# render the current tool result from structured product text.
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
tool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)
- # Latest edited-schema-incompatibility terminal outcome, set when an edited
- # extraction_schema declares fields that map to no output the block produces.
- # Surfaced into the persisted TurnOutcome so a later turn can report it.
- latest_schema_incompatibility: SchemaIncompatibility | None = None
- author_time_gate_log_only_ids: frozenset[str] = frozenset()
- author_time_gate_ablation_events: list[AuthorTimeGateAblationEvent] = field(default_factory=list)
# Single-owner turn-precedence contract. One mechanism owns a turn's steering
# at a time; a contradicting weaker claim is recorded here and yields.
turn_ownership: TurnOwnership | None = None
@@ -675,54 +570,6 @@ class AgentContext:
blocker_signal_claimant: TurnClaimant | None = None
-def cache_copilot_author_time_gate_log_only_ids(ctx: AgentContext, resolved_ids: frozenset[str]) -> None:
- ineligible_ids = resolved_ids - AUTHOR_TIME_GATE_LOG_ONLY_IDS
- for gate_id in sorted(ineligible_ids):
- LOG.info("copilot_gate_log_only_ineligible", gate_id=gate_id)
- ctx.author_time_gate_log_only_ids = resolved_ids & AUTHOR_TIME_GATE_LOG_ONLY_IDS
-
-
-def copilot_author_time_gate_log_only_enabled(ctx: AgentContext, gate_id: str) -> bool:
- local_blanket_enabled = (
- not settings.is_cloud_environment()
- and settings.WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY
- and gate_id in AUTHOR_TIME_GATE_LOG_ONLY_IDS
- )
- return local_blanket_enabled or gate_id in ctx.author_time_gate_log_only_ids
-
-
-def record_author_time_gate_ablation_event(
- ctx: AgentContext,
- *,
- gate_id: str,
- reason_code: str,
- fingerprint: str,
- blocked_tool: str | None = None,
- payload: AuthorTimeGateAblationPayload | None = None,
-) -> bool:
- if not copilot_author_time_gate_log_only_enabled(ctx, gate_id):
- return False
- event = AuthorTimeGateAblationEvent(
- gate_id=gate_id,
- reason_code=reason_code,
- fingerprint=fingerprint,
- blocked_tool=blocked_tool,
- payload=dict(payload or {}),
- log_only=True,
- )
- ctx.author_time_gate_ablation_events.append(event)
- LOG.info(
- "copilot_author_time_gate_ablation_event",
- gate_id=event.gate_id,
- reason_code=event.reason_code,
- fingerprint=event.fingerprint,
- blocked_tool=event.blocked_tool,
- log_only=event.log_only,
- payload=event.payload,
- )
- return True
-
-
def output_contract_ladder_unresolved(ctx: AgentContext) -> bool:
"""True while an output-contract signature has a live actuation ladder — a landed actuation or a GRANTED
advisory — that has not yet reached a typed terminal or a dispatched (CONSUMED) run. Loop and churn detectors
diff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
index 3d138007b..691b866ec 100644
--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py
@@ -340,7 +340,7 @@ def _result_has_terminal_or_ask_precedence(copilot_ctx: Any, data: dict[str, Any
return True
if data.get("skip_reason") == "workflow_credential_inputs_unbound":
return True
- if data.get("failure_type") in {"schema_incompatibility", "missing_credential_or_init"}:
+ if data.get("failure_type") == "missing_credential_or_init":
return True
categories = data.get("failure_categories")
if not isinstance(categories, list):
diff --git a/skyvern/forge/sdk/copilot/schema_incompatibility.py b/skyvern/forge/sdk/copilot/schema_incompatibility.py
index ff70d9eab..3d6af11a1 100644
--- a/skyvern/forge/sdk/copilot/schema_incompatibility.py
+++ b/skyvern/forge/sdk/copilot/schema_incompatibility.py
@@ -1,13 +1,12 @@
-"""Typed terminal outcome for an edited extraction schema whose fields map to no
-output the workflow produces.
+"""Typed finding for an edited extraction schema whose fields map to no output the
+workflow produces.
When a user edits a code block's confirmed ``extraction_schema`` to add fields that
overlap none of the block's known output contract (its top-level return keys plus
confirmed ``goal_value_paths``), re-authoring cannot reconcile the mismatch: there is
-nothing on the page or in the return for the new field to bind to. Surfacing it as a
-typed, non-repairable schema-incompatibility outcome — rather than letting the agent
-churn until the repair ceiling — preserves the existing draft and tells the user
-exactly which fields do not map.
+nothing on the page or in the return for the new field to bind to. The draft persists
+and the finding names the fields that do not map, because a test-run cannot: the run
+succeeds and silently omits them.
"""
from __future__ import annotations
@@ -20,11 +19,9 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
SCHEMA_INCOMPATIBILITY_REASON_CODE as SCHEMA_INCOMPATIBILITY_REASON_CODE,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
- CopilotToolBlockerSignal,
assert_clean_user_facing_text,
)
-SCHEMA_INCOMPATIBILITY_FAILURE_TYPE = "schema_incompatibility"
SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL = "update_and_run_blocks"
_DEFAULT_NEXT_ACTIONS: tuple[str, ...] = (
@@ -34,9 +31,8 @@ _DEFAULT_NEXT_ACTIONS: tuple[str, ...] = (
)
_GENERIC_USER_REASON = (
- "I couldn't apply the edited extraction schema because some of its fields don't match "
- "any value this workflow currently produces. Tell me what they should map to and I'll try again. "
- "Your current draft is unchanged."
+ "I saved the workflow, but some fields in the edited extraction schema don't match any value it "
+ "produces, so those fields will come back empty. Tell me what they should map to and I'll wire them up."
)
@@ -62,8 +58,8 @@ class SchemaIncompatibility(BaseModel):
def merge_schema_incompatibilities(items: list[SchemaIncompatibility]) -> SchemaIncompatibility | None:
- """Fold per-block incompatibilities into a single terminal record. The merged
- paths are de-duplicated and ordered; the first block label anchors the record."""
+ """Fold per-block incompatibilities into a single record. The merged paths are
+ de-duplicated and ordered; the first block label anchors the record."""
real = [item for item in items if item is not None]
if not real:
return None
@@ -103,16 +99,16 @@ def render_schema_incompatibility_user_reason(incompat: SchemaIncompatibility) -
"""Product-language reply rendered from the structured incompatibility. Falls back
to a field-free message if an exotic field name trips the user-facing safety gate."""
fields = _field_list_phrase(incompat.incompatible_paths)
- verb = "doesn't" if len(incompat.incompatible_paths) == 1 else "don't"
+ single = len(incompat.incompatible_paths) == 1
+ verb = "doesn't" if single else "don't"
+ subject = "it will" if single else "they will"
sentences = [
- f"I couldn't apply the edited extraction schema: the field {fields} {verb} match any value this workflow currently produces."
+ f"I saved the workflow, but the field {fields} {verb} match any value it produces, so {subject} come back empty."
]
if incompat.known_output_paths:
outputs = ", ".join(incompat.known_output_paths)
sentences.append(f"This workflow's data currently covers {outputs}.")
- sentences.append(
- "Tell me which existing output it should map to, or remove it, and I'll try again. Your current draft is unchanged."
- )
+ sentences.append("Tell me which existing output it should map to, or remove it, and I'll wire it up.")
candidate = " ".join(sentences)
try:
assert_clean_user_facing_text(candidate, blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL)
@@ -125,23 +121,7 @@ def render_schema_incompatibility_agent_steer(incompat: SchemaIncompatibility) -
incompatible = ", ".join(incompat.incompatible_paths) or "(unknown)"
known = ", ".join(incompat.known_output_paths) or "(none recorded)"
return (
- f"STOP: the edited extraction_schema declares field(s) [{incompatible}] that map to no output block "
- f"`{incompat.block_label}` produces [{known}]. This is not repairable by re-authoring the same draft. "
- "Report the mismatch to the user and ask which existing output the field should map to. "
- "The prior draft is preserved; do not rerun the blocks."
- )
-
-
-def build_schema_incompatibility_blocker_signal(incompat: SchemaIncompatibility) -> CopilotToolBlockerSignal:
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=render_schema_incompatibility_agent_steer(incompat),
- user_facing_reason=render_schema_incompatibility_user_reason(incompat),
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=incompat.preserves_workflow_draft,
- renders_final_reply=True,
- internal_reason_code=SCHEMA_INCOMPATIBILITY_REASON_CODE,
- blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL,
- extra={"schema_incompatibility": incompat.to_summary_dict()},
+ f"The edited extraction_schema declares field(s) [{incompatible}] that map to no output block "
+ f"`{incompat.block_label}` produces [{known}]. Re-authoring the same draft will not resolve it; "
+ "ask the user which existing output the field should map to."
)
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index 39538711c..2ae881c48 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -644,8 +644,6 @@ def _update_enforcement_from_tool(
ctx.post_update_nudge_count = 0
if tool_name in ("update_workflow", "update_and_run_blocks") and has_blocks:
ctx.synthesized_block_reopened_after_failed_run = False
- ctx.synthesized_block_reopened_for_output_coverage = False
- ctx.uncovered_output_rescout_steer_key = None
if tool_name in ("run_blocks_and_collect_debug", "update_and_run_blocks"):
ctx.test_after_update_done = True
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 5a85a35fb..ee13459f4 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -11,11 +11,9 @@ from agents import function_tool
from agents.run_context import RunContextWrapper
from skyvern.forge import app as app
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_phase import (
advance_to_testing,
)
-from skyvern.forge.sdk.copilot.build_test_outcome import recorded_outcome_grounding_requires_current_page
from skyvern.forge.sdk.copilot.composition_evidence import (
composition_page_evidence_error as composition_page_evidence_error,
)
@@ -51,9 +49,6 @@ from ._shared import (
)
from ._shared import _composition_get_html as _composition_get_html
from ._shared import _current_workflow_has_evidence_block as _current_workflow_has_evidence_block
-from ._shared import (
- _emit_tool_blocker_signal,
-)
from ._shared import _fallback_page_info as _fallback_page_info
from ._shared import _is_meaningful_extracted_data as _is_meaningful_extracted_data
from ._shared import _proxy_location_trace_value as _proxy_location_trace_value
@@ -62,11 +57,8 @@ from ._shared import _same_page_ignoring_fragment as _same_page_ignoring_fragmen
from ._shared import _unverified_current_workflow_labels as _unverified_current_workflow_labels
from .banned_blocks import _COPILOT_BANNED_BLOCK_TYPES as _COPILOT_BANNED_BLOCK_TYPES
from .banned_blocks import _banned_block_reject_message as _banned_block_reject_message
-from .banned_blocks import _challenge_http_request_reject_message as _challenge_http_request_reject_message
from .banned_blocks import _detect_new_banned_blocks as _detect_new_banned_blocks
-from .banned_blocks import _detect_timing_only_challenge_wait_blocks as _detect_timing_only_challenge_wait_blocks
from .banned_blocks import _record_banned_block_reject_span as _record_banned_block_reject_span
-from .banned_blocks import _timing_only_challenge_wait_reject_message as _timing_only_challenge_wait_reject_message
from .blockers import REPEATED_ACTION_STREAK_ABORT_AT as REPEATED_ACTION_STREAK_ABORT_AT
from .blockers import _active_block_run_budget_seconds as _active_block_run_budget_seconds
from .blockers import _analyze_run_blocks as _analyze_run_blocks
@@ -162,7 +154,6 @@ from .frontier import (
_workflow_with_runtime_frontier_starter_url_seed as _workflow_with_runtime_frontier_starter_url_seed,
)
from .guardrails import (
- _COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA,
_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,
_authority_tool_error,
)
@@ -172,9 +163,6 @@ from .guardrails import (
_request_policy_allows_update_and_skip_run,
)
from .guardrails import _turn_intent_tool_error as _turn_intent_tool_error
-from .guardrails import (
- _update_and_run_blocks_composition_evidence_precheck,
-)
from .mcp_hooks import _build_skyvern_mcp_overlays as _build_skyvern_mcp_overlays
from .mcp_hooks import _click_post_hook as _click_post_hook
from .mcp_hooks import _click_pre_hook as _click_pre_hook
@@ -250,20 +238,36 @@ from .workflow_update import _code_block_safety_errors as _code_block_safety_err
from .workflow_update import (
_impose_output_contract_envelope_after_steering as _impose_output_contract_envelope_after_steering,
)
-from .workflow_update import _metadata_contract_run_preflight_reject as _metadata_contract_run_preflight_reject
from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
from .workflow_update import _record_workflow_update_result as _record_workflow_update_result
-from .workflow_update import _run_dispatch_definition_reject as _run_dispatch_definition_reject
from .workflow_update import _scaffold_metadata_contract_for_update as _scaffold_metadata_contract_for_update
from .workflow_update import _update_workflow as _update_workflow
+from .workflow_update import carry_author_time_findings as carry_author_time_findings
from .workflow_update import (
consume_output_contract_advisory_grant_for_run_result as consume_output_contract_advisory_grant_for_run_result,
)
-from .workflow_update import metadata_same_key_terminal_preflight as metadata_same_key_terminal_preflight
LOG = structlog.get_logger()
+_CREDENTIAL_DEFERRED_DRAFT_MESSAGE = (
+ "I can save this as a draft without running it because the credentials aren't set up yet. "
+ "Add them in the Credentials UI and ask me to test the workflow."
+)
+
+
+def _mark_credential_deferred_draft(copilot_ctx: CopilotContext, result: dict[str, Any]) -> None:
+ """Credential-deferred drafts persist without a run, so they carry the same skip markers and
+ set the same flag the combined tool's skip branch does — credential-pause routing reads it."""
+ copilot_ctx.last_run_skipped_unbound_credentials = True
+ data = result.get("data")
+ if not isinstance(data, dict):
+ data = {}
+ result["data"] = data
+ data["skipped_run"] = True
+ data["skip_reason"] = "workflow_credential_inputs_unbound"
+ data["message"] = _CREDENTIAL_DEFERRED_DRAFT_MESSAGE
+
@function_tool(
name_override="update_workflow",
@@ -297,6 +301,9 @@ async def update_workflow_tool(
refs, observation refs, and terminal verifier expectations.
"""
copilot_ctx = ctx.context
+ # Mirrors the combined tool: a stale True from an earlier call in the same turn would
+ # misreport this call's authoring error as a credential ask.
+ copilot_ctx.last_run_skipped_unbound_credentials = False
serialized_code_artifact_metadata: object = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
arguments = {
@@ -308,52 +315,6 @@ async def update_workflow_tool(
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
credential_deferred_draft = _request_policy_allows_credential_deferred_draft(copilot_ctx)
- # A credential-deferred draft is redirected to update_and_run_blocks' skip-run
- # save path, which saves the draft and skips the browser run with the required
- # credential setup message.
- if credential_deferred_draft:
- agent_steering = (
- "Use update_and_run_blocks for this credential-deferred draft. It will save the workflow draft "
- "and skip the browser run with the required credential setup message."
- )
- user_facing = (
- "I can save this as a draft without running it because the credentials aren't set up yet. "
- "Add them in the Credentials UI and ask me to test the workflow."
- )
- signal = CopilotToolBlockerSignal(
- blocker_kind="authority_denied",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"update_and_run_blocks"}),
- internal_reason_code="request_policy_credential_deferred_redirect",
- blocked_tool="update_workflow",
- )
- payload = _emit_tool_blocker_signal(copilot_ctx, signal)
- result = {"ok": False, "error": payload}
- record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- return json.dumps(result)
-
- with copilot_span(
- "composition_evidence_precheck",
- data={**_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA, "tool_name": "update_workflow"},
- ):
- composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx,
- workflow_yaml,
- normalized_block_observation_refs,
- block_observation_refs,
- )
- if composition_evidence_error:
- result = {"ok": False, "error": composition_evidence_error}
- record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_workflow",
- result=result,
- )
- sanitized = sanitize_tool_result_for_llm("update_workflow", result)
- return json.dumps(sanitized)
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
@@ -364,8 +325,11 @@ async def update_workflow_tool(
"raw_code_artifact_metadata": code_artifact_metadata,
},
copilot_ctx,
- allow_missing_credentials=getattr(copilot_ctx, "allow_untested_workflow_draft", False) is True,
+ allow_missing_credentials=credential_deferred_draft
+ or getattr(copilot_ctx, "allow_untested_workflow_draft", False) is True,
)
+ if credential_deferred_draft and result.get("ok"):
+ _mark_credential_deferred_draft(copilot_ctx, result)
_record_workflow_update_result(copilot_ctx, result, prior_definition)
record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
if result.get("ok") is False:
@@ -459,21 +423,6 @@ async def run_blocks_tool(
if loop_error:
return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)
- definition_reject_result = _run_dispatch_definition_reject(copilot_ctx, copilot_ctx.workflow_yaml or "")
- if definition_reject_result is not None:
- record_tool_step_result_for_ctx(
- copilot_ctx,
- "run_blocks_and_collect_debug",
- arguments,
- definition_reject_result,
- )
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="run_blocks_and_collect_debug",
- result=definition_reject_result,
- )
- return json.dumps(definition_reject_result)
-
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
copilot_ctx, block_labels, prior_definition, prior_definition
@@ -669,69 +618,10 @@ async def update_and_run_blocks_tool(
serialized_code_artifact_metadata = scaffolded_code_artifact_metadata
arguments["code_artifact_metadata"] = serialized_code_artifact_metadata
- metadata_same_key_terminal = metadata_same_key_terminal_preflight(
- copilot_ctx,
- workflow_yaml,
- serialized_code_artifact_metadata,
- )
- if metadata_same_key_terminal is not None:
- record_tool_step_result_for_ctx(
- copilot_ctx,
- "update_and_run_blocks",
- arguments,
- metadata_same_key_terminal,
- )
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", metadata_same_key_terminal)
- return json.dumps(sanitized)
-
- if recorded_outcome_grounding_requires_current_page(copilot_ctx):
- loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
- if loop_error:
- return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
-
- metadata_contract_preflight_reject = _metadata_contract_run_preflight_reject(
- copilot_ctx,
- workflow_yaml,
- serialized_code_artifact_metadata,
- parameters or {},
- )
- if metadata_contract_preflight_reject is not None:
- record_tool_step_result_for_ctx(
- copilot_ctx,
- "update_and_run_blocks",
- arguments,
- metadata_contract_preflight_reject,
- )
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=metadata_contract_preflight_reject,
- )
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", metadata_contract_preflight_reject)
- return json.dumps(sanitized)
-
loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
if loop_error:
return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
- with copilot_span("composition_evidence_precheck", data=_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA):
- composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx,
- workflow_yaml,
- normalized_block_observation_refs,
- block_observation_refs,
- )
- if composition_evidence_error:
- result = {"ok": False, "error": composition_evidence_error}
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=result,
- )
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
- return json.dumps(sanitized)
-
_clear_pending_browser_interaction_observation(copilot_ctx)
# Snapshot the prior workflow definition BEFORE _update_workflow saves
@@ -782,6 +672,7 @@ async def update_and_run_blocks_tool(
"skip_reason": "workflow_credential_inputs_unbound",
},
}
+ carry_author_time_findings(update_result, skip_result)
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)
_record_diagnosis_repair_contract(
copilot_ctx,
@@ -795,29 +686,6 @@ async def update_and_run_blocks_tool(
)
return json.dumps(skip_result)
- exact_candidate_preflight_reject = _metadata_contract_run_preflight_reject(
- copilot_ctx,
- copilot_ctx.workflow_yaml or workflow_yaml,
- copilot_ctx.code_artifact_metadata,
- parameters or {},
- enforce_untagged_declared_inputs=True,
- )
- if exact_candidate_preflight_reject is not None:
- record_tool_step_result_for_ctx(
- copilot_ctx,
- "update_and_run_blocks",
- arguments,
- exact_candidate_preflight_reject,
- )
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=exact_candidate_preflight_reject,
- workflow_updated=True,
- )
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", exact_candidate_preflight_reject)
- return json.dumps(sanitized)
-
# Step 2: Compute frontier and run the blocks.
new_definition = None
if copilot_ctx.last_workflow is not None:
@@ -832,6 +700,7 @@ async def update_and_run_blocks_tool(
data = result.get("data")
if isinstance(data, dict):
data["workflow_updated"] = True
+ carry_author_time_findings(update_result, result)
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
_record_diagnosis_repair_contract(
copilot_ctx,
@@ -865,6 +734,7 @@ async def update_and_run_blocks_tool(
run_result,
completion_verification,
)
+ carry_author_time_findings(update_result, tool_visible_result)
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, tool_visible_result)
_record_diagnosis_repair_contract(
copilot_ctx,
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index 8100bb7fd..44114d409 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -1,18 +1,15 @@
from __future__ import annotations
-import re
from dataclasses import dataclass
from enum import StrEnum
from typing import Any
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
-from skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT
-from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from ._shared import _iter_yaml_blocks, _parse_workflow_blocks
+from ._shared import _parse_workflow_blocks
class CopilotBlockPolicyStatus(StrEnum):
@@ -413,112 +410,3 @@ def _detect_new_banned_blocks(
prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
prior_labels = {label for label, _ in _collect_banned_block_items(prior_blocks or [], active_banned_types)}
return [(label, block_type) for label, block_type in submitted_items if label not in prior_labels]
-
-
-_CHALLENGE_WAIT_PATTERN = re.compile(
- r"\b(anti[-_\s]?bot|bot[-_\s]?block|captcha|challenge|human[-_\s]?verification|ip[-_\s]?block|waf)\b",
- re.IGNORECASE,
-)
-
-
-def _has_confirmed_waf_or_site_block(ctx: Any) -> bool:
- if getattr(ctx, "last_test_anti_bot", None):
- return True
- return _get_int_attr(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT
-
-
-def _get_int_attr(ctx: Any, name: str, default: int = 0) -> int:
- value = getattr(ctx, name, default)
- return value if isinstance(value, int) else default
-
-
-def _block_challenge_wait_text(block: dict[str, Any]) -> str:
- values = []
- for key in ("label", "title", "description", "navigation_goal", "complete_criterion"):
- value = block.get(key)
- if isinstance(value, str):
- values.append(value)
- return " ".join(values)
-
-
-def _detect_timing_only_challenge_wait_blocks(submitted_yaml: str | None) -> list[str]:
- submitted_blocks = _parse_workflow_blocks(submitted_yaml)
- if submitted_blocks is None:
- return []
- labels: list[str] = []
- for block in _iter_yaml_blocks(submitted_blocks):
- raw_type = block.get("block_type")
- if not isinstance(raw_type, str) or raw_type.strip().lower() != "wait":
- continue
- label = block.get("label")
- if not isinstance(label, str):
- continue
- if _CHALLENGE_WAIT_PATTERN.search(_block_challenge_wait_text(block)):
- labels.append(label)
- return labels
-
-
-def _composition_evidence_has_challenge(ctx: AgentContext) -> bool:
- evidence = getattr(ctx, "composition_page_evidence", None)
- if not isinstance(evidence, dict):
- return False
- if evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"):
- return True
- challenge_state = evidence.get("challenge_state")
- return isinstance(challenge_state, dict) and challenge_state.get("detected") is True
-
-
-def _detect_new_http_request_blocks(submitted_yaml: str | None, prior_workflow_yaml: str | None) -> list[str]:
- submitted_blocks = _parse_workflow_blocks(submitted_yaml)
- if submitted_blocks is None:
- return []
- prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
- prior_labels: set[str] = set()
- for block in _iter_yaml_blocks(prior_blocks or []):
- if str(block.get("block_type") or "").strip().lower() != "http_request":
- continue
- label = block.get("label")
- if isinstance(label, str):
- prior_labels.add(label)
- labels: list[str] = []
- for block in _iter_yaml_blocks(submitted_blocks):
- if str(block.get("block_type") or "").strip().lower() != "http_request":
- continue
- label = block.get("label")
- if isinstance(label, str) and label not in prior_labels:
- labels.append(label)
- return labels
-
-
-def _challenge_http_request_reject_message(
- ctx: AgentContext, submitted_yaml: str | None, prior_workflow_yaml: str | None
-) -> str | None:
- if not _composition_evidence_has_challenge(ctx):
- return None
- labels = _detect_new_http_request_blocks(submitted_yaml, prior_workflow_yaml)
- if not labels:
- return None
- labels_text = ", ".join(sorted(set(labels)))
- return (
- f"{INTERNAL_VALIDATION_FAILURE_PREFIX}raw http_request blocks are not allowed for a page with observed "
- "anti-bot or human-verification challenge evidence. "
- f"Offending labels: [{labels_text}]. "
- "Use browser workflow blocks grounded in the observed page, include challenge handling only when visible, "
- "or stop and report the observed challenge blocker if it cannot be completed."
- )
-
-
-def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | None) -> str | None:
- if not _has_confirmed_waf_or_site_block(ctx):
- return None
- labels = _detect_timing_only_challenge_wait_blocks(submitted_yaml)
- if not labels:
- return None
- labels_text = ", ".join(sorted(set(labels)))
- return (
- f"{INTERNAL_VALIDATION_FAILURE_PREFIX}timing-only challenge wait blocks are not allowed after confirmed "
- "anti-bot/WAF or repeated site-block evidence. "
- f"Offending labels: [{labels_text}]. "
- "Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "
- "real action, try a materially different proxy/source if allowed, or stop and explain the blocker."
- )
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index c741098b1..d7e3e85b3 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -17,10 +17,6 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
loop_blocker_evidence_from_ctx,
refresh_held_loop_blocker_evidence,
)
-from skyvern.forge.sdk.copilot.build_test_outcome import (
- maybe_satisfy_recorded_outcome_grounding_requirement,
- recorded_outcome_grounding_requires_current_page,
-)
from skyvern.forge.sdk.copilot.challenge_evidence import (
artifact_challenge_flag_key,
is_carrier_backed_category_entry,
@@ -33,7 +29,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
TOTAL_TIMEOUT_SECONDS,
synthesized_block_persistence_signal,
terminal_challenge_blocker_signal_from_current_page_evidence,
- uncovered_output_reject_scout_steer_signal,
)
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
@@ -45,9 +40,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOA
from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
- AuthorTimeGateAblationPayload,
output_contract_ladder_unresolved,
- record_author_time_gate_ablation_event,
)
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
from skyvern.schemas.workflows import BlockType
@@ -82,18 +75,6 @@ _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS = (
| frozenset({"click", "navigate_browser", "press_key", "scroll", "select_option", "type_text"})
)
_OUTPUT_CONTRACT_LADDER_AUTHORING_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})
-_RECORDED_OUTCOME_GROUNDING_MUTATION_TOOLS = BLOCK_RUNNING_TOOLS | frozenset(
- {
- "update_workflow",
- "click",
- "fill_credential_field",
- "navigate_browser",
- "press_key",
- "scroll",
- "select_option",
- "type_text",
- }
-)
async def _safe_read_workflow_run(
@@ -1520,27 +1501,59 @@ def _challenge_gated_anti_bot_rerun_signal(
)
+# The complete vocabulary the runtime/loop plane may refuse with. It exists for the same reason
+# AUTHOR_TIME_HARD_BLOCKS does: a check added here cannot wall a turn without appearing in this set.
+# A refusal keyed on what the submitted draft says is an author-time decision, not a runtime one.
+LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset(
+ {
+ "loop_detected_credential_or_parameter_misconfig",
+ "loop_detected_repeated_failed_step",
+ "loop_detected_consecutive_same_tool",
+ "loop_detected_generic",
+ "loop_detected_no_forward_progress_interaction",
+ "loop_detected_discovery_exhausted_no_entry_url",
+ "code_authoring_guardrail_churn",
+ "credential_priority_authoring_churn",
+ "tool_error_active_run_terminal_evidence",
+ "tool_error_terminal_challenge_blocker",
+ "tool_error_synthesized_block_persistence_required",
+ "tool_error_challenge_gated_submit_disabled",
+ "tool_error_late_block_running",
+ "tool_error_non_retriable_nav",
+ "tool_error_pending_reconciliation_no_input",
+ "tool_error_pending_reconciliation_requires_input",
+ "tool_error_per_tool_budget_rerun",
+ "tool_error_post_budget_page_inspection_required",
+ "tool_error_post_budget_upstream_replay",
+ "tool_error_post_budget_challenge_result_evidence",
+ "tool_error_post_budget_challenge_blocker",
+ "tool_error_repeated_action_abort",
+ "runtime_self_heal_native_tool_blocked",
+ }
+)
+
+
+def _emit_loop_plane_refusal(ctx: AgentContext, signal: CopilotToolBlockerSignal) -> str:
+ reason = signal.internal_reason_code
+ if reason not in LOOP_PLANE_REFUSAL_REASON_CODES:
+ raise ValueError(
+ f"{reason!r} is not a declared loop-plane refusal. A check that refuses because of what "
+ "the submitted draft says is an author-time decision: mint an AuthorTimeBlock, or return a "
+ f"finding. Declared: {sorted(LOOP_PLANE_REFUSAL_REASON_CODES)}"
+ )
+ return _emit_tool_blocker_signal(ctx, signal)
+
+
def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
refresh_held_loop_blocker_evidence(ctx)
if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS:
current_page_challenge_signal = _current_page_terminal_challenge_signal(ctx, arguments, tool_name)
if current_page_challenge_signal is not None:
- return _emit_tool_blocker_signal(ctx, current_page_challenge_signal)
-
- uncovered_output_steer = uncovered_output_reject_scout_steer_signal(ctx, tool_name)
- if uncovered_output_steer is not None:
- return _emit_tool_blocker_signal(ctx, uncovered_output_steer)
+ return _emit_loop_plane_refusal(ctx, current_page_challenge_signal)
persistence_signal = synthesized_block_persistence_signal(ctx, tool_name, arguments)
if persistence_signal is not None:
- grounding_signal = _recorded_outcome_grounding_signal(ctx, tool_name)
- if grounding_signal is not None:
- LOG.info(
- "copilot recorded outcome grounding enforced over persistence force",
- tool_name=tool_name,
- )
- return _emit_tool_blocker_signal(ctx, grounding_signal)
- return _emit_tool_blocker_signal(ctx, persistence_signal)
+ return _emit_loop_plane_refusal(ctx, persistence_signal)
# While a typed output-contract actuation ladder is still unresolved for the authoring tools, the
# bounded ladder (not a generic loop backstop) owns the turn's outcome; the guards re-engage the moment
@@ -1548,15 +1561,11 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
output_contract_owns_turn = (
tool_name in _OUTPUT_CONTRACT_LADDER_AUTHORING_TOOLS and output_contract_ladder_unresolved(ctx)
)
- grounding_owns_mutation = (
- tool_name in _RECORDED_OUTCOME_GROUNDING_MUTATION_TOOLS
- and recorded_outcome_grounding_requires_current_page(ctx)
- )
- if not output_contract_owns_turn and not grounding_owns_mutation:
+ if not output_contract_owns_turn:
detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
if detected is not None:
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
_build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),
)
@@ -1571,11 +1580,10 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
isinstance(tracker, list)
and tool_name not in _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS
and not output_contract_owns_turn
- and not grounding_owns_mutation
):
detected = detect_tool_loop(tracker, tool_name, arguments)
if detected is not None:
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
_build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),
)
@@ -1583,7 +1591,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
if tool_name == "update_workflow" or tool_name in BLOCK_RUNNING_TOOLS:
active_terminal_signal = _active_run_terminal_evidence_signal(ctx, tool_name)
if active_terminal_signal is not None:
- return _emit_tool_blocker_signal(ctx, active_terminal_signal)
+ return _emit_loop_plane_refusal(ctx, active_terminal_signal)
# Hard-abort when the agent has re-fired the same action sequence against
# the page N times without intervening success. This is the signal that
@@ -1603,38 +1611,38 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
if isinstance(pending_run_id, str) and pending_run_id:
if getattr(ctx, "pending_reconciliation_requires_user_input", False) is True:
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
_pending_reconciliation_requires_input_signal(
pending_run_id=pending_run_id, blocked_tool=tool_name
),
)
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
_pending_reconciliation_no_input_signal(pending_run_id=pending_run_id, blocked_tool=tool_name),
)
inspection_signal = _post_budget_page_inspection_signal(ctx, tool_name)
if inspection_signal is not None:
- return _emit_tool_blocker_signal(ctx, inspection_signal)
+ return _emit_loop_plane_refusal(ctx, inspection_signal)
# Terminal anti-bot evidence should produce the final user-facing reply
# before the generic budget rerun path can ask for another attempt.
post_budget_challenge_signal = _post_budget_terminal_challenge_signal(ctx, arguments, tool_name)
if post_budget_challenge_signal is not None:
- return _emit_tool_blocker_signal(ctx, post_budget_challenge_signal)
+ return _emit_loop_plane_refusal(ctx, post_budget_challenge_signal)
budget_signal = _per_tool_budget_problem_rerun_signal(ctx, arguments, tool_name)
if budget_signal is not None:
- return _emit_tool_blocker_signal(ctx, budget_signal)
+ return _emit_loop_plane_refusal(ctx, budget_signal)
upstream_replay_signal = _post_budget_upstream_replay_signal(ctx, arguments, tool_name)
if upstream_replay_signal is not None:
- return _emit_tool_blocker_signal(ctx, upstream_replay_signal)
+ return _emit_loop_plane_refusal(ctx, upstream_replay_signal)
challenge_signal = _challenge_gated_anti_bot_rerun_signal(ctx, arguments, tool_name)
if challenge_signal is not None:
- return _emit_tool_blocker_signal(ctx, challenge_signal)
+ return _emit_loop_plane_refusal(ctx, challenge_signal)
streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)
streak = streak_raw if isinstance(streak_raw, int) else 0
@@ -1650,7 +1658,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
user_facing = (
"I tried the same actions a few times without making progress. The site looks blocked. I'll stop here."
)
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
CopilotToolBlockerSignal(
blocker_kind="tool_error",
@@ -1680,7 +1688,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
"explaining the failure and asking them to verify the URL."
)
user_facing = "The URL I tried isn't reachable. Tell me the correct address and I'll try again."
- return _emit_tool_blocker_signal(
+ return _emit_loop_plane_refusal(
ctx,
CopilotToolBlockerSignal(
blocker_kind="tool_error",
@@ -1695,59 +1703,13 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
late_signal = _late_block_running_call_signal(ctx, tool_name)
if late_signal is not None:
- return _emit_tool_blocker_signal(ctx, late_signal)
- grounding_signal = _recorded_outcome_grounding_signal(ctx, tool_name)
- if grounding_signal is not None:
- return _emit_tool_blocker_signal(ctx, grounding_signal)
+ return _emit_loop_plane_refusal(ctx, late_signal)
return None
_build_loop_blocker_signal = build_loop_blocker_signal
-def _recorded_outcome_grounding_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- if tool_name not in _RECORDED_OUTCOME_GROUNDING_MUTATION_TOOLS:
- return None
- if maybe_satisfy_recorded_outcome_grounding_requirement(ctx):
- return None
- if not recorded_outcome_grounding_requires_current_page(ctx):
- return None
- requirement = ctx.recorded_outcome_grounding_requirement
- if requirement is None:
- return None
- payload: AuthorTimeGateAblationPayload = {
- "phase": requirement.phase,
- "outcome_reason_code": requirement.reason_code,
- "workflow_run_id": requirement.workflow_run_id,
- "block_labels": list(requirement.block_labels),
- "required_tool": requirement.required_tool,
- "required_target_url": requirement.required_target_url,
- }
- if record_author_time_gate_ablation_event(
- ctx,
- gate_id="recorded_outcome_grounding",
- reason_code="recorded_outcome_grounding_required",
- fingerprint=requirement.structural_key,
- blocked_tool=tool_name,
- payload=payload,
- ):
- return None
- return CopilotToolBlockerSignal(
- blocker_kind="missing_required_context",
- agent_steering_text=(
- "The repeated recorded build-test outcome is still ungrounded. Call "
- 'inspect_page_for_composition(target_url="current_page") and use that bounded page '
- "evidence before the next workflow mutation or browser mutation."
- ),
- user_facing_reason="I need to inspect the current page before changing the workflow again.",
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"inspect_page_for_composition"}),
- renders_final_reply=False,
- internal_reason_code="recorded_outcome_grounding_required",
- blocked_tool=tool_name,
- )
-
-
def _analyze_run_blocks(
result: dict[str, Any], copilot_ctx: Any | None = None
) -> tuple[str | None, bool, list[dict] | None]:
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index fe6364179..663c66be7 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -7,18 +7,13 @@ from typing import Any
import structlog
from agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData
+from skyvern.forge.sdk.copilot.author_time_block import CREDENTIAL_SCOUT_BLOCK_ID, AuthorTimeBlock
from skyvern.forge.sdk.copilot.blocker_signal import BlockerKind, CopilotToolBlockerSignal, RecoveryHint
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
from skyvern.forge.sdk.copilot.build_test_outcome import (
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
)
-from skyvern.forge.sdk.copilot.composition_evidence import (
- composition_page_evidence_error,
- turn_has_scout_interaction,
- workflow_target_url,
-)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.credential_literal_rebind import (
CredentialRebindResult,
rebind_scouted_credential_literals,
@@ -29,11 +24,11 @@ from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_res
from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyReason,
OutputPolicyVerdict,
+ demote_author_time_steer_reasons,
evaluate_output_policy,
format_output_policy_tool_error,
output_policy_verdict_to_trace_data,
)
-from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget, code_is_download_intent
from skyvern.forge.sdk.copilot.request_policy import CREDENTIAL_DEFERRED_DRAFT_REASONS, RequestPolicy
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.turn_intent import (
@@ -60,7 +55,6 @@ from ._shared import (
_emit_tool_blocker_signal,
_workflow_yaml_blocks_by_label,
)
-from .banned_blocks import _copilot_block_authoring_policy
LOG = structlog.get_logger()
@@ -102,11 +96,14 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
# at the log seam (`redact_registered_secrets`) and the persistence seam; final-output surfaces
# still hard-block on RAW_SECRET_LEAK via `_FINAL_OUTPUT_HARD_BLOCK_REASONS`.
verdict.remove(OutputPolicyReason.RAW_SECRET_LEAK)
+ steered_reasons = demote_author_time_steer_reasons(verdict)
trace_data = output_policy_verdict_to_trace_data(
verdict,
surface="tool_input",
tool_name=getattr(tool_context, "tool_name", None),
)
+ if steered_reasons:
+ trace_data = {**trace_data, "steered_reason_codes": [reason.value for reason in steered_reasons]}
residual_selectors = rebind_result.residual_selectors if rebind_result is not None else ()
if verdict.allowed and residual_selectors:
# The rebind could not bind these selectors to a parameter. That used to reject the turn; the
@@ -125,18 +122,33 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
LOG.info("copilot output policy tool guardrail verdict", **trace_data)
return ToolGuardrailFunctionOutput.allow(output_info=trace_data)
LOG.info("copilot output policy tool guardrail verdict", **trace_data)
- error = format_output_policy_tool_error(verdict)
+ block = AuthorTimeBlock(block_id=CREDENTIAL_SCOUT_BLOCK_ID, error=format_output_policy_tool_error(verdict))
+ trace_data = {**trace_data, "block_id": block.block_id}
+ tool_name = getattr(tool_context, "tool_name", None)
+ if isinstance(tool_name, str) and tool_name:
+ _record_output_policy_guardrail_churn(
+ getattr(tool_context, "context", None), tool_name, effective_yaml, verdict
+ )
+ return _guardrail_block(tool_context, tool_arguments, block, trace_data)
+
+
+def _guardrail_block(
+ tool_context: Any,
+ tool_arguments: dict[str, Any],
+ block: AuthorTimeBlock,
+ trace_data: dict[str, Any],
+) -> ToolGuardrailFunctionOutput:
+ """The tool-input half of the author-time decision point: only an ``AuthorTimeBlock``
+ turns a guardrail verdict into a rejection the model cannot author past."""
tool_name = getattr(tool_context, "tool_name", None)
- copilot_ctx = getattr(tool_context, "context", None)
if isinstance(tool_name, str) and tool_name:
record_consecutive_tool_result_boundary_for_ctx(
- copilot_ctx,
+ getattr(tool_context, "context", None),
tool_name,
- {"ok": False, "error": error},
+ {"ok": False, "error": block.error, "block_id": block.block_id},
arguments=tool_arguments,
)
- _record_output_policy_guardrail_churn(copilot_ctx, tool_name, effective_yaml, verdict)
- return ToolGuardrailFunctionOutput.reject_content(error, output_info=trace_data)
+ return ToolGuardrailFunctionOutput.reject_content(block.error, output_info=trace_data)
def _rebind_scouted_credential_literals_in_place(
@@ -231,81 +243,6 @@ _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL = ToolInputGuardrail(
name="workflow_yaml_output_policy_guardrail",
)
-_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA = {
- "surface": "tool_pre_side_effect",
- "tool_name": "update_and_run_blocks",
- "reason": "composition_page_evidence",
-}
-
-
-def _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx: Any,
- workflow_yaml: str | None,
- normalized_block_observation_refs: dict[str, int],
- raw_block_observation_refs: Any,
-) -> str | None:
- if copilot_ctx is None or workflow_yaml is None:
- LOG.warning(
- "update_and_run_blocks composition evidence precheck missing context or workflow yaml",
- has_context=copilot_ctx is not None,
- has_workflow_yaml=workflow_yaml is not None,
- )
- return None
-
- evidence_error = composition_page_evidence_error(
- copilot_ctx,
- workflow_yaml,
- block_observation_refs=normalized_block_observation_refs,
- raw_block_observation_refs=raw_block_observation_refs,
- )
-
- if evidence_error:
- LOG.info(
- "copilot composition page evidence pre-side-effect rejected workflow",
- workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
- target_url=workflow_target_url(workflow_yaml),
- surface="tool_pre_side_effect",
- )
- return evidence_error
-
- return None
-
-
-_DOWNLOAD_SCOUT_REQUIRED_STEERING = (
- "This block downloads a file, and the terminal download step is compiled from the observed "
- "download affordance. Scout it first: reach the page that exposes the download control and "
- "call skyvern_evaluate to observe it (skyvern_evaluate cannot click — use the click tool to "
- "act), then re-author the block. The terminal download step will be compiled for you from the "
- "observed target, so you do not author the expect_download idiom yourself."
-)
-# After this many scout-act rejections in a turn, stop blocking and let the model halt honestly: a
-# download whose affordance the scout never resolves is not authorable, so looping author->scout
-# wastes turns. A genuinely scoutable download clears the gate on the first re-author.
-_DOWNLOAD_SCOUT_REQUIRED_MAX_REJECTIONS = 3
-_DOWNLOAD_SCOUT_UNREACHABLE_HALT = (
- "Repeated scout-acts did not resolve a single download affordance on this page, so the download "
- "step cannot be compiled. Tell the user you could not locate the download control and ask them to "
- "confirm where the file downloads from; do not author a hand-rolled download block."
-)
-
-
-def _download_intent_block_labels(workflow_yaml: str | None) -> list[str]:
- labels: list[str] = []
- for label, block in _workflow_yaml_blocks_by_label(workflow_yaml).items():
- if block.get("block_type") != "code":
- continue
- code = block.get("code")
- if isinstance(code, str) and code_is_download_intent(code):
- labels.append(label)
- return labels
-
-
-def _has_reached_download_target(ctx: Any) -> bool:
- target = getattr(ctx, "reached_download_target", None)
- if not isinstance(target, ReachedDownloadTarget):
- return False
- return target.already_registered or bool(target.selector.strip())
-
def _request_policy_allows_untested_code_block_draft(ctx: Any) -> bool:
policy = getattr(ctx, "request_policy", None)
@@ -320,114 +257,6 @@ def _request_policy_allows_untested_code_block_draft(ctx: Any) -> bool:
)
-def _download_scout_required_error(copilot_ctx: Any, workflow_yaml: str | None) -> str | None:
- """Reject authoring a download-intent code block until the affordance has been scout-acted
- this turn, so the skyvern_evaluate post-hook can populate the reached-download target and the
- synthesizer can compile the terminal download step. Active in code-only browser mode."""
- if _copilot_block_authoring_policy(copilot_ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- if copilot_ctx is None or workflow_yaml is None:
- return None
- download_labels = _download_intent_block_labels(workflow_yaml)
- if not download_labels:
- return None
- if _request_policy_allows_untested_code_block_draft(copilot_ctx):
- LOG.info(
- "copilot download scout-act gate skipped for untested credential draft",
- workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
- download_intent_block_labels=download_labels,
- surface="tool_pre_side_effect",
- )
- return None
- if _has_reached_download_target(copilot_ctx):
- return None
- if turn_has_scout_interaction(copilot_ctx):
- return None
- prior_rejections = int(getattr(copilot_ctx, "download_scout_required_rejections", 0) or 0)
- if prior_rejections >= _DOWNLOAD_SCOUT_REQUIRED_MAX_REJECTIONS:
- LOG.info(
- "copilot download scout-act gate exhausted retries; halting honestly",
- workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
- download_intent_block_labels=download_labels,
- download_scout_required_rejections=prior_rejections,
- surface="tool_pre_side_effect",
- )
- return _DOWNLOAD_SCOUT_UNREACHABLE_HALT
- if hasattr(copilot_ctx, "download_scout_required_rejections"):
- copilot_ctx.download_scout_required_rejections = prior_rejections + 1
- LOG.info(
- "copilot download scout-act pre-side-effect rejected workflow",
- workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
- download_intent_block_labels=download_labels,
- download_scout_required_rejections=prior_rejections + 1,
- surface="tool_pre_side_effect",
- )
- return _DOWNLOAD_SCOUT_REQUIRED_STEERING
-
-
-def _submitted_code_blocks(workflow_yaml: str | None) -> list[str]:
- codes: list[str] = []
- for block in _workflow_yaml_blocks_by_label(workflow_yaml).values():
- if block.get("block_type") != "code":
- continue
- code = block.get("code")
- if isinstance(code, str):
- codes.append(code)
- return codes
-
-
-_DOWNLOAD_BINDING_REQUIRED_STEERING = (
- "A correct scout-act reached a download affordance on the current page, but the authored block "
- "does not fire the browser download. Author ONE terminal download code block that clicks the "
- "captured target with the expect_download idiom; the captured selector is `{selector}`"
- "{affordance_hint}. The terminal download step is compiled for you when you scout-act the "
- "affordance, so re-author the block against the reached target rather than a static fetch."
-)
-_DOWNLOAD_BINDING_UNREACHABLE_HALT = (
- "The reached download affordance could not be bound to a terminal download step after repeated "
- "attempts. Tell the user you reached the download control but could not author the download step, "
- "and ask them to confirm where the file downloads from; do not author a download-less block."
-)
-
-
-def _download_binding_required_error(ctx: AgentContext | None, workflow_yaml: str | None) -> str | None:
- """Reject a code block authored after the scout reached a download affordance when that block does
- not fire the browser download. Keyed on the typed `ctx.reached_download_target` the model cannot
- edit, not on the submitted code form. Active in code-only browser mode."""
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- if ctx is None or workflow_yaml is None:
- return None
- target = getattr(ctx, "reached_download_target", None)
- if not isinstance(target, ReachedDownloadTarget) or target.already_registered:
- return None
- if any(code_is_download_intent(code) for code in _submitted_code_blocks(workflow_yaml)):
- return None
- # Shared per-turn download-steering budget: this gate and _download_scout_required_error draw down the
- # same ctx.download_scout_required_rejections counter (combined cap, not 3 per gate).
- prior_rejections = int(getattr(ctx, "download_scout_required_rejections", 0) or 0)
- if prior_rejections >= _DOWNLOAD_SCOUT_REQUIRED_MAX_REJECTIONS:
- LOG.info(
- "copilot download binding gate exhausted retries; halting honestly",
- workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),
- reached_download_selector=target.selector,
- download_scout_required_rejections=prior_rejections,
- surface="tool_pre_side_effect",
- )
- return _DOWNLOAD_BINDING_UNREACHABLE_HALT
- if hasattr(ctx, "download_scout_required_rejections"):
- ctx.download_scout_required_rejections = prior_rejections + 1
- LOG.info(
- "copilot download binding pre-side-effect rejected workflow",
- workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),
- reached_download_selector=target.selector,
- download_scout_required_rejections=prior_rejections + 1,
- surface="tool_pre_side_effect",
- )
- affordance_hint = f" (affordance text: {target.affordance_text})" if target.affordance_text else ""
- return _DOWNLOAD_BINDING_REQUIRED_STEERING.format(selector=target.selector, affordance_hint=affordance_hint)
-
-
def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
policy = getattr(ctx, "request_policy", None)
if not isinstance(policy, RequestPolicy):
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index cd8993de0..fa7e88a5a 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -426,36 +426,6 @@ async def _screenshot_post_hook(
return result
-async def _maybe_replay_captured_never_captured_obligation(result: dict[str, Any], ctx: AgentContext) -> None:
- """Complete the capture/reconciliation handshake without another model-authored update call."""
- try:
- from .workflow_update import _replay_captured_never_captured_obligation
-
- replay = await _replay_captured_never_captured_obligation(ctx)
- except Exception:
- # The browser action already succeeded. Preserve its capture so the turn can
- # retry reconciliation instead of converting a save failure into capture loss.
- LOG.warning("copilot_never_captured_obligation_replay_failed", exc_info=True)
- return
- if replay is None:
- return
- result_data = result.get("data")
- if not isinstance(result_data, dict):
- result_data = {}
- result["data"] = result_data
- replay_data = replay.get("data")
- reconciliation: dict[str, Any] = {"ok": replay.get("ok") is True}
- if replay.get("ok") is True:
- message = replay_data.get("message") if isinstance(replay_data, dict) else None
- reconciliation["message"] = message or "Workflow reconciled after capturing the missing browser action."
- else:
- if replay.get("user_facing_summary"):
- reconciliation["user_facing_summary"] = replay["user_facing_summary"]
- if replay.get("error"):
- reconciliation["error"] = replay["error"]
- result_data["workflow_reconciliation"] = reconciliation
-
-
async def _click_post_hook(
result: dict[str, Any],
raw: dict[str, Any],
@@ -521,7 +491,6 @@ async def _click_post_hook(
await _attach_reperception_targets_on_non_advancing_click(result, raw, ctx, attempted_selector)
except Exception:
LOG.warning("copilot_click_reperception_attach_failed", exc_info=True)
- await _maybe_replay_captured_never_captured_obligation(result, ctx)
return result
@@ -839,7 +808,6 @@ async def _type_text_post_hook(
result["data"]["observation_step"] = observation_step
if page_evidence is not None:
_attach_scout_page_summary(result, page_evidence)
- await _maybe_replay_captured_never_captured_obligation(result, ctx)
return result
@@ -948,7 +916,6 @@ async def _select_option_post_hook(
result["data"]["observation_step"] = observation_step
if page_evidence is not None:
_attach_scout_page_summary(result, page_evidence)
- await _maybe_replay_captured_never_captured_obligation(result, ctx)
return result
@@ -976,7 +943,6 @@ async def _press_key_post_hook(
source_url=source_url,
key=data.get("key", ""),
)
- await _maybe_replay_captured_never_captured_obligation(result, ctx)
return result
@@ -1061,20 +1027,19 @@ def _build_skyvern_mcp_overlays(
),
"click": SchemaOverlay(
description=(
- "Click an element in the browser. Prefer a CSS selector ALONE for a target "
- "you can identify from page evidence — a selector-only click is instant and "
- "deterministic. Use `intent` only when you cannot derive a selector: an "
- "`intent`-only click routes through a slower full-page AI scan, and if you "
- "pass both, the selector wins and the `intent` is ignored. When a shared class "
- "matches many elements (e.g. one button per result row), scope the selector to "
- "the specific item (its container, a unique attribute, or :nth-of-type) instead "
- "of relying on `intent` to disambiguate. "
+ "Click an element in the browser by CSS selector. The click is instant and "
+ "deterministic. Derive the selector from page evidence — the selectors reported "
+ "by page inspection are verified to match exactly one element. When a shared "
+ "class matches many elements (e.g. one button per result row), scope the "
+ "selector to the specific item (its container, a unique attribute, or "
+ ":nth-of-type). If a selector does not resolve, inspect the page again and "
+ "derive a better one. "
"IMPORTANT: jQuery pseudo-selectors like :contains(), :eq(), :first, "
":visible are NOT valid CSS. Use standard selectors: "
"'button.download', 'a[href*=\"pdf\"]', '#submit-btn', "
"'table tr:nth-of-type(2) td a'."
),
- hide_params=frozenset({"session_id", "cdp_url", "button", "click_count"}),
+ hide_params=frozenset({"session_id", "cdp_url", "button", "click_count", "intent"}),
forced_args={"selector_mode": "direct"},
requires_browser=True,
timeout=15,
@@ -1083,18 +1048,16 @@ def _build_skyvern_mcp_overlays(
),
"type_text": SchemaOverlay(
description=(
- "Type text into an input element. Prefer a CSS selector ALONE to target the "
- "field — a selector-only type is instant and deterministic. Use `intent` only "
- "when you cannot derive a selector: an `intent`-only type routes through a "
- "slower full-page AI scan, and if you pass both, the selector wins and the "
- "`intent` is ignored. "
+ "Type text into an input element by CSS selector. The type is instant and "
+ "deterministic. Derive the selector from page evidence; if it does not resolve, "
+ "inspect the page again and derive a better one. "
"Optionally clear the field first. Use this for form filling. "
"NEVER type inline passwords, API keys, tokens, cookies, TOTP/OTP "
"codes, private keys, or other raw credentials/secrets received in "
"chat — stop and follow the CREDENTIAL HANDLING refusal rule in the "
"system prompt instead."
),
- hide_params=frozenset({"session_id", "cdp_url", "delay"}),
+ hide_params=frozenset({"session_id", "cdp_url", "delay", "intent"}),
forced_args={"selector_mode": "direct"},
required_overrides=["text"],
arg_transforms={"clear_first": "clear"},
@@ -1125,11 +1088,10 @@ def _build_skyvern_mcp_overlays(
"select_option": SchemaOverlay(
description=(
"Select an option from a <select> dropdown. Provide the value to select and a "
- "selector to target the element precisely; use `intent` (alone) only when you "
- "cannot derive a selector — passing both lets the selector win and ignores the "
- "`intent`. For free-text inputs, use type_text instead."
+ "CSS selector to target the element precisely. For free-text inputs, use "
+ "type_text instead."
),
- hide_params=frozenset({"session_id", "cdp_url", "timeout"}),
+ hide_params=frozenset({"session_id", "cdp_url", "timeout", "intent"}),
forced_args={"selector_mode": "direct"},
required_overrides=["value"],
requires_browser=True,
@@ -1140,10 +1102,10 @@ def _build_skyvern_mcp_overlays(
"press_key": SchemaOverlay(
description=(
"Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.). "
- "Optionally focus an element first via selector or intent. "
+ "Optionally focus an element first via CSS selector. "
"Use for form submission, tab navigation, or closing dialogs."
),
- hide_params=frozenset({"session_id", "cdp_url"}),
+ hide_params=frozenset({"session_id", "cdp_url", "intent"}),
required_overrides=["key"],
requires_browser=True,
pre_hook=_press_key_pre_hook,
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index a9842fb19..83bcca749 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -76,7 +76,6 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
build_diagnosis_repair_contract,
)
from skyvern.forge.sdk.copilot.enforcement import (
- consume_uncovered_output_reopen_event,
repair_ceiling_stop_signal,
reset_no_progress_interaction_count,
)
@@ -207,7 +206,7 @@ from .guardrails import (
_placeholder_for_parameter_type,
)
from .scouting import _mark_page_inspected, _mark_post_run_page_observed
-from .workflow_update import output_contract_value_bearing_run_reject, record_output_contract_run_output_evidence
+from .workflow_update import record_output_contract_run_output_evidence
LOG = structlog.get_logger()
@@ -1548,23 +1547,6 @@ async def _run_blocks_and_collect_debug(
ctx.last_executed_block_labels = []
return runtime_security_failure
- # The lane asserts a saved-workflow property, so its evidence set is every saved code
- # block, never just the selected subset (security lanes above stay selection-scoped).
- value_bearing_reject = output_contract_value_bearing_run_reject(
- ctx,
- {
- code_input.label: code_input.code
- for code_input in _selected_code_security_inputs(
- _workflow_definition_blocks_for_code_security(workflow.workflow_definition),
- selected_labels=set(),
- include_descendants=True,
- )
- },
- )
- if value_bearing_reject is not None:
- ctx.last_executed_block_labels = []
- return value_bearing_reject
-
credential_ids = list(
dict.fromkeys(
_extract_credential_ids_from_tool_value(params.get("parameters") or {})
@@ -3505,14 +3487,6 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRe
if progressed:
reset_no_progress_interaction_count(copilot_ctx)
- if not progressed and consume_uncovered_output_reopen_event(copilot_ctx):
- contract.repair_loop_state = RepairLoopState(
- streak_token=copilot_ctx.last_repair_non_convergence_signature,
- consecutive_identical_repair_count=copilot_ctx.consecutive_non_converging_repair_count,
- ceiling_reached=False,
- )
- return
-
signature = _repair_non_convergence_signature(copilot_ctx, contract)
if signature is None or progressed:
copilot_ctx.consecutive_non_converging_repair_count = 0
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 7d840b5bb..e4b90616c 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -31,10 +31,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_is_positional_selector,
dynamic_row_evidence_fingerprint,
dynamic_row_period_matches_match_selected_row,
- locator_selector_literals,
- normalized_locator_expr,
normalized_scout_selector,
- synthesize_code_block,
validated_dynamic_row_period_matches,
)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
@@ -590,87 +587,6 @@ def _next_trajectory_index(trajectory: list[ScoutedInteraction]) -> int:
return highest + 1 if highest >= 0 else len(trajectory)
-def _maybe_complete_never_captured_obligation(
- ctx: AgentContext, *, interaction: ScoutedInteraction, trajectory_index: int
-) -> None:
- obligation = getattr(ctx, "never_captured_obligation", None)
- if obligation is None or obligation.state != "armed":
- return
- if obligation.turn_id != str(getattr(ctx, "turn_id", "")):
- return
- tool_name = str(interaction.get("tool_name") or "")
- if tool_name != obligation.expected_tool_name or trajectory_index <= obligation.armed_after_trajectory_index:
- return
- # Bare locator obligations can reject unrelated same-tool events without paying for a full
- # trajectory synthesis. Non-bare canonical locators still fall through to the exact emitted
- # interaction comparison below.
- if obligation.normalized_receiver.startswith("page.locator("):
- expected_selectors = {
- normalized_scout_selector(candidate)
- for candidate in locator_selector_literals(obligation.normalized_receiver)
- }
- captured_selector = str(interaction.get("selector") or "").strip()
- if normalized_scout_selector(captured_selector) not in expected_selectors:
- return
- expected_argument = obligation.expected_argument_literal
- if expected_argument is not None:
- if tool_name == "press_key":
- captured_argument = str(interaction.get("key") or "")
- elif tool_name == "select_option":
- captured_argument = str(interaction.get("value") or "")
- elif tool_name == "type_text":
- captured_argument = str(interaction.get("raw_typed_value") or interaction.get("typed_value") or "")
- else:
- captured_argument = ""
- if captured_argument != expected_argument:
- return
- synthesized = synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- if synthesized is None:
- return
- current_position = next(
- (
- position
- for position, item in enumerate(ctx.scout_trajectory)
- if item.get("trajectory_index") == trajectory_index
- ),
- None,
- )
- if current_position is None:
- return
- emitted = next(
- (
- record
- for record in synthesized.diagnostics.emitted_interactions
- if record.get("trajectory_index") == current_position
- ),
- None,
- )
- if emitted is None:
- return
- method = str(emitted.get("method") or "")
- locator = normalized_locator_expr(str(emitted.get("locator") or ""))
- if method != obligation.method or locator != obligation.normalized_receiver:
- return
- ctx.never_captured_obligation = replace(
- obligation,
- captured_trajectory_index=trajectory_index,
- state="captured",
- )
- ctx.synthesized_block_reopened_for_capture_obligation = True
- LOG.info(
- "copilot_never_captured_obligation_completed",
- identity_digest=obligation.identity_digest,
- turn_id=obligation.turn_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- draft_fingerprint=obligation.draft_fingerprint,
- block_label=obligation.block_label,
- site=obligation.site,
- trajectory_index=trajectory_index,
- method=method,
- locator=locator,
- )
-
-
def _record_scouted_interaction(
ctx: AgentContext,
*,
@@ -782,11 +698,6 @@ def _record_scouted_interaction(
trajectory_artifact["trajectory_index"] = _next_trajectory_index(trajectory)
trajectory.append(trajectory_artifact)
ctx.scout_trajectory = _capped_with_eviction_accounting(trajectory, collection="scout_trajectory")
- _maybe_complete_never_captured_obligation(
- ctx,
- interaction=trajectory_artifact,
- trajectory_index=trajectory_artifact["trajectory_index"],
- )
LOG.info(
"copilot_scout_interaction_captured",
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 428c492dd..c45fc0c8d 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -9,7 +9,7 @@ import keyword
import re
import textwrap
import tokenize
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from dataclasses import replace
@@ -25,6 +25,12 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationError
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
+from skyvern.forge.sdk.copilot.author_time_block import (
+ BANNED_BLOCKS_BLOCK_ID,
+ CODE_SAFETY_BLOCK_ID,
+ CREDENTIAL_SCOUT_BLOCK_ID,
+ AuthorTimeBlock,
+)
from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
AuthoringParameterBindingCandidate,
AuthoringParameterBindingDirective,
@@ -44,39 +50,23 @@ from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
)
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
- blocker_signal_is_genuinely_terminal,
- build_definition_contract_unsatisfied_blocker_signal,
- build_metadata_reject_same_key_terminal_signal,
- build_output_source_unobservable_blocker_signal,
clear_terminal_evidence_on_workflow_edit,
- stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestOutcomeReasonCode,
- MetadataRejectFamily,
- MetadataRejectLadderDecision,
- MetadataRejectLadderInput,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
_stable_hash,
- adjudicate_metadata_reject_ladder,
- authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
- latest_recorded_build_test_outcome_repeated,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
recorded_outcome_from_authoring_repair_context,
- run_backed_repair_evidence_exists,
)
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
- CodeBlockRenderDiagnostic,
author_time_code_block_diagnostics,
- code_block_render_diagnostic,
sandbox_unresolved_name_diagnostics,
sandbox_unresolved_name_repair_diagnostic,
strip_redundant_sandbox_imports,
@@ -100,7 +90,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
SynthesisDiagnostics,
SynthesizedCodeBlock,
_bare_drop_superseded_on_screen,
- _bare_locator_call_selector,
_credential_field_accesses,
_get_by_role_expr_strict,
_is_ignorable_entry_opener_drop,
@@ -109,11 +98,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_selector_refines,
artifact_dependency_id,
artifact_observation_ref_id,
-)
-from skyvern.forge.sdk.copilot.code_block_synthesis import (
- block_has_unguarded_credential_fill as _block_has_unguarded_credential_fill,
-)
-from skyvern.forge.sdk.copilot.code_block_synthesis import (
build_input_templated_locator,
build_same_month_file_match_locator,
credential_scout_gap,
@@ -126,7 +110,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
locator_selector_literals,
missing_rung_text,
normalized_locator_expr,
- normalized_scout_selector,
obligation_finding_reason_code,
obligation_finding_selector,
produce_covered_static_return_envelope,
@@ -145,9 +128,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import wrapped_code_ast as _
from skyvern.forge.sdk.copilot.completion_verification import grade_definition_criteria
from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
- composition_page_evidence_error,
normalize_block_observation_refs,
- workflow_target_url,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.context import (
@@ -161,7 +142,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
_record_code_authoring_guardrail_reject,
_scouted_spine_open_obligation,
arm_credential_scout_reopen,
- code_authoring_churn_stop_would_claim,
download_satisfied_requested_output_paths,
requested_output_extraction_plan,
requested_output_extraction_plan_changed,
@@ -174,7 +154,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
from skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXT
-from skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violations
from skyvern.forge.sdk.copilot.output_contracts import (
OutputContractActuation,
OutputContractActuationEvidence,
@@ -183,13 +162,13 @@ from skyvern.forge.sdk.copilot.output_contracts import (
classify_output_contract_bail_family,
code_block_available_binding_keys_by_label,
declared_string_workflow_parameter_keys,
- declared_workflow_parameter_keys,
resolve_output_contract_actuation,
)
from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyReason,
OutputPolicyVerdict,
+ demote_author_time_steer_reasons,
evaluate_output_policy,
format_output_policy_tool_error,
output_policy_verdict_to_trace_data,
@@ -211,45 +190,26 @@ from skyvern.forge.sdk.copilot.request_policy import (
)
from skyvern.forge.sdk.copilot.result_evidence import loaded_result_source_producible
from skyvern.forge.sdk.copilot.runtime import (
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
- OUTPUT_CONTRACT_ACTUATION_GATE_ID,
- RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
- SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
AgentContext,
- AuthorTimeGateAblationPayload,
- CodeArtifactMetadataValue,
- NeverCapturedObligation,
- NeverCapturedReplayPayload,
- RejectedCodeArtifactMetadataCapture,
ScoutedInteraction,
- copilot_author_time_gate_log_only_enabled,
- record_author_time_gate_ablation_event,
)
from skyvern.forge.sdk.copilot.schema_incompatibility import (
- SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,
+ SCHEMA_INCOMPATIBILITY_REASON_CODE,
SchemaIncompatibility,
- build_schema_incompatibility_blocker_signal,
merge_schema_incompatibilities,
render_schema_incompatibility_agent_steer,
+ render_schema_incompatibility_user_reason,
)
from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.copilot.turn_halt import (
- stash_repair_ceiling_turn_halt,
- stash_turn_halt_from_blocker_signal,
-)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.copilot.turn_ownership import (
TurnClaimant,
- claim_and_stash_blocker_signal,
claim_turn,
- current_turn_owner,
)
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
credential_param_ids,
- credential_params,
parse_workflow_yaml,
workflow_blocks,
)
@@ -270,12 +230,10 @@ from ._shared import (
)
from .banned_blocks import (
_banned_block_reject_message,
- _challenge_http_request_reject_message,
_copilot_banned_block_types,
_copilot_block_authoring_policy,
_detect_new_banned_blocks,
_record_banned_block_reject_span,
- _timing_only_challenge_wait_reject_message,
)
from .blockers import _clear_resolved_per_tool_budget_problem_labels
from .credentials import (
@@ -284,16 +242,12 @@ from .credentials import (
_credential_reference_validation_error,
)
from .frontier import (
- _detect_stale_block_metadata,
_get_prior_workflow,
_invalidate_verified_state_on_edit,
- _stale_block_metadata_message,
_workflow_requires_canonical_persist,
)
from .guardrails import (
_authority_tool_error,
- _download_binding_required_error,
- _download_scout_required_error,
_request_policy_allows_untested_code_block_draft,
)
@@ -672,9 +626,9 @@ def _normalize_code_artifact_metadata_detailed(
if schema_incompatibility is not None:
schema_incompatibilities.append(schema_incompatibility)
item_violations.append(render_schema_incompatibility_agent_steer(schema_incompatibility))
- download_shape_error = _download_return_shape_error(label, dumped, block_code)
- if download_shape_error is not None:
- item_violations.append(download_shape_error)
+ descriptor_leak = _download_descriptor_leak_finding(label, block_code)
+ if descriptor_leak is not None:
+ item_violations.append(descriptor_leak)
if item_violations:
violations.extend(item_violations)
offending_labels.append(label)
@@ -1000,59 +954,6 @@ def _default_missing_extraction_schema_provenance(raw_item: Any) -> Any:
return item
-def _downgrade_stale_selected_metadata_item(item: dict[str, Any], selected_label: str) -> None:
- if str(item.get("block_label") or "").strip() != selected_label:
- return
- for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
- for row in _artifact_mutable_rows(item.get(field_name)):
- row.pop("goal_value_paths", None)
- if field_name == "terminal_verifier_expectations":
- row.pop("criteria_ids", None)
- row.pop("claimed_outcome_ids", None)
- criteria = _artifact_mutable_rows(item.get("completion_criteria"))
- if not criteria:
- item["completion_criteria"] = [
- {
- "id": f"criterion:{_artifact_label_fragment(selected_label)}_diagnostic",
- "text": str(item.get("declared_goal") or selected_label).strip() or selected_label,
- "level": "outcome",
- "terminal": False,
- }
- ]
- criteria = _artifact_mutable_rows(item.get("completion_criteria"))
- for criterion in criteria:
- criterion["level"] = "outcome"
- criterion["terminal"] = False
- criterion_ids = [str(criterion.get("id") or "").strip() for criterion in criteria]
- criterion_ids = [criterion_id for criterion_id in criterion_ids if criterion_id]
- for row in _artifact_mutable_rows(item.get("claimed_outcomes")):
- row.pop("criteria_ids", None)
- if criterion_ids:
- row["covered_criteria"] = list(criterion_ids)
- for row in _artifact_mutable_rows(item.get("terminal_verifier_expectations")):
- if criterion_ids:
- row["criteria_ids"] = list(criterion_ids)
-
-
-def _downgrade_stale_selected_goal_value_paths(raw_metadata: Any, selected_label: str) -> Any:
- if raw_metadata in (None, [], {}) or not selected_label:
- return raw_metadata
- scrubbed = copy.deepcopy(raw_metadata)
- if isinstance(scrubbed, list):
- for item in scrubbed:
- if isinstance(item, dict):
- _downgrade_stale_selected_metadata_item(item, selected_label)
- elif isinstance(scrubbed, dict):
- if "block_label" in scrubbed:
- _downgrade_stale_selected_metadata_item(scrubbed, selected_label)
- else:
- for block_label, value in scrubbed.items():
- if block_label == selected_label and isinstance(value, dict):
- value.setdefault("block_label", selected_label)
- _downgrade_stale_selected_metadata_item(value, selected_label)
- return scrubbed
-
-
def _workflow_yaml_code_blocks_by_label(workflow_yaml: str | None) -> dict[str, Mapping[str, Any]]:
if workflow_yaml is None:
return {}
@@ -1116,42 +1017,6 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)
return errors
-def _code_block_render_bound_names(workflow_yaml: str) -> set[str]:
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return set()
- names = set(declared_workflow_parameter_keys(parsed))
- workflow_definition = parsed.get("workflow_definition")
- if isinstance(workflow_definition, dict):
- for key in credential_params(workflow_definition.get("parameters")):
- names.add(f"{key}_real_username")
- names.add(f"{key}_real_password")
- for block in workflow_blocks(parsed):
- label = str(block.get("label") or "").strip()
- if label:
- names.add(label)
- names.add(f"{label}_output")
- return names
-
-
-def _code_block_render_reject(
- workflow_yaml: str, prior_yaml: str | None
-) -> tuple[str, CodeBlockRenderDiagnostic] | None:
- prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)
- bound_names = _code_block_render_bound_names(workflow_yaml)
- for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():
- code = str(block.get("code") or "")
- if not code.strip():
- continue
- prior_block = prior_blocks.get(label)
- if prior_block is not None and str(prior_block.get("code") or "") == code:
- continue
- diagnostic = code_block_render_diagnostic(code, bound_names)
- if diagnostic is not None:
- return label, diagnostic
- return None
-
-
def _human_facing_code_safety_errors(errors: list[str | CodeBlockSecurityError]) -> list[str | CodeBlockSecurityError]:
preflight_reason_codes = {
reason_code for error in errors if (reason_code := _generated_code_preflight_reason_code(error)) is not None
@@ -1443,152 +1308,6 @@ def _credential_scout_reopen_identity_digest(workflow_yaml: str) -> str:
return f"{structural_identity}|{_stable_hash(binding)}"
-def _code_artifact_metadata_reject_payload(
- *,
- workflow_yaml: str,
- raw_metadata: object,
- offending_labels: list[str],
- violation_categories: list[str],
- missing_labels: list[str] | None = None,
-) -> Mapping[str, object] | None:
- labels = sorted(dict.fromkeys([*offending_labels, *(missing_labels or [])]))
- if not labels:
- labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- payload = {
- "reason_code": "metadata_reject",
- "offending_labels": labels,
- "required_fields": [
- "declared_goal",
- *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS,
- "evidence_refs_or_observation_refs",
- ],
- "missing_fields_by_label": _metadata_missing_required_fields_by_label(
- raw_metadata,
- labels=labels,
- missing_labels=missing_labels or [],
- ),
- "output_path_roots": _metadata_output_path_roots(raw_metadata),
- "output_path_roots_by_label": _metadata_output_path_roots_by_label(raw_metadata),
- "code_block_output_status": _metadata_reject_code_block_output_status(
- workflow_yaml,
- raw_metadata=raw_metadata,
- labels=labels,
- ),
- "violation_categories": sorted(dict.fromkeys(violation_categories)),
- }
- return payload
-
-
-def _metadata_missing_required_fields_by_label(
- raw_metadata: object,
- *,
- labels: list[str],
- missing_labels: list[str],
-) -> dict[str, list[str]]:
- items_by_label: dict[str, Mapping[str, Any]] = {}
- for raw_item in _code_artifact_metadata_items(raw_metadata):
- item = _raw_metadata_item_mapping(raw_item)
- if item is None:
- continue
- label = str(item.get("block_label") or "").strip()
- if label and label not in items_by_label:
- items_by_label[label] = item
-
- missing_label_set = set(missing_labels)
- missing_by_label: dict[str, list[str]] = {}
- for label in labels:
- item = items_by_label.get(label)
- if item is None or label in missing_label_set:
- missing_by_label[label] = [
- "declared_goal",
- *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS,
- "evidence_refs_or_observation_refs",
- ]
- continue
- missing_fields: list[str] = []
- if not str(item.get("declared_goal") or "").strip():
- missing_fields.append("declared_goal")
- for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:
- value = item.get(field_name)
- if not isinstance(value, list) or not value:
- missing_fields.append(field_name)
- if not item.get("evidence_refs") and not item.get("observation_refs"):
- missing_fields.append("evidence_refs_or_observation_refs")
- if missing_fields:
- missing_by_label[label] = missing_fields
- return missing_by_label
-
-
-def _metadata_output_path_roots(raw_metadata: object) -> list[str]:
- roots: set[str] = set()
- for label_roots in _metadata_output_path_roots_by_label(raw_metadata).values():
- roots.update(label_roots)
- return sorted(roots)
-
-
-def _metadata_output_paths(raw_metadata: object) -> set[str]:
- paths: set[str] = set()
- for raw_item in _code_artifact_metadata_items(raw_metadata):
- item = _raw_metadata_item_mapping(raw_item)
- if item is not None:
- paths.update(_metadata_item_output_paths(item))
- return paths
-
-
-def _metadata_extraction_schema_paths(raw_metadata: object) -> set[str]:
- paths: set[str] = set()
- for raw_item in _code_artifact_metadata_items(raw_metadata):
- item = _raw_metadata_item_mapping(raw_item)
- if item is not None:
- paths.update(_metadata_item_extraction_schema_paths(item))
- return paths
-
-
-def _metadata_output_path_roots_by_label(raw_metadata: object) -> dict[str, list[str]]:
- roots_by_label: dict[str, set[str]] = {}
- for label, paths in _metadata_output_paths_by_label(raw_metadata).items():
- roots = roots_by_label.setdefault(label, set())
- for path in paths:
- root = _top_level_path_segment(path)
- if root:
- roots.add(root)
- return {label: sorted(roots) for label, roots in sorted(roots_by_label.items()) if roots}
-
-
-def _metadata_output_paths_by_label(raw_metadata: object) -> dict[str, list[str]]:
- paths_by_label: dict[str, set[str]] = {}
- unlabeled_index = 0
- for raw_item in _code_artifact_metadata_items(raw_metadata):
- item = _raw_metadata_item_mapping(raw_item)
- if item is None:
- continue
- label = str(item.get("block_label") or "").strip()
- if not label:
- unlabeled_index += 1
- label = f"<unlabeled:{unlabeled_index}>"
- paths = paths_by_label.setdefault(label, set())
- paths.update(_metadata_item_output_paths(item))
- return {label: sorted(paths) for label, paths in sorted(paths_by_label.items()) if paths}
-
-
-def _metadata_item_output_path_roots(item: Mapping[str, Any]) -> set[str]:
- roots: set[str] = set()
- for path in _metadata_item_output_paths(item):
- root = _top_level_path_segment(path)
- if root:
- roots.add(root)
- return roots
-
-
-def _metadata_item_output_paths(item: Mapping[str, Any]) -> set[str]:
- paths: set[str] = set()
- for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
- for row in _artifact_rows(item.get(field_name)):
- paths.update(_artifact_goal_value_paths(row.get("goal_value_paths")))
- paths.update(_metadata_item_extraction_schema_paths(item))
- return paths
-
-
def _metadata_item_extraction_schema_paths(item: Mapping[str, Any]) -> set[str]:
for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
for row in _artifact_rows(item.get(field_name)):
@@ -1598,39 +1317,6 @@ def _metadata_item_extraction_schema_paths(item: Mapping[str, Any]) -> set[str]:
return set()
-def _metadata_reject_code_block_output_status(
- workflow_yaml: str,
- *,
- raw_metadata: object,
- labels: list[str],
-) -> dict[str, Mapping[str, object]]:
- code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- metadata_by_label: dict[str, Mapping[str, Any]] = {}
- for raw_item in _code_artifact_metadata_items(raw_metadata):
- item = _raw_metadata_item_mapping(raw_item)
- if item is None:
- continue
- label = str(item.get("block_label") or "").strip()
- if label and label not in metadata_by_label:
- metadata_by_label[label] = item
-
- status: dict[str, Mapping[str, object]] = {}
- for label in labels:
- block = code_blocks.get(label)
- if block is None:
- continue
- block_code = str(block.get("code") or "")
- metadata = metadata_by_label.get(label)
- status[label] = {
- "block_type": _enum_or_string_name(block.get("block_type")) or str(block.get("block_type") or ""),
- "has_code": bool(block_code.strip()),
- "declares_output_intent": _block_declares_output_intent(block),
- "declares_output_roots": sorted(_metadata_item_output_path_roots(metadata)) if metadata else [],
- "has_meaningful_output": _code_block_has_meaningful_output(block_code, metadata),
- }
- return status
-
-
def _schema_property_roots(schema: Mapping[str, object]) -> set[str]:
properties = schema.get("properties")
if isinstance(properties, Mapping):
@@ -1661,87 +1347,6 @@ def _schema_property_paths(schema: Mapping[str, object], *, prefix: str = "") ->
return set()
-def _metadata_violation_categories(violations: list[str]) -> list[str]:
- categories: list[str] = []
- for violation in violations:
- if "requires non-empty" in violation:
- categories.append("missing_required_list")
- elif "requires a non-empty `declared_goal`" in violation:
- categories.append("missing_declared_goal")
- elif "requires `evidence_refs` or `observation_refs`" in violation:
- categories.append("missing_artifact_refs")
- elif "requires `source_tool`" in violation:
- categories.append("missing_source_tool")
- elif "goal_value_paths" in violation:
- categories.append("invalid_goal_value_paths")
- elif "extraction_schema" in violation:
- categories.append("invalid_extraction_schema")
- elif "return" in violation or "output" in violation:
- categories.append("invalid_output_shape")
- else:
- categories.append("metadata_contract_violation")
- return categories
-
-
-class _ConvergenceReject(NamedTuple):
- authored_structure_signature: str
- reason: Literal["identical_authored_structure", "frontier_unchanged"]
- commit_early_terminal: bool
-
-
-def _recorded_outcome_convergence_reject(
- ctx: AgentContext,
- *,
- workflow_yaml: str,
- code_artifact_metadata: object,
-) -> _ConvergenceReject | None:
- latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
- if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
- return None
- candidate_signature = authored_structure_signature_from_workflow(workflow_yaml, code_artifact_metadata)
- if candidate_signature is None:
- return None
- if candidate_signature == latest.authored_structure_signature:
- return _ConvergenceReject(candidate_signature, "identical_authored_structure", False)
- constraint = getattr(ctx, "recorded_outcome_binding_constraint", None)
- if not isinstance(constraint, RecordedOutcomeBindingConstraint):
- return None
- # An author-time reject re-keys `latest` without an executed run, so keep the binding
- # anchored to its run-outcome key across consecutive frontier-unchanged rejects until a
- # real run legitimately re-keys it.
- if latest.phase != "author_time_reject" and constraint.repeated_structural_key != latest.structural_key:
- return None
- candidate_block_signatures = authored_block_signatures_from_workflow(workflow_yaml, code_artifact_metadata)
- if constraint.owning_block_frontier_moved(candidate_block_signatures):
- return None
- return _ConvergenceReject(candidate_signature, "frontier_unchanged", constraint.frontier_uncrossable)
-
-
-def _commit_recorded_outcome_early_terminal(ctx: AgentContext) -> None:
- constraint = ctx.recorded_outcome_binding_constraint
- diagnostic_reason = (
- constraint.diagnostic_reason if isinstance(constraint, RecordedOutcomeBindingConstraint) else "none"
- )
- signal = CopilotToolBlockerSignal(
- blocker_kind="loop_detected",
- agent_steering_text=(
- f"The recorded build-test outcome names an uncrossable page frontier ({diagnostic_reason}) and the "
- "frontier block is unchanged; stop retrying and report the recorded blocker from the preserved draft."
- ),
- user_facing_reason="I can't get past this page, so I'll stop here and report what I found.",
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code="repair_ceiling_reached",
- blocked_tool="update_workflow",
- )
- stash_blocker_signal(ctx, signal)
- stash_repair_ceiling_turn_halt(
- ctx, signal, consecutive_identical_repair_count=ctx.consecutive_non_converging_repair_count
- )
-
-
def _recorded_outcome_requires_output_candidate(ctx: AgentContext) -> bool:
latest = ctx.latest_recorded_build_test_outcome
if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
@@ -2243,65 +1848,6 @@ def _definition_plane_preflight_reject(
)
-def _synthesized_business_input_floor_reject(
- ctx: AgentContext,
- workflow_yaml: str,
- imposition: _SynthesizedCodeImpositionResult,
-) -> _DefinitionPlaneReject | None:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- if not (
- imposition.substitutions is not None or ctx.synthesized_block_offered or ctx.spine_imposition_owned_attempt
- ):
- return None
- criteria = _active_completion_criteria(ctx)
- has_business_goal = any(
- criterion.kind == "terminal_action"
- or (
- criterion.level == "run"
- and criterion.requested_output_floor_rekeyed
- and bool(criterion.floor_rekeyed_from_path)
- )
- for criterion in criteria
- )
- if not has_business_goal:
- return None
- rejection = _definition_plane_preflight_reject(
- ctx,
- workflow_yaml,
- enforce_untagged_declared_inputs=True,
- )
- retained_required_keys = set(getattr(ctx, "synthesized_business_required_parameter_keys", set()))
- if not retained_required_keys:
- return rejection if rejection is not None and rejection.unreferenced_parameter_keys else None
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return rejection if rejection is not None and rejection.unreferenced_parameter_keys else None
- runtime_sources = _workflow_runtime_parameter_sources(parsed)
- if runtime_sources is None:
- return rejection if rejection is not None and rejection.unreferenced_parameter_keys else None
- missing_retained_keys = retained_required_keys - runtime_sources
- if not missing_retained_keys:
- return rejection if rejection is not None and rejection.unreferenced_parameter_keys else None
- current_missing_keys = set(rejection.unreferenced_parameter_keys) if rejection is not None else set()
- return _DefinitionPlaneReject(
- criterion_ids=(
- rejection.criterion_ids if rejection is not None else tuple(criterion.id for criterion in criteria)
- ),
- reason_codes=(
- rejection.reason_codes if rejection is not None else ("synthesized_required_parameters_unreferenced",)
- ),
- unreferenced_parameter_keys=tuple(sorted(current_missing_keys | missing_retained_keys)),
- )
-
-
-def _reopen_scout_after_synthesized_business_input_floor(ctx: AgentContext) -> None:
- ctx.synthesized_block_offered = False
- ctx.synthesized_block_offered_trajectory_len = 0
- ctx.synthesized_block_offered_goal_complete = False
- ctx.spine_imposition_owned_attempt = False
-
-
def _definition_plane_reject_error(rejection: _DefinitionPlaneReject) -> str:
if rejection.unreferenced_parameter_keys:
keys = ", ".join(f"`{key}`" for key in rejection.unreferenced_parameter_keys)
@@ -2347,105 +1893,6 @@ def _definition_plane_structural_key(
return outcome.structural_key or authored_signature or ""
-def _record_definition_plane_reject(
- ctx: AgentContext,
- workflow_yaml: str,
- rejection: _DefinitionPlaneReject,
- *,
- code_artifact_metadata: object = None,
-) -> None:
- structural_payload, authored_signature = _definition_plane_structural_payload(
- workflow_yaml,
- rejection,
- code_artifact_metadata,
- )
- block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- _record_author_time_reject_outcome(
- ctx,
- reason_code="definition_contract_unsatisfied",
- summary=_definition_plane_reject_error(rejection),
- structural_payload=structural_payload,
- authored_structure_signature=authored_signature,
- block_labels=block_labels,
- )
-
-
-def _record_definition_plane_ablation_event(
- ctx: AgentContext,
- workflow_yaml: str,
- rejection: _DefinitionPlaneReject,
- *,
- code_artifact_metadata: object = None,
-) -> bool:
- fingerprint = authored_structure_signature_from_workflow(workflow_yaml, code_artifact_metadata)
- if fingerprint is None:
- return False
- payload: AuthorTimeGateAblationPayload = {
- "criterion_ids": list(rejection.criterion_ids),
- "definition_reason_codes": list(rejection.reason_codes),
- "unreferenced_parameter_keys": list(rejection.unreferenced_parameter_keys),
- }
- return record_author_time_gate_ablation_event(
- ctx,
- gate_id=DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- reason_code="definition_contract_unsatisfied",
- fingerprint=fingerprint,
- blocked_tool="update_workflow",
- payload=payload,
- )
-
-
-def _stash_unresolved_recorded_outcome_grounding_halt(
- ctx: AgentContext,
- unresolved_parameter_keys: Iterable[str],
-) -> bool:
- requirement = ctx.recorded_outcome_grounding_requirement
- constraint = ctx.recorded_outcome_binding_constraint
- latest = ctx.latest_recorded_build_test_outcome
- if (
- not isinstance(requirement, RecordedOutcomeGroundingRequirement)
- or requirement.phase != "author_time_reject"
- or not requirement.satisfied
- or not isinstance(constraint, RecordedOutcomeBindingConstraint)
- or constraint.repeated_structural_key != requirement.structural_key
- or not isinstance(latest, RecordedBuildTestOutcome)
- or latest.structural_key != requirement.structural_key
- or latest_recorded_build_test_outcome_repeated(ctx) is not True
- ):
- return False
- keys = sorted(set(unresolved_parameter_keys))
- if not keys:
- return False
- payload: AuthorTimeGateAblationPayload = {
- "phase": requirement.phase,
- "outcome_reason_code": requirement.reason_code,
- "workflow_run_id": requirement.workflow_run_id,
- "block_labels": list(requirement.block_labels),
- "unresolved_parameter_keys": keys,
- }
- if record_author_time_gate_ablation_event(
- ctx,
- gate_id=RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
- reason_code="definition_contract_unsatisfied",
- fingerprint=requirement.structural_key,
- blocked_tool="update_workflow",
- payload=payload,
- ):
- return False
- signal = build_definition_contract_unsatisfied_blocker_signal(
- unresolved_parameter_keys=keys,
- grounding_unresolved=True,
- )
- claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, signal, force_stash=True)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="recorded_outcome_regrounding")
- LOG.info(
- "copilot recorded outcome regrounding halted before reauthor",
- structural_key=requirement.structural_key,
- unresolved_parameter_keys=keys,
- )
- return True
-
-
def _scout_trajectory_index(interaction: Mapping[str, Any], position: int) -> int:
raw_index = interaction.get("trajectory_index")
return raw_index if isinstance(raw_index, int) and raw_index >= 0 else position
@@ -2856,30 +2303,6 @@ def _authoring_parameter_binding_resolution(
return _AuthoringParameterBindingResolution(snapshot, directive)
-def _authoring_parameter_binding_resolution_for_definition(
- ctx: AgentContext,
- parsed: Mapping[str, Any],
- runtime_parameters: Mapping[str, Any] | None,
- code_artifact_metadata: object = None,
-) -> _AuthoringParameterBindingResolution | None:
- workflow_yaml = yaml.safe_dump(dict(parsed), sort_keys=False)
- rejection = _definition_plane_preflight_reject(ctx, workflow_yaml)
- if rejection is None:
- return None
- return _authoring_parameter_binding_resolution(
- ctx,
- parsed,
- runtime_parameters,
- unreferenced_parameter_keys=rejection.unreferenced_parameter_keys,
- carrier_code="",
- diagnostic_structural_key=_definition_plane_structural_key(
- workflow_yaml,
- rejection,
- code_artifact_metadata,
- ),
- )
-
-
def _judgment_output_paths(ctx: AgentContext) -> set[str]:
# Judgment-boolean paths are dropped from the producer's static-bind set, not moved to the
# declaration lane; declaring one would fabricate a None where the run's own judgment belongs. The
@@ -3029,8 +2452,6 @@ _SEPARATED_BROWSER_SPINE_PLUS_EXTRACTION_STRUCTURE = "separated_browser_spine_pl
_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"
_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE = "value_bearing_output_required"
_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH = "output"
-_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
-_MAX_OUTPUT_CONTRACT_REJECTS = 4
_MAX_OUTPUT_CONTRACT_DEFERRALS = 3
_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY = "missing_code_artifact_metadata"
_METADATA_NORMALIZATION_REJECT_FAMILY = "metadata_normalization"
@@ -3045,8 +2466,6 @@ _METADATA_FAMILY_REJECT_FAMILIES = frozenset(
}
)
_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3
-_OUTPUT_CONTRACT_ABLATION_GATE_ID = OUTPUT_CONTRACT_ACTUATION_GATE_ID
-_METADATA_PREFLIGHT_ABLATION_GATE_ID = METADATA_RUN_PREFLIGHT_REJECT_GATE_ID
_VALUE_BEARING_ROOT_GUIDANCE_PATH = "output"
_VALUE_BEARING_PREARM_FINGERPRINT_PREFIX = "value-bearing:prearm:"
_VALUE_BEARING_GUIDANCE_FINGERPRINT_PREFIX = "value-bearing:guidance:"
@@ -3080,63 +2499,6 @@ class _OutputContractEvaluation:
)
-def _record_output_contract_ablation_event(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- gate_id: str,
- blocked_tool: str,
- fingerprint: str,
-) -> bool:
- reason_code = str(
- evaluation.payload.get("reason_code") or evaluation.reason_code or _OUTPUT_CONTRACT_REJECT_REASON_CODE
- )
- payload: AuthorTimeGateAblationPayload = {
- "block_label": evaluation.block_label,
- "canonical_output_contract_signature": evaluation.canonical_signature,
- "canonical_required_child_paths": sorted(evaluation.required_paths),
- "missing_metadata_paths": list(evaluation.missing_metadata_paths),
- "missing_schema_paths": list(evaluation.missing_schema_paths),
- "missing_return_paths": list(evaluation.missing_return_paths),
- "shape_violations": list(evaluation.shape_violations),
- "can_attempt_run": evaluation.can_attempt_run,
- }
- return record_author_time_gate_ablation_event(
- ctx,
- gate_id=gate_id,
- reason_code=reason_code,
- fingerprint=fingerprint,
- blocked_tool=blocked_tool,
- payload=payload,
- )
-
-
-def _record_output_contract_actuation_ablation_event(
- ctx: AgentContext,
- workflow_yaml: str,
- *,
- reason_code: str,
- signature: str,
- block_labels: Sequence[str],
- required_paths: Iterable[str],
-) -> bool:
- if not copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
- return False
- payload: AuthorTimeGateAblationPayload = {
- "block_labels": list(block_labels),
- "canonical_output_contract_signature": signature,
- "canonical_required_child_paths": sorted(required_paths),
- }
- return record_author_time_gate_ablation_event(
- ctx,
- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
- reason_code=reason_code,
- fingerprint=_output_contract_structural_fingerprint(workflow_yaml, signature),
- blocked_tool="update_workflow",
- payload=payload,
- )
-
-
@dataclass(frozen=True)
class _RuntimeOutputRepairContract:
required_paths: set[str]
@@ -3502,28 +2864,7 @@ def _stable_output_contract_key(
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
-def _output_contract_author_time_structural_payload(
- ctx: AgentContext,
- required_paths: set[str],
- *,
- block_label: str = "",
- deficiency_family: str = "output_contract_unsatisfied",
-) -> Mapping[str, object] | None:
- signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
- if not signature:
- return None
- payload: dict[str, object] = {
- "version": "metadata_reject_output_contract:v1",
- "canonical_output_contract_signature": signature,
- "canonical_required_child_paths": sorted(required_paths),
- "deficiency_family": deficiency_family,
- }
- if block_label.strip():
- payload["block_label"] = block_label.strip()
- return payload
-
-
-def _output_contract_signature(
+def _output_contract_signature(
*,
ctx: AgentContext | None = None,
scope_key: str = "",
@@ -4067,26 +3408,6 @@ def _statically_lacks_value_bearing_observation_paths(code: str, observation_pat
return False
-def _statically_valueless_return_envelope(code: str) -> bool:
- try:
- tree = ast.parse(textwrap.dedent(code).strip() or "pass")
- return_nodes = [node for node in _iter_top_level_scope(tree.body) if isinstance(node, ast.Return)]
- if not return_nodes:
- return False
- assignments, uncertain_names = _top_level_static_assignments(tree)
- for node in return_nodes:
- if node.value is None:
- return False
- resolved = _resolve_static_expression(node.value, assignments, uncertain_names)
- if not isinstance(resolved, ast.Dict):
- return False
- if _static_leaf_value_state(resolved) != "empty":
- return False
- return True
- except (RecursionError, SyntaxError, ValueError):
- return False
-
-
def _static_value_bearing_violations(code: str, observation_paths: set[str]) -> list[str]:
if not _statically_lacks_value_bearing_observation_paths(code, observation_paths):
return []
@@ -4349,1266 +3670,137 @@ def _evaluate_output_contract_for_code_block(
)
-def _adjudicate_output_contract_ladder_after_reject(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- workflow_yaml: str,
- current_fingerprint: str,
- steer_only: bool = False,
- structure_directive_paths: set[str] | None = None,
-) -> OutputContractActuation | None:
- """Run the actuation ladder at the shared deficiency seam so a signature whose formation
- remains incomplete advances toward an advisory-consumed run or a typed terminal within the
- existing caps, keeping the loop/churn defer bounded. A bail with no owner block is left to the
- owner-directive path, not granted an inert run."""
- if ctx.turn_halt is not None or ctx.output_contract_bail_actuated_this_call:
- return None
- # A statically valueless contract cannot be adjudicated by dispatching a run. The native
- # reject seam may still ask the same ladder for steer-only structure guidance, but ordinary
- # callers remain inert so they cannot mint an advisory grant that would yield the preflight.
- if _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in evaluation.shape_violations and not steer_only:
- return None
- signature = evaluation.canonical_signature
- block_label = evaluation.block_label
- if not signature or not block_label:
- return None
- if _output_contract_advisory_state(ctx, signature) in {
- OutputContractAdvisoryState.GRANTED,
- OutputContractAdvisoryState.CONSUMED,
- }:
- return None
- block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(block_label)
- target_code = str(block.get("code") or "") if block is not None else ""
- blockers = list(evaluation.shape_violations) or [_OUTPUT_CONTRACT_REJECT_REASON_CODE]
- actuation = _actuate_output_contract_bail(
- ctx,
- blockers=blockers,
- target_code=target_code,
- required_paths=(
- structure_directive_paths
- if steer_only and structure_directive_paths is not None
- else evaluation.observation_paths
- ),
- signature=signature,
- current_fingerprint=current_fingerprint,
- advisory_run_grantable=blockers == [_OUTPUT_CONTRACT_REJECT_REASON_CODE],
- declaration_paths=evaluation.declaration_paths,
- steer_only=steer_only,
- )
- if actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL:
- _stash_output_source_unobservable_terminal(
- ctx,
- reason_code=actuation.reason_code,
- required_paths=evaluation.required_paths,
- block_label=block_label,
- signature=signature,
- blockers=blockers,
- )
- elif actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE:
- _record_armed_directive_fingerprint(ctx, signature, current_fingerprint)
- return actuation
-
-
_METADATA_CONVERGENCE_DIRECTIVE_BLOCKER = "missing_code_artifact_metadata"
-def _metadata_reject_directive_payload(
- *,
- missing_fields_by_label: dict[str, list[str]],
- required_paths: set[str],
- escalate: bool,
-) -> dict[str, Any]:
- directive: dict[str, Any] = {
- "rung": 2 if escalate else 1,
- "missing_fields_by_label": {label: list(fields) for label, fields in missing_fields_by_label.items()},
- }
- if escalate:
- # "evidence_refs_or_observation_refs" is a messaging label for the either-one requirement;
- # the skeleton must only offer real CodeArtifactMetadata fields the model can set.
- directive["metadata_fill_in_skeleton"] = {
- label: {
- field: "" if field == "declared_goal" else []
- for raw_field in fields
- for field in (
- ("evidence_refs", "observation_refs")
- if raw_field == "evidence_refs_or_observation_refs"
- else (raw_field,)
- )
- }
- for label, fields in missing_fields_by_label.items()
- }
- if required_paths:
- directive["required_output_paths"] = sorted(required_paths)
- directive["extraction_schema_template"] = _schema_template_for_required_paths(required_paths)
- return directive
-
-
-def _emit_metadata_convergence_directive(
+def _loaded_result_source_producible_for_signature(
+ ctx: AgentContext,
*,
+ target_code: str,
signature: str,
- block_label: str,
- missing_fields_by_label: dict[str, list[str]],
- required_paths: set[str],
- escalate: bool,
-) -> dict[str, Any]:
- LOG.info(
- "copilot_output_contract_spine_structure_directive_emitted",
- block_label=block_label,
- canonical_output_contract_signature=signature,
- spine_split_blockers=[_METADATA_CONVERGENCE_DIRECTIVE_BLOCKER],
- spine_stage_count=None,
- rung=2 if escalate else 1,
- )
- return _metadata_reject_directive_payload(
- missing_fields_by_label=missing_fields_by_label,
- required_paths=required_paths,
- escalate=escalate,
- )
-
-
-def _metadata_reject_seam_fingerprint(
- workflow_yaml: str, signature: str, missing_fields_by_label: dict[str, list[str]]
-) -> str:
- return hashlib.sha256(
- (
- _output_contract_structural_fingerprint(workflow_yaml, signature)
- + json.dumps(missing_fields_by_label, sort_keys=True)
- ).encode("utf-8")
- ).hexdigest()
-
-
-@dataclass(frozen=True)
-class _ValueBearingEvaluationGuidance:
- required_output_paths: tuple[str, ...]
- shape_violations: tuple[str, ...]
- satisfying_templates: dict[str, Any]
-
- def fingerprint_payload(self) -> dict[str, Any]:
- return {
- "required_output_paths": list(self.required_output_paths),
- "shape_violations": list(self.shape_violations),
- "satisfying_templates": self.satisfying_templates,
- }
-
-
-def _value_bearing_evaluation_guidance(
- evaluation: _OutputContractEvaluation,
-) -> _ValueBearingEvaluationGuidance | None:
- raw_paths = evaluation.payload.get("canonical_required_child_paths")
- raw_shape_violations = evaluation.payload.get("shape_violations")
- raw_satisfying_templates = evaluation.payload.get("satisfying_templates")
- if (
- not isinstance(raw_paths, list)
- or not raw_paths
- or any(not isinstance(path, str) or not path for path in raw_paths)
- or not isinstance(raw_shape_violations, list)
- or any(not isinstance(violation, str) or not violation for violation in raw_shape_violations)
- or _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE not in raw_shape_violations
- or not isinstance(raw_satisfying_templates, dict)
- ):
- return None
- paths = tuple(sorted(set(raw_paths)))
- if any(
- path != _canonical_requested_output_path(path) or _output_path_root(path) != _VALUE_BEARING_ROOT_GUIDANCE_PATH
- for path in paths
- ):
- return None
- return _ValueBearingEvaluationGuidance(
- required_output_paths=paths,
- shape_violations=tuple(raw_shape_violations),
- satisfying_templates=dict(raw_satisfying_templates),
- )
-
-
-def _value_bearing_reject_seam_fingerprint(
- signature: str,
- guidance: _ValueBearingEvaluationGuidance,
-) -> str:
- digest = hashlib.sha256(
- json.dumps(
- {
- "directive_family": _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- "canonical_output_contract_signature": signature,
- "guidance": guidance.fingerprint_payload(),
- },
- sort_keys=True,
- ).encode("utf-8")
- ).hexdigest()
- return f"{_VALUE_BEARING_GUIDANCE_FINGERPRINT_PREFIX}{digest}"
+) -> bool:
+ """Loaded-result evidence is producible for this contract only while it is unclaimed or
+ already claimed by this signature; evidence claimed by another contract does not carry
+ over. The first contract to find it producible claims it."""
+ if not loaded_result_source_producible(ctx.latest_evaluate_result_composition_steer, target_code=target_code):
+ return False
+ claimed_signature = ctx.latest_evaluate_result_composition_signature
+ if claimed_signature not in (None, signature):
+ return False
+ ctx.latest_evaluate_result_composition_signature = signature
+ return True
-def _value_bearing_reject_directive_payload(
+def _ensure_metadata_contract_rows(
+ item: dict[str, Any],
*,
- guidance: _ValueBearingEvaluationGuidance,
- escalate: bool,
-) -> dict[str, Any]:
- return {
- "rung": 2 if escalate else 1,
- "required_output_paths": list(guidance.required_output_paths),
- "shape_violations": list(guidance.shape_violations),
- "satisfying_templates": guidance.satisfying_templates,
- }
+ goal_value_paths: set[str],
+ schema_text: str,
+) -> None:
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ rows = _artifact_mutable_rows(item.get(field_name))
+ if not rows:
+ item[field_name] = [{"goal_value_paths": sorted(goal_value_paths)}]
+ rows = _artifact_mutable_rows(item.get(field_name))
+ for row in rows:
+ if goal_value_paths and not _artifact_goal_value_paths(row.get("goal_value_paths")):
+ row["goal_value_paths"] = sorted(goal_value_paths)
+ if schema_text and not str(row.get("extraction_schema") or "").strip():
+ row["extraction_schema"] = schema_text
-def _emit_value_bearing_convergence_directive(
- *,
- signature: str,
- block_label: str,
- guidance: _ValueBearingEvaluationGuidance,
- guidance_fingerprint: str,
- candidate_fingerprint: str,
- escalate: bool,
-) -> dict[str, Any]:
- directive = _value_bearing_reject_directive_payload(
- guidance=guidance,
- escalate=escalate,
- )
- LOG.info(
- "copilot_output_contract_spine_structure_directive_emitted",
- directive_family=_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- block_label=block_label,
- canonical_output_contract_signature=signature,
- canonical_required_child_paths=list(guidance.required_output_paths),
- guidance_fingerprint=guidance_fingerprint,
- candidate_fingerprint=candidate_fingerprint,
- spine_split_blockers=list(guidance.shape_violations),
- spine_stage_count=None,
- rung=directive["rung"],
- )
- return directive
+def _metadata_item_declares_extraction_schema(item: Mapping[str, Any] | None) -> bool:
+ if item is None:
+ return False
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
+ if str(row.get("extraction_schema") or "").strip():
+ return True
+ return False
-def _value_bearing_definition_reject_corrective(
+def _output_contract_scaffold_target_label(
ctx: AgentContext,
- *,
workflow_yaml: str,
- raw_metadata: object,
-) -> dict[str, Any] | None:
- if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
- return None
- evaluation = _evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- raw_metadata,
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- if evaluation is None or _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE not in evaluation.shape_violations:
- return None
- guidance = _value_bearing_evaluation_guidance(evaluation)
- if guidance is None or not evaluation.canonical_signature or not evaluation.block_label:
- return None
- guidance_fingerprint = _value_bearing_reject_seam_fingerprint(
- evaluation.canonical_signature,
- guidance,
- )
- escalate = _prior_output_contract_directive_unconsumed(
- ctx,
- evaluation.canonical_signature,
- guidance_fingerprint,
- )
- actuation = _adjudicate_output_contract_ladder_after_reject(
+ raw_code_artifact_metadata: object,
+ required_paths: set[str],
+ *,
+ declaration_paths: set[str] | None = None,
+ allow_missing_static_return: bool = False,
+) -> str:
+ declaration_paths = declaration_paths or set()
+ label, owner_labels = _target_output_contract_block_label(
ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- current_fingerprint=guidance_fingerprint,
- steer_only=True,
- structure_directive_paths=set(guidance.required_output_paths),
- )
- if actuation is None or actuation.kind != OutputContractActuationKind.STRUCTURE_DIRECTIVE:
- return None
- candidate_fingerprint = _output_contract_structural_fingerprint(
workflow_yaml,
- evaluation.canonical_signature,
- )
- directive = _emit_value_bearing_convergence_directive(
- signature=evaluation.canonical_signature,
- block_label=evaluation.block_label,
- guidance=guidance,
- guidance_fingerprint=guidance_fingerprint,
- candidate_fingerprint=candidate_fingerprint,
- escalate=escalate,
+ raw_code_artifact_metadata,
+ required_paths,
)
- return {
- "canonical_output_contract_signature": evaluation.canonical_signature,
- "canonical_required_child_paths": list(guidance.required_output_paths),
- "shape_violations": list(guidance.shape_violations),
- "satisfying_templates": guidance.satisfying_templates,
- "output_contract_actuation": actuation.kind.value,
- "value_bearing_convergence_directive": directive,
- }
+ if not label or owner_labels != [label]:
+ return ""
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ target_block = code_blocks.get(label)
+ if target_block is None:
+ return ""
+ target_code = str(target_block.get("code") or "")
+ if not allow_missing_static_return and not (
+ required_paths | declaration_paths
+ ) <= _code_block_produced_output_paths(target_code):
+ return ""
+ if not allow_missing_static_return and _scout_spine_requires_separated_blocks(ctx):
+ synthesized = synthesize_code_block(
+ ctx.scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if synthesized is not None and _browser_surface_contains_full_action_spine(target_code, synthesized.code):
+ return ""
+ return label
-def _adjudicate_metadata_reject_ladder(
+def _apply_metadata_contract_scaffold(
ctx: AgentContext,
- *,
workflow_yaml: str,
- raw_metadata: object,
- missing_labels: list[str],
+ raw_code_artifact_metadata: object,
+ *,
required_paths: set[str],
-) -> dict[str, Any] | None:
- """Adjudicate an enforcing metadata reject without coupling it to unrelated outcome state."""
- if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
- return None
- signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
- block_label = missing_labels[0] if len(missing_labels) == 1 else ""
- if not signature or not block_label:
- return None
- missing_fields_by_label = _metadata_missing_required_fields_by_label(
- raw_metadata, labels=missing_labels, missing_labels=missing_labels
- )
- reject_payload = _code_artifact_metadata_reject_payload(
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- offending_labels=[],
- missing_labels=missing_labels,
- violation_categories=["missing_code_artifact_metadata"],
- )
- latest_outcome = ctx.latest_recorded_build_test_outcome
- candidate_outcome = (
- latest_outcome
- if isinstance(latest_outcome, RecordedBuildTestOutcome)
- and latest_outcome.phase == "author_time_reject"
- and latest_outcome.reason_code == "metadata_reject"
- else _build_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary="Submitted workflow is missing required code artifact metadata.",
- structural_payload=reject_payload,
- block_labels=missing_labels,
- missing_requested_output_facts=_missing_requested_output_facts(
- required_paths,
- reason_code=_required_child_output_paths_for_authoring(ctx)[2],
- ),
- )
- )
- seam_fingerprint = _metadata_reject_seam_fingerprint(workflow_yaml, signature, missing_fields_by_label)
- if ctx.metadata_reject_ladder_state is None and _prior_output_contract_directive_unconsumed(
- ctx,
- signature,
- seam_fingerprint,
- ):
- structural_key = candidate_outcome.structural_key
- if structural_key is not None:
- ctx.metadata_reject_ladder_state = adjudicate_metadata_reject_ladder(
- None,
- MetadataRejectLadderInput(
- reject_family="missing_code_artifact_metadata",
- structural_key=structural_key,
- missing_fields_by_label=missing_fields_by_label,
- ),
- ).state
- decision = _metadata_reject_ladder_decision(
+ source: str,
+ reason_code: str,
+ declaration_paths: set[str] | None = None,
+ allow_missing_static_return: bool = False,
+) -> object:
+ declaration_paths = declaration_paths or set()
+ union_paths = required_paths | declaration_paths
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return raw_code_artifact_metadata
+ if not union_paths:
+ return raw_code_artifact_metadata
+ label = _output_contract_scaffold_target_label(
ctx,
- candidate_outcome=candidate_outcome,
- reject_family="missing_code_artifact_metadata",
- missing_fields_by_label=missing_fields_by_label,
- )
- if decision is None:
- return None
- if decision.action == "terminal":
- return _metadata_reject_terminal_data(decision)
- assert decision.rung is not None
- _record_armed_directive_fingerprint(ctx, signature, seam_fingerprint)
- return _emit_metadata_convergence_directive(
- signature=signature,
- block_label=block_label,
- missing_fields_by_label=missing_fields_by_label,
- required_paths=required_paths,
- escalate=decision.rung == 2,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths,
+ declaration_paths=declaration_paths,
+ allow_missing_static_return=allow_missing_static_return,
)
-
-
-def _metadata_reject_ladder_decision(
- ctx: AgentContext,
- *,
- candidate_outcome: RecordedBuildTestOutcome,
- reject_family: MetadataRejectFamily,
- missing_fields_by_label: dict[str, list[str]],
-) -> MetadataRejectLadderDecision | None:
- structural_key = candidate_outcome.structural_key
- if structural_key is None or not missing_fields_by_label:
- return None
- prior_state = ctx.metadata_reject_ladder_state
- latest_outcome = ctx.latest_recorded_build_test_outcome
- if (
- prior_state is not None
- and isinstance(latest_outcome, RecordedBuildTestOutcome)
- and latest_outcome.phase == "author_time_reject"
- and (
- latest_outcome.reason_code != "metadata_reject"
- or latest_outcome.structural_key != prior_state.structural_key
- )
+ if not label:
+ return raw_code_artifact_metadata
+ existing_item = _metadata_item_for_block_label(raw_code_artifact_metadata, label)
+ prior_item = _metadata_item_for_block_label(getattr(ctx, "code_artifact_metadata", None), label)
+ if _metadata_item_declares_extraction_schema(existing_item) and not _metadata_item_effective_schema_text(
+ existing_item, union_paths
):
- _reset_metadata_reject_ladder_state(
- ctx,
- superseding_reject_family=latest_outcome.reason_code,
+ return raw_code_artifact_metadata
+ if _metadata_item_declares_extraction_schema(prior_item) and not _metadata_item_effective_schema_text(
+ prior_item, union_paths
+ ):
+ return raw_code_artifact_metadata
+ schema_text = (
+ _metadata_item_effective_schema_text(existing_item, union_paths)
+ or _metadata_item_effective_schema_text(
+ prior_item,
+ union_paths,
)
- prior_state = None
- decision = adjudicate_metadata_reject_ladder(
- prior_state,
- MetadataRejectLadderInput(
- reject_family=reject_family,
- structural_key=structural_key,
- missing_fields_by_label=missing_fields_by_label,
- ),
- )
- ctx.metadata_reject_ladder_state = decision.state
- if decision.state.streak_count >= 2:
- LOG.info(
- "copilot_metadata_reject_same_key_detected",
- reject_family=reject_family,
- structural_key=structural_key,
- gate_id=decision.gate_id,
- streak_count=decision.state.streak_count,
- missing_fields_by_label=decision.missing_fields_by_label,
- )
- if decision.action == "terminal":
- signal = build_metadata_reject_same_key_terminal_signal(
- structural_key=structural_key,
- reject_family=reject_family,
- missing_fields_by_label=decision.missing_fields_by_label,
- )
- claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, signal)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
- LOG.info(
- "copilot_metadata_reject_same_key_terminal_selected",
- reject_family=reject_family,
- structural_key=structural_key,
- gate_id=decision.gate_id,
- streak_count=decision.state.streak_count,
- missing_fields_by_label=decision.missing_fields_by_label,
- )
- return decision
-
-
-def _reset_metadata_reject_ladder_state(ctx: AgentContext, *, superseding_reject_family: str) -> None:
- prior_state = ctx.metadata_reject_ladder_state
- if prior_state is None:
- return
- ctx.metadata_reject_ladder_state = None
- LOG.info(
- "copilot_metadata_reject_streak_reset",
- prior_reject_family=prior_state.reject_family,
- prior_structural_key=prior_state.structural_key,
- prior_streak_count=prior_state.streak_count,
- superseding_reject_family=superseding_reject_family,
- )
-
-
-def _metadata_reject_terminal_data(decision: MetadataRejectLadderDecision) -> dict[str, CodeArtifactMetadataValue]:
- return cast(
- dict[str, CodeArtifactMetadataValue],
- {
- "reason_code": METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
- "reject_family": decision.state.reject_family,
- "structural_key": decision.state.structural_key,
- "gate_id": decision.gate_id,
- "missing_fields_by_label": decision.missing_fields_by_label,
- "preserves_workflow_draft": True,
- },
- )
-
-
-def _synthesized_metadata_reject_directive(
- ctx: AgentContext,
- *,
- workflow_yaml: str,
- raw_metadata: object,
- label_candidates: list[str],
- required_paths: set[str],
-) -> dict[str, Any] | None:
- if len(label_candidates) != 1:
- _reset_metadata_reject_ladder_state(
- ctx,
- superseding_reject_family="recorded_outcome_metadata_reject",
- )
- return None
- label = label_candidates[0]
- full_required_fields = {"declared_goal", *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS, "evidence_refs_or_observation_refs"}
- missing_fields_by_label = _metadata_missing_required_fields_by_label(
- raw_metadata, labels=[label], missing_labels=[]
- )
- if set(missing_fields_by_label.get(label) or []) != full_required_fields:
- _reset_metadata_reject_ladder_state(
- ctx,
- superseding_reject_family="recorded_outcome_metadata_reject",
- )
- return None
- return _adjudicate_metadata_reject_ladder(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- missing_labels=[label],
- required_paths=required_paths,
- )
-
-
-def _record_output_contract_reject(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- summary: str,
- authored_structural_fingerprint: str = "",
- workflow_yaml: str = "",
- raw_metadata: object = None,
-) -> dict[str, Any]:
- # A candidate so incomplete that no required child paths are derivable must still consume the
- # reject budget: an empty set would silently skip counting and degrade the turn to the generic
- # churn stop (loop_detected) instead of this ladder's typed terminal.
- value_bearing_reject = _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in evaluation.shape_violations
- value_bearing_guidance = _value_bearing_evaluation_guidance(evaluation) if value_bearing_reject else None
- value_bearing_directive_paths = (
- set(value_bearing_guidance.required_output_paths) if value_bearing_guidance is not None else set()
- )
- counted_paths = (
- value_bearing_directive_paths or evaluation.required_paths or {_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH}
- )
- count = _record_output_contract_family_reject(
- ctx,
- counted_paths,
- reject_family=str(evaluation.payload.get("reason_code") or _OUTPUT_CONTRACT_REJECT_REASON_CODE),
- authored_structural_fingerprint=authored_structural_fingerprint,
- )
- _capture_rejected_code_artifact_metadata(ctx)
- payload = dict(evaluation.payload)
- payload["output_contract_reject_count"] = count
- payload["output_contract_reject_budget"] = _MAX_OUTPUT_CONTRACT_REJECTS
- value_bearing_directive_eligible = bool(
- value_bearing_reject
- and evaluation.canonical_signature
- and evaluation.block_label
- and value_bearing_guidance is not None
- )
- missing_fields_by_label = (
- _metadata_missing_required_fields_by_label(raw_metadata, labels=[evaluation.block_label], missing_labels=[])
- if evaluation.block_label
- else {}
- )
- if value_bearing_directive_eligible:
- assert value_bearing_guidance is not None
- directive_fingerprint = _value_bearing_reject_seam_fingerprint(
- evaluation.canonical_signature,
- value_bearing_guidance,
- )
- else:
- directive_fingerprint = (
- _metadata_reject_seam_fingerprint(workflow_yaml, evaluation.canonical_signature, missing_fields_by_label)
- if missing_fields_by_label
- else authored_structural_fingerprint
- )
- directive_escalate = _prior_output_contract_directive_unconsumed(
- ctx, evaluation.canonical_signature, directive_fingerprint
- )
- budget_terminal = value_bearing_directive_eligible and _adjudicate_output_contract_budget(
- ctx,
- counted_paths,
- count=count,
- block_label=evaluation.block_label,
- )
- actuation = (
- None
- if budget_terminal
- else _adjudicate_output_contract_ladder_after_reject(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- current_fingerprint=directive_fingerprint,
- steer_only=value_bearing_directive_eligible,
- structure_directive_paths=value_bearing_directive_paths if value_bearing_directive_eligible else None,
- )
- )
- value_bearing_directive_emitted = False
- if actuation is not None:
- payload["output_contract_actuation"] = actuation.kind.value
- if actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE and value_bearing_directive_eligible:
- assert value_bearing_guidance is not None
- candidate_fingerprint = _output_contract_structural_fingerprint(
- workflow_yaml, evaluation.canonical_signature
- )
- value_bearing_directive = _emit_value_bearing_convergence_directive(
- signature=evaluation.canonical_signature,
- block_label=evaluation.block_label,
- guidance=value_bearing_guidance,
- guidance_fingerprint=directive_fingerprint,
- candidate_fingerprint=candidate_fingerprint,
- escalate=directive_escalate,
- )
- payload["value_bearing_convergence_directive"] = value_bearing_directive
- value_bearing_directive_emitted = True
- latest_outcome = ctx.latest_recorded_build_test_outcome
- if isinstance(latest_outcome, RecordedBuildTestOutcome) and latest_outcome.phase == "author_time_reject":
- record_build_test_outcome(ctx, None)
- if (
- actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE
- and not value_bearing_directive_eligible
- and evaluation.block_label
- and missing_fields_by_label
- ):
- payload["metadata_convergence_directive"] = _emit_metadata_convergence_directive(
- signature=evaluation.canonical_signature,
- block_label=evaluation.block_label,
- missing_fields_by_label=missing_fields_by_label,
- required_paths=evaluation.required_paths,
- escalate=directive_escalate,
- )
- if actuation.kind != OutputContractActuationKind.STRUCTURE_DIRECTIVE:
- latest_outcome = ctx.latest_recorded_build_test_outcome
- if (
- isinstance(latest_outcome, RecordedBuildTestOutcome)
- and latest_outcome.phase == "author_time_reject"
- and latest_outcome.reason_code == "metadata_reject"
- ):
- # The typed adjudication now owns this same deficiency. Keep the historical reject for
- # ceiling/liveness evidence, but do not render it again as a fresh rejection next turn.
- record_build_test_outcome(ctx, None)
- if not value_bearing_directive_eligible:
- budget_terminal = _adjudicate_output_contract_budget(
- ctx,
- counted_paths,
- count=count,
- block_label=evaluation.block_label,
- )
- if budget_terminal:
- payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- if (
- actuation is None or actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE
- ) and not value_bearing_directive_emitted:
- structural_payload = _output_contract_author_time_structural_payload(
- ctx,
- evaluation.required_paths,
- block_label=evaluation.block_label,
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=summary,
- structural_payload=structural_payload or payload,
- block_labels=[evaluation.block_label] if evaluation.block_label else [],
- missing_requested_output_facts=payload.get("missing_requested_output_facts")
- if isinstance(payload.get("missing_requested_output_facts"), list)
- else None,
- )
- if count < 1:
- _record_code_authoring_guardrail_reject(ctx)
- return payload
-
-
-def _record_output_contract_family_reject(
- ctx: AgentContext,
- required_paths: set[str],
- *,
- reject_family: str,
- authored_structural_fingerprint: str = "",
-) -> int:
- if not required_paths:
- return 0
- scope_key = _output_contract_scope_key(ctx)
- signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
- if not signature:
- LOG.info(
- "copilot_output_contract_reject_count_unscoped",
- reject_family=reject_family,
- canonical_required_child_paths=sorted(required_paths),
- )
- return 0
- count_by_signature = getattr(ctx, "output_contract_reject_count_by_signature", None)
- if not isinstance(count_by_signature, dict):
- count_by_signature = {}
- if authored_structural_fingerprint:
- prior_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)
- imposed_since = ctx.output_contract_imposed_since_last_reject_by_signature.get(signature, False)
- fingerprint_rotated = prior_fingerprint is not None and prior_fingerprint != authored_structural_fingerprint
- rotation_resets = fingerprint_rotated and reject_family not in _METADATA_FAMILY_REJECT_FAMILIES
- if imposed_since or rotation_resets:
- count_by_signature[signature] = 0
- ctx.output_contract_imposed_since_last_reject_by_signature[signature] = False
- LOG.info(
- "copilot_output_contract_reject_streak_reset",
- canonical_output_contract_signature=signature,
- imposed_since_last_reject=imposed_since,
- reject_family=reject_family,
- )
- ctx.output_contract_last_reject_fingerprint_by_signature[signature] = authored_structural_fingerprint
- count = int(count_by_signature.get(signature, 0) or 0) + 1
- count_by_signature[signature] = count
- ctx.output_contract_reject_count_by_signature = count_by_signature
- LOG.info(
- "copilot_output_contract_reject_counted",
- output_contract_scope_key=scope_key,
- canonical_output_contract_signature=signature,
- output_contract_reject_count=count,
- output_contract_reject_budget=_MAX_OUTPUT_CONTRACT_REJECTS,
- reject_family=reject_family,
- canonical_required_child_paths=sorted(required_paths),
- )
- return count
-
-
-def _record_output_contract_deferral(ctx: AgentContext, required_paths: set[str]) -> int:
- if not required_paths:
- return 0
- signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
- if not signature:
- return 0
- count = int(ctx.output_contract_deferral_count_by_signature.get(signature, 0) or 0) + 1
- ctx.output_contract_deferral_count_by_signature[signature] = count
- return count
-
-
-def _adjudicate_output_contract_budget(
- ctx: AgentContext,
- required_paths: set[str],
- *,
- count: int,
- block_label: str,
-) -> bool:
- if count < _MAX_OUTPUT_CONTRACT_REJECTS:
- return False
- signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
- if not run_backed_repair_evidence_exists(ctx):
- deferral_count = _record_output_contract_deferral(ctx, required_paths)
- if deferral_count < _MAX_OUTPUT_CONTRACT_DEFERRALS:
- LOG.info(
- "copilot_output_contract_budget_rewrite_deferred_no_run",
- block_label=block_label,
- canonical_output_contract_signature=signature,
- output_contract_reject_count=count,
- output_contract_deferral_count=deferral_count,
- )
- return False
- LOG.info(
- "copilot_output_contract_budget_deferral_cap_reached",
- block_label=block_label,
- canonical_output_contract_signature=signature,
- output_contract_reject_count=count,
- output_contract_deferral_count=deferral_count,
- )
- _stash_output_contract_reject_budget_terminal(
- ctx, required_paths=required_paths, block_label=block_label, signature=signature
- )
- return True
-
-
-def _stash_output_contract_reject_budget_terminal(
- ctx: AgentContext,
- *,
- required_paths: set[str],
- block_label: str,
- signature: str,
-) -> None:
- if ctx.turn_halt is not None or blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
- return
- signal = build_output_source_unobservable_blocker_signal(
- reason_code=_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE,
- required_paths=required_paths,
- block_label=block_label,
- )
- stash_blocker_signal(ctx, signal)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
- LOG.info(
- "copilot_output_contract_reject_budget_terminal",
- block_label=block_label,
- canonical_output_contract_signature=signature,
- canonical_required_child_paths=sorted(required_paths),
- )
-
-
-@dataclass(frozen=True)
-class _OutputContractCeilingResolution:
- """Outcome of the impose-or-credit move at the convergence ceiling: the draft (possibly
- rewritten server-side) plus the crediting re-evaluation that admits it."""
-
- workflow_yaml: str
- code_artifact_metadata: object
- evaluation: _OutputContractEvaluation
- mode: Literal["credited", "imposed"]
-
-
-def _loaded_result_source_producible_for_signature(
- ctx: AgentContext,
- *,
- target_code: str,
- signature: str,
-) -> bool:
- """Loaded-result evidence is producible for this contract only while it is unclaimed or
- already claimed by this signature; evidence claimed by another contract does not carry
- over. The first contract to find it producible claims it."""
- if not loaded_result_source_producible(ctx.latest_evaluate_result_composition_steer, target_code=target_code):
- return False
- claimed_signature = ctx.latest_evaluate_result_composition_signature
- if claimed_signature not in (None, signature):
- return False
- ctx.latest_evaluate_result_composition_signature = signature
- return True
-
-
-def _every_required_output_value_observed(ctx: AgentContext, required_paths: set[str]) -> bool:
- """Imposition writes a binding for every required path, so authorization needs observation
- coverage for every one of them — a single overlapping path would otherwise license composing
- the rest as constants the recording never saw."""
- if not required_paths:
- return False
- return all(_observed_required_output_values(ctx, {path}) for path in required_paths)
-
-
-def _output_contract_binding_source_on_record(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- workflow_yaml: str,
-) -> bool:
- """Whether the recorded exploration already pins where the requested value lives: either the
- scout observed every required path, or a producible loaded-result composition anchors the
- target block's selectors. Imposition is only justified when this is on record."""
- binding_paths = evaluation.observation_paths or evaluation.required_paths
- if _every_required_output_value_observed(ctx, binding_paths):
- return True
- target_block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(evaluation.block_label)
- if target_block is None:
- return False
- return _loaded_result_source_producible_for_signature(
- ctx,
- target_code=str(target_block.get("code") or ""),
- signature=evaluation.canonical_signature,
- )
-
-
-def _impose_output_contract_binding_at_ceiling(
- ctx: AgentContext,
- workflow_yaml: str,
- raw_code_artifact_metadata: object,
- evaluation: _OutputContractEvaluation,
-) -> tuple[str, object] | None:
- """Compose the required output binding onto the submitted draft using the same static-return
- and metadata-scaffold composers the steer directives describe to the model. Abstains (returns
- None) when no value-bearing envelope can be composed, leaving the reject path byte-identical."""
- label = evaluation.block_label
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return None
- target_block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(label)
- if target_block is None:
- return None
- target_code = str(target_block.get("code") or "")
- contract = _output_contract_required_paths_source(ctx)
- binding_paths = set(contract.union) or _value_bearing_directive_paths(contract)
- if not binding_paths:
- return None
- keyed_code, violations = _extraction_code_with_value_bearing_static_return(
- target_code,
- required_paths=binding_paths,
- declaration_paths=contract.declaration_paths,
- )
- if violations:
- return None
- imposed_yaml = workflow_yaml
- if keyed_code != textwrap.dedent(target_code).strip():
- applied = False
- for block in _workflow_code_blocks(parsed):
- if str(block.get("label") or "").strip() == label:
- block["code"] = keyed_code.rstrip() + "\n"
- applied = True
- break
- if not applied:
- return None
- imposed_yaml = yaml.safe_dump(parsed, sort_keys=False)
- scaffolded_metadata = _apply_metadata_contract_scaffold(
- ctx,
- imposed_yaml,
- raw_code_artifact_metadata,
- required_paths=contract.observation_paths or (binding_paths - contract.declaration_paths),
- source=contract.source or evaluation.source,
- reason_code=contract.reason_code or evaluation.reason_code,
- declaration_paths=contract.declaration_paths,
- allow_missing_static_return=True,
- )
- return imposed_yaml, scaffolded_metadata
-
-
-def _output_contract_steering_exhausted(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- workflow_yaml: str,
-) -> bool:
- """Steering has stopped moving the draft: this submission came back with the same canonical
- structural fingerprint the last reject already steered on, so re-emitting the directive cannot
- converge. Keyed on that fingerprint rather than reject counts, which inflate, and unreachable
- on a first submission because no prior reject fingerprint exists yet."""
- signature = evaluation.canonical_signature
- if not signature or not evaluation.block_label:
- return False
- if not _prior_output_contract_actuation(ctx, signature):
- return False
- steered_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)
- if not steered_fingerprint:
- return False
- return steered_fingerprint == _output_contract_structural_fingerprint(workflow_yaml, signature)
-
-
-def _credit_output_contract_submission(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- workflow_yaml: str,
- raw_metadata: object,
-) -> _OutputContractEvaluation | None:
- """Re-adjudicate the submission with static uncertainty credited, so a draft that binds the
- required paths but that the liveness prover can only call "unknown" ships instead of looping.
- Returns None when the deficiency is genuine."""
- credited = _evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- raw_metadata,
- enforce_value_bearing_liveness=True,
- credit_convergence_ceiling=True,
- )
- if credited is None or credited.has_deficiencies:
- return None
- return credited
-
-
-def _credit_output_contract_submission_at_ceiling(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- workflow_yaml: str,
- raw_metadata: object,
-) -> _OutputContractEvaluation | None:
- resolution = _credit_or_impose_output_contract(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- credit_only=True,
- )
- return resolution.evaluation if resolution is not None else None
-
-
-def _resolve_output_contract_pass_path(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- workflow_yaml: str,
- raw_metadata: object,
- credit_only: bool,
- steering_exhausted: bool = False,
-) -> OutputContractActuation:
- """Convert the pass-path preconditions to typed evidence and let the single resolver decide.
- Crediting and composing are probed here because the resolver is pure; probing mutates nothing
- and abstains exactly where the strict path already rejects.
-
- ``steering_exhausted`` lets a caller contribute its own lane's exhaustion evidence — the
- code-authoring churn ceiling is the same fact reached by a different lane."""
- credited = _credit_output_contract_submission(
- ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata
- )
- impose_available = (
- not credit_only
- and credited is None
- and _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml)
- and _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation) is not None
- )
- evidence = OutputContractActuationEvidence(
- imposed_available=False,
- click_only_spine=False,
- observed_required_values=bool(evaluation.observation_paths)
- and _observed_required_output_values(ctx, evaluation.observation_paths),
- prior_actuation=_prior_output_contract_actuation(ctx, evaluation.canonical_signature),
- prior_directive_unconsumed=False,
- steering_exhausted=steering_exhausted or _output_contract_steering_exhausted(ctx, evaluation, workflow_yaml),
- credit_available=credited is not None,
- scouted_binding_impose_available=impose_available,
- )
- return resolve_output_contract_actuation(
- family=classify_output_contract_bail_family(evaluation.shape_violations), evidence=evidence
- )
-
-
-def _credit_or_impose_output_contract(
- ctx: AgentContext,
- evaluation: _OutputContractEvaluation,
- *,
- workflow_yaml: str,
- raw_metadata: object,
- credit_only: bool = False,
- steering_exhausted: bool = False,
-) -> _OutputContractCeilingResolution | None:
- """Resolve a submission the steer loop can no longer move: credit it when the strict prover
- was merely uncertain, else impose the binding the recording pins. Returns None when the
- resolver picks neither, leaving the caller's reject/ablation path untouched."""
- actuation = _resolve_output_contract_pass_path(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- credit_only=credit_only,
- steering_exhausted=steering_exhausted,
- )
- if actuation.kind not in {
- OutputContractActuationKind.CREDIT,
- OutputContractActuationKind.IMPOSE_SCOUTED_BINDING,
- }:
- return None
- if actuation.kind == OutputContractActuationKind.CREDIT:
- credited = _credit_output_contract_submission(
- ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata
- )
- if credited is None:
- return None
- LOG.info(
- "copilot_output_contract_submission_credited_at_ceiling",
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- credited_shape_violations=list(evaluation.shape_violations),
- credited_missing_code_return_paths=list(evaluation.missing_return_paths),
- )
- return _OutputContractCeilingResolution(
- workflow_yaml=workflow_yaml,
- code_artifact_metadata=raw_metadata,
- evaluation=credited,
- mode="credited",
- )
- imposed = _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation)
- if imposed is None:
- return None
- imposed_yaml, imposed_metadata = imposed
- reevaluated = _evaluate_output_contract_for_code_block(
- ctx,
- imposed_yaml,
- imposed_metadata,
- enforce_value_bearing_liveness=True,
- credit_convergence_ceiling=True,
- )
- if reevaluated is None or reevaluated.has_deficiencies:
- return None
- LOG.info(
- "copilot_output_contract_binding_imposed_at_ceiling",
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- )
- return _OutputContractCeilingResolution(
- workflow_yaml=imposed_yaml,
- code_artifact_metadata=imposed_metadata,
- evaluation=reevaluated,
- mode="imposed",
- )
-
-
-def _consult_guardrail_ceiling_actuation(
- ctx: AgentContext,
- *,
- workflow_yaml: str,
- raw_metadata: object,
-) -> _OutputContractCeilingResolution | None:
- """At the code-authoring guardrail give-up, re-evaluate the live local draft and, when the
- requested output's binding is already on record, consult the existing ceiling actuation (credit
- or impose) before any stopped envelope. Side-effect-free until the actuation itself resolves."""
- evaluation = _evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- raw_metadata,
- enforce_value_bearing_liveness=True,
- )
- if evaluation is None or not evaluation.canonical_signature or not evaluation.block_label:
- return None
- if not _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml):
- return None
- LOG.info(
- "copilot_guardrail_ceiling_actuation_consulted",
- reject_count=ctx.code_authoring_guardrail_reject_count,
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- binding_source_on_record=True,
- )
- return _credit_or_impose_output_contract(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- steering_exhausted=True,
- )
-
-
-def _guardrail_ceiling_actuation_resolution(
- ctx: AgentContext,
- *,
- workflow_yaml: str,
- raw_metadata: object,
- frontier_unchanged: bool = False,
-) -> _OutputContractCeilingResolution | None:
- if not code_authoring_churn_stop_would_claim(ctx, frontier_unchanged=frontier_unchanged):
- return None
- return _consult_guardrail_ceiling_actuation(ctx, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata)
-
-
-def _capture_rejected_code_artifact_metadata(ctx: AgentContext) -> None:
- snapshot = ctx.submitted_code_artifact_metadata_snapshot
- if snapshot is None:
- return
- captures = ctx.rejected_code_artifact_metadata_captures
- captures.append(RejectedCodeArtifactMetadataCapture(payload=snapshot))
- del captures[:-5]
- LOG.info(
- "copilot_rejected_code_artifact_metadata_captured",
- capture_count=len(captures),
- submitted_metadata_labels=sorted(snapshot) if isinstance(snapshot, dict) else None,
- submitted_metadata_type=type(snapshot).__name__,
- rejected_code_artifact_metadata_payload=snapshot,
- )
-
-
-def _output_contract_reject_result(
- evaluation: _OutputContractEvaluation,
- *,
- payload: dict[str, Any] | None = None,
- tool_name: str = "update_workflow",
-) -> dict[str, Any]:
- data = payload or evaluation.payload
- observation_paths = {str(path) for path in data.get("canonical_required_child_paths", []) or []} - {
- str(path) for path in data.get("declaration_only_child_paths", []) or []
- }
- if data.get("reason_code") == _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE:
- error = (
- "The workflow output contract repair budget is exhausted for this canonical requested-output contract. "
- "Return with the typed output-contract payload instead of trying another variant."
- )
- elif data.get("reason_code") == _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE and not observation_paths:
- error = (
- f"{tool_name} cannot proceed: every requested value-bearing output has degraded out of the "
- "completion criteria, so no code revision can satisfy the contract. Re-establish at least one "
- "requested value-bearing output before authoring again."
- )
- else:
- path_text = ", ".join(str(path) for path in data.get("canonical_required_child_paths", []) or [])
- path_suffix = f" Required requested output paths: {path_text}." if path_text else ""
- declaration_suffix = _declaration_repair_sentence(
- str(path) for path in data.get("declaration_only_child_paths", []) or []
- )
- error = (
- f"{tool_name} cannot proceed until the submitted workflow satisfies the requested output contract. "
- "Use the returned code_artifact_metadata, extraction_schema, and return skeleton templates exactly for "
- "the canonical required child paths." + path_suffix + declaration_suffix
- )
- return {
- "ok": False,
- "error": error,
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": data,
- }
-
-
-def _ensure_metadata_contract_rows(
- item: dict[str, Any],
- *,
- goal_value_paths: set[str],
- schema_text: str,
-) -> None:
- for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
- rows = _artifact_mutable_rows(item.get(field_name))
- if not rows:
- item[field_name] = [{"goal_value_paths": sorted(goal_value_paths)}]
- rows = _artifact_mutable_rows(item.get(field_name))
- for row in rows:
- if goal_value_paths and not _artifact_goal_value_paths(row.get("goal_value_paths")):
- row["goal_value_paths"] = sorted(goal_value_paths)
- if schema_text and not str(row.get("extraction_schema") or "").strip():
- row["extraction_schema"] = schema_text
-
-
-def _metadata_item_declares_extraction_schema(item: Mapping[str, Any] | None) -> bool:
- if item is None:
- return False
- for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
- for row in _artifact_rows(item.get(field_name)):
- if str(row.get("extraction_schema") or "").strip():
- return True
- return False
-
-
-def _output_contract_scaffold_target_label(
- ctx: AgentContext,
- workflow_yaml: str,
- raw_code_artifact_metadata: object,
- required_paths: set[str],
- *,
- declaration_paths: set[str] | None = None,
- allow_missing_static_return: bool = False,
-) -> str:
- declaration_paths = declaration_paths or set()
- label, owner_labels = _target_output_contract_block_label(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- required_paths,
- )
- if not label or owner_labels != [label]:
- return ""
- code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- target_block = code_blocks.get(label)
- if target_block is None:
- return ""
- target_code = str(target_block.get("code") or "")
- if not allow_missing_static_return and not (
- required_paths | declaration_paths
- ) <= _code_block_produced_output_paths(target_code):
- return ""
- if not allow_missing_static_return and _scout_spine_requires_separated_blocks(ctx):
- synthesized = synthesize_code_block(
- ctx.scout_trajectory,
- strict_selectors=True,
- reached_download_target=getattr(ctx, "reached_download_target", None),
- )
- if synthesized is not None and _browser_surface_contains_full_action_spine(target_code, synthesized.code):
- return ""
- return label
-
-
-def _apply_metadata_contract_scaffold(
- ctx: AgentContext,
- workflow_yaml: str,
- raw_code_artifact_metadata: object,
- *,
- required_paths: set[str],
- source: str,
- reason_code: str,
- declaration_paths: set[str] | None = None,
- allow_missing_static_return: bool = False,
-) -> object:
- declaration_paths = declaration_paths or set()
- union_paths = required_paths | declaration_paths
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return raw_code_artifact_metadata
- if not union_paths:
- return raw_code_artifact_metadata
- label = _output_contract_scaffold_target_label(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- required_paths,
- declaration_paths=declaration_paths,
- allow_missing_static_return=allow_missing_static_return,
- )
- if not label:
- return raw_code_artifact_metadata
- existing_item = _metadata_item_for_block_label(raw_code_artifact_metadata, label)
- prior_item = _metadata_item_for_block_label(getattr(ctx, "code_artifact_metadata", None), label)
- if _metadata_item_declares_extraction_schema(existing_item) and not _metadata_item_effective_schema_text(
- existing_item, union_paths
- ):
- return raw_code_artifact_metadata
- if _metadata_item_declares_extraction_schema(prior_item) and not _metadata_item_effective_schema_text(
- prior_item, union_paths
- ):
- return raw_code_artifact_metadata
- schema_text = (
- _metadata_item_effective_schema_text(existing_item, union_paths)
- or _metadata_item_effective_schema_text(
- prior_item,
- union_paths,
- )
- or _schema_template_text_for_required_paths(union_paths, declaration_paths)
+ or _schema_template_text_for_required_paths(union_paths, declaration_paths)
)
items = [
copy.deepcopy(item)
@@ -5774,16 +3966,6 @@ def _workflow_needs_contract_readback_persist(
return True
-def _output_contract_reject_count(ctx: AgentContext, signature: str) -> int:
- count_by_signature = getattr(ctx, "output_contract_reject_count_by_signature", None)
- if not isinstance(count_by_signature, Mapping):
- return 0
- try:
- return int(count_by_signature.get(signature, 0) or 0)
- except (TypeError, ValueError):
- return 0
-
-
def _runtime_output_repair_attempt_key(ctx: AgentContext, required_paths: set[str]) -> str:
runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
outcome = getattr(ctx, "latest_recorded_build_test_outcome", None)
@@ -5798,13 +3980,6 @@ def _runtime_output_repair_attempt_key(ctx: AgentContext, required_paths: set[st
return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
-def _runtime_output_repair_attempt_recorded(ctx: AgentContext, attempt_key: str) -> bool:
- if not attempt_key:
- return True
- attempts = getattr(ctx, "runtime_output_repair_attempt_by_signature", None)
- return isinstance(attempts, Mapping) and bool(attempts.get(attempt_key))
-
-
def _record_runtime_output_repair_attempt(ctx: AgentContext, attempt_key: str) -> None:
if not attempt_key:
return
@@ -6177,16 +4352,6 @@ def _output_contract_advisory_granted(ctx: AgentContext, signature: str) -> bool
return _output_contract_advisory_state(ctx, signature) == OutputContractAdvisoryState.GRANTED
-def _revoke_output_contract_advisory_run(ctx: AgentContext, signature: str) -> None:
- if _output_contract_advisory_granted(ctx, signature):
- ctx.output_contract_actuation_by_signature.pop(signature, None)
- LOG.info(
- "copilot_output_contract_advisory_run_revoked",
- canonical_output_contract_signature=signature,
- reason_code=_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- )
-
-
def _grant_output_contract_advisory_run(ctx: AgentContext, signature: str) -> None:
if not signature:
return
@@ -6239,74 +4404,19 @@ def consume_output_contract_advisory_grant_for_run_result(
for signature in consumed:
LOG.info(
"copilot_output_contract_advisory_run_dispatched_at_seam",
- canonical_output_contract_signature=signature,
- workflow_run_id=workflow_run_id,
- )
- return consumed
-
-
-def _workflow_run_id_from_run_result(run_result: Mapping[str, object]) -> str | None:
- data = run_result.get("data")
- if isinstance(data, Mapping):
- workflow_run_id = data.get("workflow_run_id")
- if isinstance(workflow_run_id, str) and workflow_run_id.strip():
- return workflow_run_id
- return None
-
-
-def _stash_output_source_unobservable_terminal(
- ctx: AgentContext,
- *,
- reason_code: str,
- required_paths: set[str],
- block_label: str,
- signature: str,
- blockers: list[str],
-) -> None:
- if ctx.turn_halt is not None:
- return
- payload: AuthorTimeGateAblationPayload = {
- "block_label": block_label,
- "canonical_output_contract_signature": signature,
- "canonical_required_child_paths": sorted(required_paths),
- "spine_split_blockers": list(blockers),
- }
- if (
- copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID)
- and ctx.output_contract_bail_actuated_this_call
- ):
- return
- if record_author_time_gate_ablation_event(
- ctx,
- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
- reason_code=reason_code,
- fingerprint=signature,
- blocked_tool="update_workflow",
- payload=payload,
- ):
- return
- signal = build_output_source_unobservable_blocker_signal(
- reason_code=reason_code,
- required_paths=required_paths,
- block_label=block_label,
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code=cast(BuildTestOutcomeReasonCode, reason_code),
- summary=signal.user_facing_reason,
- structural_payload=_output_contract_author_time_structural_payload(ctx, required_paths, block_label=block_label)
- or {},
- block_labels=[block_label] if block_label else [],
- )
- stash_blocker_signal(ctx, signal)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
- LOG.info(
- "copilot_output_contract_blocked_terminal",
- reason=reason_code,
- block_label=block_label,
- canonical_output_contract_signature=signature,
- spine_split_blockers=list(blockers),
- )
+ canonical_output_contract_signature=signature,
+ workflow_run_id=workflow_run_id,
+ )
+ return consumed
+
+
+def _workflow_run_id_from_run_result(run_result: Mapping[str, object]) -> str | None:
+ data = run_result.get("data")
+ if isinstance(data, Mapping):
+ workflow_run_id = data.get("workflow_run_id")
+ if isinstance(workflow_run_id, str) and workflow_run_id.strip():
+ return workflow_run_id
+ return None
def _run_authority_permits_dispatch(ctx: AgentContext) -> bool:
@@ -6331,21 +4441,6 @@ def _arm_pending_run_evidence(ctx: AgentContext, signature: str, required_paths:
ctx.output_contract_run_bound_required_path_by_signature.pop(signature, None)
-def _apply_output_contract_ceiling_bookkeeping(ctx: AgentContext, resolution: _OutputContractCeilingResolution) -> str:
- """Arm pending-run evidence for a credited/imposed ceiling draft so a run that binds nothing
- reopens the ladder, and for an imposed draft mark the imposition so a later reject in the same
- call refunds the streak. Returns the imposed signature (empty when credited)."""
- _arm_pending_run_evidence(
- ctx,
- resolution.evaluation.canonical_signature,
- set(resolution.evaluation.observation_paths),
- )
- if resolution.mode == "imposed":
- _mark_output_contract_imposed(ctx, resolution.evaluation.canonical_signature)
- return resolution.evaluation.canonical_signature
- return ""
-
-
def _registered_output_paths(result: object) -> set[str]:
paths: set[str] = set()
data = result.get("data") if isinstance(result, dict) else None
@@ -6440,7 +4535,6 @@ def _actuate_output_contract_bail(
) -> OutputContractActuation:
click_only_spine = _output_contract_click_only_spine(target_code, declaration_paths)
observed_required_values = _observed_required_output_values(ctx, required_paths)
- loaded_result_evidence = ctx.latest_evaluate_result_composition_steer
result_source_producible = _loaded_result_source_producible_for_signature(
ctx, target_code=target_code, signature=signature
)
@@ -6472,34 +4566,7 @@ def _actuate_output_contract_bail(
OutputContractActuationKind.BLOCKED_TERMINAL,
}:
actuation = OutputContractActuation(OutputContractActuationKind.STRUCTURE_DIRECTIVE, actuation.family)
- payload: AuthorTimeGateAblationPayload = {
- "actuation_kind": actuation.kind.value,
- "family": actuation.family.value,
- "blockers": list(blockers),
- "canonical_required_child_paths": sorted(required_paths),
- "advisory_state": evidence.advisory_state.value,
- "advisory_run_grantable": evidence.advisory_run_grantable,
- "loaded_result_source_producible": evidence.loaded_result_source_producible,
- "loaded_result_structure_signature": loaded_result_evidence.structure_signature
- if result_source_producible and loaded_result_evidence is not None
- else "",
- "loaded_result_source_tool": loaded_result_evidence.source_tool
- if result_source_producible and loaded_result_evidence is not None
- else "",
- "loaded_result_source_url": loaded_result_evidence.source_url
- if result_source_producible and loaded_result_evidence is not None
- else "",
- }
ctx.output_contract_bail_actuated_this_call = True
- if record_author_time_gate_ablation_event(
- ctx,
- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
- reason_code=actuation.reason_code or actuation.kind.value,
- fingerprint=current_fingerprint,
- blocked_tool="update_workflow",
- payload=payload,
- ):
- return actuation
if actuation.kind == OutputContractActuationKind.ADVISORY_RUN:
_grant_output_contract_advisory_run(ctx, signature)
_arm_pending_run_evidence(ctx, signature, required_paths)
@@ -6674,13 +4741,10 @@ def _impose_covered_static_return_envelope(
download_paths = observation_paths & download_registered
download = ctx.reached_download_target
expects_download = download is not None and bool(download.selector) and _code_uses_expect_download(target_code)
- block_metadata = _metadata_item_for_block_label(raw_code_artifact_metadata, label) or {}
if not scalar_paths:
if not (download_paths and expects_download):
return None
- if _download_return_shape_error(label, block_metadata, target_code) is not None:
- return None
scaffolded_metadata = _apply_metadata_contract_scaffold(
ctx,
workflow_yaml,
@@ -6716,8 +4780,6 @@ def _impose_covered_static_return_envelope(
)
if revalidation:
return None
- if download_paths and _download_return_shape_error(label, block_metadata, envelope.code) is not None:
- return None
for block in _workflow_code_blocks(parsed):
if str(block.get("label") or "").strip() == label:
@@ -6747,7 +4809,6 @@ def _impose_covered_static_return_envelope(
"copilot_output_contract_envelope_imposed_after_steering",
block_label=label,
canonical_output_contract_signature=signature,
- steering_reject_count=_output_contract_reject_count(ctx, signature),
return_envelope_applied=True,
metadata_scaffold_applied=scaffolded_metadata is not raw_code_artifact_metadata,
)
@@ -6856,19 +4917,10 @@ def _impose_output_contract_envelope_after_steering(
advisory_run_grantable=True,
declaration_paths=declaration_paths,
)
- if actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL:
- _stash_output_source_unobservable_terminal(
- ctx,
- reason_code=actuation.reason_code,
- required_paths=required_paths,
- block_label=label,
- signature=signature,
- blockers=split.blockers,
- )
- return workflow_yaml, raw_code_artifact_metadata, declaration_stamped
- if actuation.kind == OutputContractActuationKind.ADVISORY_RUN:
- return workflow_yaml, raw_code_artifact_metadata, declaration_stamped
- if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
+ if actuation.kind in {
+ OutputContractActuationKind.BLOCKED_TERMINAL,
+ OutputContractActuationKind.ADVISORY_RUN,
+ }:
return workflow_yaml, raw_code_artifact_metadata, declaration_stamped
attempt_key = _output_contract_spine_directive_attempt_key(
signature=signature, block_label=label, workflow_yaml=workflow_yaml
@@ -6899,506 +4951,79 @@ def _impose_output_contract_envelope_after_steering(
observation_paths=observation_paths,
declaration_paths=declaration_paths,
source=source,
- reason_code=reason_code,
- signature=signature,
- runtime_attempt_key=runtime_attempt_key,
- )
- if produced is not None:
- produced_yaml, produced_metadata, produced_changed = produced
- return produced_yaml, produced_metadata, produced_changed or declaration_stamped
- scaffolded_metadata = _apply_metadata_contract_scaffold(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- required_paths=observation_paths,
- source=source,
- reason_code=reason_code,
- declaration_paths=declaration_paths,
- allow_missing_static_return=True,
- )
- scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
- if scaffolded:
- _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
- LOG.info(
- "copilot_output_contract_formation_applied",
- block_label=label,
- canonical_output_contract_signature=signature,
- return_envelope_applied=False,
- metadata_scaffold_applied=True,
- )
- LOG.info(
- "copilot_output_contract_envelope_scaffold_bailed",
- reason="static_return_envelope_unavailable",
- block_label=label,
- canonical_output_contract_signature=signature,
- violations=violations,
- )
- actuation = _actuate_output_contract_bail(
- ctx,
- blockers=["static_return_envelope_unavailable"],
- target_code=target_code,
- required_paths=observation_paths,
- signature=signature,
- current_fingerprint=current_fingerprint,
- declaration_paths=declaration_paths,
- )
- if actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL:
- _stash_output_source_unobservable_terminal(
- ctx,
- reason_code=actuation.reason_code,
- required_paths=required_paths,
- block_label=label,
- signature=signature,
- blockers=["static_return_envelope_unavailable"],
- )
- return workflow_yaml, scaffolded_metadata, scaffolded or declaration_stamped
- changed_code = keyed_code != textwrap.dedent(target_code).strip()
- if changed_code:
- for block in _workflow_code_blocks(parsed):
- if str(block.get("label") or "").strip() == label:
- block["code"] = keyed_code.rstrip() + "\n"
- break
- workflow_yaml = yaml.safe_dump(parsed, sort_keys=False)
- scaffolded_metadata = _apply_metadata_contract_scaffold(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- required_paths=observation_paths,
- source=source,
- reason_code=reason_code,
- declaration_paths=declaration_paths,
- )
- scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
- applied = changed_code or scaffolded
- if applied:
- _mark_output_contract_imposed(ctx, signature)
- _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
- LOG.info(
- "copilot_output_contract_envelope_imposed_after_steering",
- block_label=label,
- canonical_output_contract_signature=signature,
- steering_reject_count=_output_contract_reject_count(ctx, signature),
- return_envelope_applied=changed_code,
- metadata_scaffold_applied=scaffolded,
- )
- return workflow_yaml, scaffolded_metadata, applied or declaration_stamped
-
-
-def _has_granted_output_contract_advisory_run(ctx: AgentContext) -> bool:
- return any(
- state == OutputContractAdvisoryState.GRANTED for state in ctx.output_contract_actuation_by_signature.values()
- )
-
-
-def _metadata_preflight_reject_yields_to_ladder(ctx: AgentContext) -> bool:
- """Register the preflight reject's claim and report whether it must yield to a live GRANTED advisory run.
-
- The reject always registers its (transient) claim. It yields only when a GRANTED advisory — the ladder's
- decision to dispatch despite deficiencies — owns the turn: yielding lets the forced dispatch consume the grant
- instead of re-blocking it. An ordinary reject cycle (a landed actuation without a grant) is the ladder's own
- mechanism, not a contradiction, so the reject still fires; and it does not yield to a genuinely-terminal owner
- (e.g. a just-committed early terminal), which legitimately ends the turn rather than dispatching a run behind it.
- """
- owner = current_turn_owner(ctx)
- claim_turn(ctx, TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT)
- return (
- _has_granted_output_contract_advisory_run(ctx)
- and owner is not None
- and owner.claimant is TurnClaimant.OUTPUT_CONTRACT_ACTUATION
- )
-
-
-def _run_dispatch_definition_reject(ctx: AgentContext, workflow_yaml: str) -> dict[str, Any] | None:
- # A keyless draft may persist (draft-first authoring), but a run must never dispatch
- # while declared string keys are unreferenced — mirrors the update_and_run gate.
- if not workflow_yaml:
- return None
- definition_reject = _definition_plane_preflight_reject(
- ctx,
- workflow_yaml,
- enforce_untagged_declared_inputs=True,
- )
- if definition_reject is None:
- return None
- halted = _stash_unresolved_recorded_outcome_grounding_halt(
- ctx,
- definition_reject.unreferenced_parameter_keys,
- )
- if not halted:
- if _record_definition_plane_ablation_event(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=ctx.code_artifact_metadata,
- ):
- return None
- _record_definition_plane_reject(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=ctx.code_artifact_metadata,
- )
- return {
- "ok": False,
- "error": _definition_plane_reject_error(definition_reject),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": {
- "reason_code": "definition_contract_unsatisfied",
- "definition_criterion_ids": list(definition_reject.criterion_ids),
- "definition_reason_codes": list(definition_reject.reason_codes),
- "unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),
- },
- }
-
-
-def _metadata_contract_run_preflight_reject(
- ctx: AgentContext,
- workflow_yaml: str,
- raw_code_artifact_metadata: object,
- runtime_parameters: Mapping[str, Any] | None = None,
- *,
- enforce_untagged_declared_inputs: bool = False,
-) -> dict[str, Any] | None:
- if isinstance(ctx, AgentContext):
- ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(raw_code_artifact_metadata)
- definition_reject = (
- _definition_plane_preflight_reject(
- ctx,
- workflow_yaml,
- enforce_untagged_declared_inputs=enforce_untagged_declared_inputs,
- )
- if isinstance(ctx, AgentContext)
- else None
- )
- parsed = parse_workflow_yaml(workflow_yaml)
- binding_resolution = (
- _authoring_parameter_binding_resolution_for_definition(
- ctx,
- parsed,
- runtime_parameters,
- raw_code_artifact_metadata,
- )
- if isinstance(ctx, AgentContext) and isinstance(parsed, Mapping)
- else None
- )
- binding_path_available = binding_resolution is not None and (
- binding_resolution.snapshot is not None or binding_resolution.directive is not None
- )
- if definition_reject is not None and (enforce_untagged_declared_inputs or not binding_path_available):
- definition_corrective = (
- None
- if copilot_author_time_gate_log_only_enabled(ctx, DEFINITION_CONTRACT_UNSATISFIED_GATE_ID)
- else _value_bearing_definition_reject_corrective(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_code_artifact_metadata,
- )
- )
- definition_data = {
- "reason_code": "definition_contract_unsatisfied",
- "definition_criterion_ids": list(definition_reject.criterion_ids),
- "definition_reason_codes": list(definition_reject.reason_codes),
- "unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),
- **(definition_corrective or {}),
- }
- halted = _stash_unresolved_recorded_outcome_grounding_halt(
- ctx,
- definition_reject.unreferenced_parameter_keys,
- )
- if halted:
- return {
- "ok": False,
- "error": _definition_plane_reject_error(definition_reject),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": definition_data,
- }
- if not _record_definition_plane_ablation_event(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=raw_code_artifact_metadata,
- ):
- _record_definition_plane_reject(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=raw_code_artifact_metadata,
- )
- return {
- "ok": False,
- "error": _definition_plane_reject_error(definition_reject),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": definition_data,
- }
- convergence_reject = _recorded_outcome_convergence_reject(
- ctx,
- workflow_yaml=workflow_yaml,
- code_artifact_metadata=raw_code_artifact_metadata,
- )
- if convergence_reject is not None and not binding_path_available:
- block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- _record_author_time_reject_outcome(
- ctx,
- reason_code="unchanged_after_recorded_outcome",
- summary="The authored code and output structure are unchanged after the last recorded test outcome.",
- structural_payload={
- "reason_code": "unchanged_after_recorded_outcome",
- "authored_structure_signature": convergence_reject.authored_structure_signature,
- "block_labels": block_labels,
- },
- authored_structure_signature=convergence_reject.authored_structure_signature,
- block_labels=block_labels,
- )
- _record_code_authoring_guardrail_reject(
- ctx, frontier_unchanged=convergence_reject.reason == "frontier_unchanged"
- )
- LOG.info(
- "copilot recorded outcome convergence behavior",
- convergence_reason=convergence_reject.reason,
- commit_early_terminal=convergence_reject.commit_early_terminal,
- block_labels=block_labels,
- )
- if convergence_reject.commit_early_terminal:
- _commit_recorded_outcome_early_terminal(ctx)
- if _metadata_preflight_reject_yields_to_ladder(ctx):
- return None
- return {
- "ok": False,
- "error": (
- "Submitted workflow left the frontier the last recorded test outcome named unchanged. "
- "Revise the code block or output metadata that owns that frontier before testing again."
- ),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(),
- }
- evaluation = _evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- raw_code_artifact_metadata,
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- if evaluation is None or not evaluation.has_deficiencies:
- return None
- if (
- _credit_output_contract_submission_at_ceiling(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_code_artifact_metadata,
- )
- is not None
- ):
- # The steer loop consumed its budget on this contract and the submission satisfies it
- # under ceiling crediting: let the run proceed instead of re-emitting the directive.
- return None
- if _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in evaluation.shape_violations:
- _revoke_output_contract_advisory_run(ctx, evaluation.canonical_signature)
- authored_fingerprint = _output_contract_structural_fingerprint(workflow_yaml, evaluation.canonical_signature)
- advisory_granted = _output_contract_advisory_granted(ctx, evaluation.canonical_signature)
- # A granted advisory must arm run-output evidence before the grant is consumed, even when the
- # workflow is otherwise run-attemptable — otherwise the dispatched run records no evidence.
- if evaluation.can_attempt_run and not advisory_granted:
- return None
- if _record_output_contract_ablation_event(
- ctx,
- evaluation,
- gate_id=_METADATA_PREFLIGHT_ABLATION_GATE_ID,
- blocked_tool="update_and_run_blocks",
- fingerprint=authored_fingerprint,
- ):
- return None
- payload = _record_output_contract_reject(
- ctx,
- evaluation,
- summary="Submitted workflow does not satisfy the requested output contract before run.",
- authored_structural_fingerprint=authored_fingerprint,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_code_artifact_metadata,
- )
- if (
- evaluation.can_attempt_run or _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE not in evaluation.shape_violations
- ) and (advisory_granted or _output_contract_advisory_granted(ctx, evaluation.canonical_signature)):
- _arm_pending_run_evidence(ctx, evaluation.canonical_signature, set(evaluation.observation_paths))
- return None
- if evaluation.repair_context is not None:
- ctx.last_code_authoring_repair_context = evaluation.repair_context
- if _metadata_preflight_reject_yields_to_ladder(ctx):
- return None
- if payload.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE:
- assert ctx.blocker_signal is not None
- return {
- "ok": False,
- "error": ctx.blocker_signal.user_facing_reason,
- "user_facing_summary": ctx.blocker_signal.user_facing_reason,
- "data": payload,
- }
- if "value_bearing_convergence_directive" in payload:
- return _output_contract_reject_result(
- evaluation,
- payload=payload,
- tool_name="update_and_run_blocks",
- )
- payload = dict(payload)
- payload["output_contract_reason_code"] = payload.get("reason_code")
- payload["reason_code"] = _METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE
- payload["reject_reason"] = _METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE
- block_label = evaluation.block_label or "the target output block"
- return {
- "ok": False,
- "error": (
- "update_and_run_blocks cannot attempt a run until structurally complete "
- f"`code_artifact_metadata` is submitted for `{block_label}`."
- ),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": payload,
- }
-
-
-def metadata_same_key_terminal_preflight(
- ctx: AgentContext,
- workflow_yaml: str,
- raw_code_artifact_metadata: object,
-) -> dict[str, CodeArtifactMetadataValue] | None:
- prior_state = ctx.metadata_reject_ladder_state
- if (
- prior_state is None
- or prior_state.streak_count < 2
- or copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID)
- ):
- return None
- missing_error = _missing_code_artifact_metadata_error(workflow_yaml, ctx, raw_code_artifact_metadata)
- if missing_error is None:
- return None
- missing_labels = _missing_code_artifact_metadata_labels(workflow_yaml, ctx, raw_code_artifact_metadata)
- if len(missing_labels) != 1:
- return None
- required_paths, _, coverage_reason_code = _required_child_output_paths_for_authoring(ctx)
- reject_payload = _code_artifact_metadata_reject_payload(
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_code_artifact_metadata,
- offending_labels=[],
- missing_labels=missing_labels,
- violation_categories=["missing_code_artifact_metadata"],
- )
- candidate_outcome = _build_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=missing_error,
- structural_payload=reject_payload,
- block_labels=missing_labels,
- missing_requested_output_facts=_missing_requested_output_facts(
- required_paths,
- reason_code=coverage_reason_code,
- ),
- )
- structural_key = candidate_outcome.structural_key
- missing_fields_by_label = _metadata_missing_required_fields_by_label(
- raw_code_artifact_metadata,
- labels=missing_labels,
- missing_labels=missing_labels,
- )
- if (
- structural_key is None
- or prior_state.reject_family != "missing_code_artifact_metadata"
- or prior_state.structural_key != structural_key
- or prior_state.gate_id != "code_artifact_metadata"
- or prior_state.missing_fields_by_label != missing_fields_by_label
- ):
- return None
- decision = _metadata_reject_ladder_decision(
- ctx,
- candidate_outcome=candidate_outcome,
- reject_family="missing_code_artifact_metadata",
- missing_fields_by_label=missing_fields_by_label,
- )
- if decision is None or decision.action != "terminal":
- return None
- assert ctx.blocker_signal is not None
- return {
- "ok": False,
- "error": ctx.blocker_signal.user_facing_reason,
- "user_facing_summary": ctx.blocker_signal.user_facing_reason,
- "data": _metadata_reject_terminal_data(decision),
- }
-
-
-def output_contract_value_bearing_run_reject(
- ctx: AgentContext,
- code_by_label: Mapping[str, str],
-) -> dict[str, Any] | None:
- """Refuses dispatch of a saved workflow whose author contract is statically provably
- valueless. Observation paths stay fail-open when unknown; a root-only repair must prove its envelope."""
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- contract = _output_contract_required_paths_source(ctx)
- declaration_only_contract = bool(contract.declaration_paths) and not contract.observation_paths
- root_only_liveness_contract = (
- contract.liveness is _OutputContractLiveness.DEGRADED_EMPTY or declaration_only_contract
- )
- if root_only_liveness_contract:
- statically_valueless = not code_by_label or all(
- _root_output_envelope_state(code) != "proven"
- or _statically_lacks_value_bearing_observation_paths(code, {_VALUE_BEARING_ROOT_GUIDANCE_PATH})
- for code in code_by_label.values()
+ reason_code=reason_code,
+ signature=signature,
+ runtime_attempt_key=runtime_attempt_key,
)
- else:
- statically_valueless = False
- if contract.liveness is _OutputContractLiveness.VALUE_REQUIRED and code_by_label:
- statically_valueless = all(
- _statically_lacks_value_bearing_observation_paths(code, contract.observation_paths)
- for code in code_by_label.values()
- )
- # The code lane is criteria-independent so the invariant survives drift in the persisted
- # criteria source (an unparseable row must not readmit a statically hollow envelope).
- statically_empty_return_envelope = False
- if not statically_valueless and code_by_label:
- statically_empty_return_envelope = all(
- _statically_valueless_return_envelope(code) for code in code_by_label.values()
- )
- statically_valueless = statically_empty_return_envelope
- if not statically_valueless:
- return None
- LOG.info(
- "copilot_value_bearing_dispatch_preflight_reject",
- contract_liveness=contract.liveness.value,
- canonical_required_child_paths=sorted(contract.union),
- declaration_only_child_paths=sorted(contract.declaration_paths),
- statically_empty_return_envelope=statically_empty_return_envelope,
- block_labels=sorted(code_by_label),
- )
- if statically_empty_return_envelope and not contract.union:
- error = (
- "Cannot run the saved workflow: its code statically returns no values (every return "
- "envelope is empty or null-only). Revise the code to return at least one real value "
- "before running."
+ if produced is not None:
+ produced_yaml, produced_metadata, produced_changed = produced
+ return produced_yaml, produced_metadata, produced_changed or declaration_stamped
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=observation_paths,
+ source=source,
+ reason_code=reason_code,
+ declaration_paths=declaration_paths,
+ allow_missing_static_return=True,
)
- elif contract.liveness is _OutputContractLiveness.DEGRADED_EMPTY or declaration_only_contract:
- error = (
- "Cannot run the saved workflow: its code does not prove a non-empty top-level `output` "
- "value. Revise the owning code block to return a real value under `output` before running."
+ scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
+ if scaffolded:
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_formation_applied",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ return_envelope_applied=False,
+ metadata_scaffold_applied=True,
+ )
+ LOG.info(
+ "copilot_output_contract_envelope_scaffold_bailed",
+ reason="static_return_envelope_unavailable",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ violations=violations,
)
- else:
- error = (
- "Cannot run the saved workflow: its output contract carries no value-bearing output "
- "(only a blocker declaration or statically empty values). Revise the owning code block "
- "to produce at least one requested value-bearing output before running."
+ actuation = _actuate_output_contract_bail(
+ ctx,
+ blockers=["static_return_envelope_unavailable"],
+ target_code=target_code,
+ required_paths=observation_paths,
+ signature=signature,
+ current_fingerprint=current_fingerprint,
+ declaration_paths=declaration_paths,
)
- return {
- "ok": False,
- "error": error,
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": {
- "reason_code": _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- "reject_reason": _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- "contract_liveness": contract.liveness.value,
- "canonical_required_child_paths": sorted(contract.union),
- "declaration_only_child_paths": sorted(contract.declaration_paths),
- "statically_empty_return_envelope": statically_empty_return_envelope,
- "degraded_request_slots": [slot.to_payload() for slot in contract.degraded_request_slots],
- "can_attempt_run": False,
- },
- }
+ return workflow_yaml, scaffolded_metadata, scaffolded or declaration_stamped
+ changed_code = keyed_code != textwrap.dedent(target_code).strip()
+ if changed_code:
+ for block in _workflow_code_blocks(parsed):
+ if str(block.get("label") or "").strip() == label:
+ block["code"] = keyed_code.rstrip() + "\n"
+ break
+ workflow_yaml = yaml.safe_dump(parsed, sort_keys=False)
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=observation_paths,
+ source=source,
+ reason_code=reason_code,
+ declaration_paths=declaration_paths,
+ )
+ scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
+ applied = changed_code or scaffolded
+ if applied:
+ _mark_output_contract_imposed(ctx, signature)
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_envelope_imposed_after_steering",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ return_envelope_applied=changed_code,
+ metadata_scaffold_applied=scaffolded,
+ )
+ return workflow_yaml, scaffolded_metadata, applied or declaration_stamped
def _output_path_root(path: str) -> str:
@@ -7409,49 +5034,6 @@ def _output_path_has_child(path: str) -> bool:
return "." in path or "[" in path
-def _candidate_missing_required_output_paths(
- workflow_yaml: str,
- code_artifact_metadata: object,
- *,
- required_paths: set[str],
-) -> list[str]:
- if not required_paths:
- return []
- required_paths = {path for path in (str(item).strip() for item in required_paths) if path}
- declared_paths_by_label = {
- label: set(paths) for label, paths in _metadata_output_paths_by_label(code_artifact_metadata).items()
- }
- produced_by_label = _workflow_yaml_produced_output_roots_by_label(workflow_yaml)
- covered_paths: set[str] = set()
- abstained_declared_paths: set[str] = set()
- for label, declared_paths in declared_paths_by_label.items():
- produced = produced_by_label.get(label)
- if produced is None:
- continue
- for required_path in required_paths & declared_paths:
- if _top_level_path_segment(required_path) in produced.roots:
- covered_paths.add(required_path)
- if produced.abstained:
- abstained_declared_paths.update(required_paths & declared_paths)
- missing_paths = required_paths - covered_paths
- if missing_paths:
- missing_paths -= abstained_declared_paths
- return sorted(missing_paths)
-
-
-def _recorded_runtime_produced_output_roots(ctx: AgentContext, workflow_yaml: str) -> set[str]:
- verified_outputs = getattr(ctx, "verified_block_outputs", None)
- if not isinstance(verified_outputs, Mapping):
- return set()
- code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- roots: set[str] = set()
- for label, output in verified_outputs.items():
- if not isinstance(label, str) or label not in code_block_labels:
- continue
- roots.update(_meaningful_runtime_output_roots(output))
- return roots
-
-
def _meaningful_runtime_output_roots(value: object, *, prefix: str = "") -> set[str]:
roots: set[str] = set()
if isinstance(value, Mapping):
@@ -7489,73 +5071,6 @@ class _ProducedOutputRoots(NamedTuple):
abstained: bool = False
-def _workflow_yaml_produced_output_roots_by_label(workflow_yaml: str) -> dict[str, _ProducedOutputRoots]:
- return {
- label: _code_block_produced_output_roots(str(block.get("code") or ""))
- for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items()
- }
-
-
-def _workflow_yaml_produced_output_roots(workflow_yaml: str) -> set[str]:
- roots: set[str] = set()
- for produced in _workflow_yaml_produced_output_roots_by_label(workflow_yaml).values():
- roots.update(produced.roots)
- return roots
-
-
-def _workflow_yaml_produced_output_paths(workflow_yaml: str) -> set[str]:
- paths: set[str] = set()
- for block in _workflow_yaml_code_blocks_by_label(workflow_yaml).values():
- paths.update(_code_block_produced_output_paths(str(block.get("code") or "")))
- return paths
-
-
-def _code_block_produced_output_roots(code: str) -> _ProducedOutputRoots:
- try:
- tree = ast.parse(textwrap.dedent(code).strip() or "pass")
- except SyntaxError:
- return _ProducedOutputRoots(set(), True)
- scope_statements = list(_iter_top_level_scope(tree.body))
- assigned_roots = _assigned_top_level_names(tree.body)
- dict_assignments: dict[str, set[str]] = {}
- dynamic_dict_assignment_names: set[str] = set()
- helper_return_roots = _helper_function_literal_return_roots(tree.body)
- returned_roots: set[str] = set()
- abstained = False
- saw_return = False
- for node in scope_statements:
- if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
- if isinstance(node.value, ast.Dict):
- produced = _dict_literal_output_roots(node.value)
- dict_assignments[node.targets[0].id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(node.targets[0].id)
- elif isinstance(node, ast.Assign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.AnnAssign):
- if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):
- produced = _dict_literal_output_roots(node.value)
- dict_assignments[node.target.id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(node.target.id)
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.AugAssign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.Return):
- saw_return = True
- if node.value is not None:
- returned = _return_output_roots(
- node.value,
- dict_assignments,
- dynamic_dict_assignment_names,
- helper_return_roots,
- )
- returned_roots.update(returned.roots)
- abstained = abstained or returned.abstained
- roots = returned_roots if saw_return else assigned_roots
- return _ProducedOutputRoots(roots, abstained)
-
-
def _code_block_produced_output_paths(code: str) -> set[str]:
try:
tree = ast.parse(textwrap.dedent(code).strip() or "pass")
@@ -7883,54 +5398,6 @@ def _list_literal_output_roots(node: ast.List) -> _ProducedOutputRoots:
return _ProducedOutputRoots(set(), True)
-def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, _ProducedOutputRoots]:
- helpers: dict[str, _ProducedOutputRoots] = {}
- for statement in statements:
- if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
- continue
- dict_assignments: dict[str, set[str]] = {}
- dynamic_dict_assignment_names: set[str] = set()
- roots: set[str] = set()
- abstained = False
- saw_return = False
- for helper_statement in _iter_top_level_scope(statement.body):
- if (
- isinstance(helper_statement, ast.Assign)
- and len(helper_statement.targets) == 1
- and isinstance(helper_statement.targets[0], ast.Name)
- ):
- if isinstance(helper_statement.value, ast.Dict):
- produced = _dict_literal_output_roots(helper_statement.value)
- dict_assignments[helper_statement.targets[0].id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(helper_statement.targets[0].id)
- elif isinstance(helper_statement, ast.Assign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
- elif isinstance(helper_statement, ast.AnnAssign):
- if isinstance(helper_statement.target, ast.Name) and isinstance(helper_statement.value, ast.Dict):
- produced = _dict_literal_output_roots(helper_statement.value)
- dict_assignments[helper_statement.target.id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(helper_statement.target.id)
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
- elif isinstance(helper_statement, ast.AugAssign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
- elif isinstance(helper_statement, ast.Return):
- saw_return = True
- if helper_statement.value is not None:
- returned = _return_output_roots(
- helper_statement.value,
- dict_assignments,
- dynamic_dict_assignment_names,
- {},
- )
- roots.update(returned.roots)
- abstained = abstained or returned.abstained
- if roots or abstained or saw_return:
- helpers[statement.name] = _ProducedOutputRoots(roots, abstained)
- return helpers
-
-
def _helper_function_literal_return_paths(statements: list[ast.stmt]) -> dict[str, set[str]]:
helpers: dict[str, set[str]] = {}
for statement in statements:
@@ -7975,16 +5442,6 @@ def _dict_literal_output_roots(node: ast.Dict) -> _ProducedOutputRoots:
return _ProducedOutputRoots(roots, abstained)
-def _dict_literal_string_key_roots(node: ast.expr) -> set[str]:
- if not isinstance(node, ast.Dict):
- return set()
- roots: set[str] = set()
- for key in node.keys:
- if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.strip():
- roots.add(key.value.strip())
- return roots
-
-
def _dict_literal_string_key_paths(
node: ast.expr,
dict_assignments: Mapping[str, set[str]],
@@ -7994,202 +5451,27 @@ def _dict_literal_string_key_paths(
if isinstance(node, ast.List):
array_prefix = f"{prefix}[]" if prefix else "[]"
array_paths: set[str] = set()
- for item in node.elts:
- array_paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
- return array_paths
- if not isinstance(node, ast.Dict):
- return set()
- paths: set[str] = set()
- for key_node, value_node in zip(node.keys, node.values):
- if not (isinstance(key_node, ast.Constant) and isinstance(key_node.value, str) and key_node.value.strip()):
- continue
- path = f"{prefix}.{key_node.value.strip()}" if prefix else key_node.value.strip()
- paths.add(path)
- if isinstance(value_node, ast.Dict):
- paths.update(_dict_literal_string_key_paths(value_node, dict_assignments, prefix=path))
- elif isinstance(value_node, ast.List):
- array_prefix = f"{path}[]"
- for item in value_node.elts:
- paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
- elif isinstance(value_node, ast.Name):
- for child_path in dict_assignments.get(value_node.id, set()):
- paths.add(f"{path}{child_path}" if child_path.startswith("[]") else f"{path}.{child_path}")
- return paths
-
-
-def _output_empty_code_block_labels(workflow_yaml: str, code_artifact_metadata: object) -> list[str]:
- code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- if not code_blocks:
- return []
- metadata_by_label = code_artifact_metadata if isinstance(code_artifact_metadata, Mapping) else {}
- labels_without_output = [
- label
- for label, block in code_blocks.items()
- if not _code_block_has_meaningful_output(
- str(block.get("code") or ""),
- metadata_by_label.get(label),
- )
- ]
- return sorted(labels_without_output) if len(labels_without_output) == len(code_blocks) else []
-
-
-def _code_block_has_meaningful_output(code: str, artifact: object) -> bool:
- try:
- tree = ast.parse(code)
- except SyntaxError:
- return False
- helper_return_roots = _helper_function_literal_return_roots(tree.body)
-
- class ReturnVisitor(ast.NodeVisitor):
- def __init__(self) -> None:
- self.has_meaningful_return = False
-
- def visit_Return(self, node: ast.Return) -> None:
- if _return_value_is_meaningful(node.value, helper_return_roots):
- self.has_meaningful_return = True
-
- def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
- return
-
- def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
- return
-
- def visit_ClassDef(self, node: ast.ClassDef) -> None:
- return
-
- visitor = ReturnVisitor()
- visitor.visit(tree)
- if visitor.has_meaningful_return:
- return True
- if _code_block_has_top_level_return(tree):
- return False
- if isinstance(artifact, Mapping):
- goal_roots = _artifact_goal_value_roots(artifact)
- if goal_roots and _missing_declared_output_roots(code, goal_roots) is None:
- return True
- return False
-
-
-@dataclass(frozen=True)
-class _ScoutedSelectOptionInteraction:
- selector: str
- source_origin: str
- value_present: bool
- label_present: bool
-
-
-def _scouted_select_option_interactions(ctx: AgentContext) -> list[_ScoutedSelectOptionInteraction]:
- trajectory = getattr(ctx, "scout_trajectory", None)
- if not isinstance(trajectory, list):
- return []
- interactions: list[_ScoutedSelectOptionInteraction] = []
- seen: set[tuple[str, str, bool, bool]] = set()
- for interaction in trajectory:
- if not isinstance(interaction, Mapping):
- continue
- if str(interaction.get("tool_name") or "").strip() != "select_option":
- continue
- selector = _safe_selector_repair_atom(interaction.get("selector"))
- if not selector:
- continue
- raw_source_url = str(interaction.get("source_url") or "").strip()
- source_origin = (url_origin(raw_source_url) or "") if raw_source_url else ""
- entry = _ScoutedSelectOptionInteraction(
- selector=selector,
- source_origin=source_origin,
- value_present=bool(str(interaction.get("value") or "").strip()),
- label_present=bool(str(interaction.get("label") or interaction.get("option_label") or "").strip()),
- )
- key = (entry.selector, entry.source_origin, entry.value_present, entry.label_present)
- if key in seen:
- continue
- seen.add(key)
- interactions.append(entry)
- return interactions
-
-
-def _select_option_text_click_repair_context(
- workflow_yaml: str, ctx: AgentContext
-) -> CodeAuthoringRepairContext | None:
- expected_interactions = _scouted_select_option_interactions(ctx)
- if not expected_interactions:
- return None
- code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- candidate_selectors = _workflow_select_option_call_selectors(code_blocks.values())
- if None in candidate_selectors:
- return None
- missing_interactions = [
- interaction for interaction in expected_interactions if interaction.selector not in candidate_selectors
- ]
- if not missing_interactions:
- return None
- for label, block in code_blocks.items():
- tree = _parsed_code_tree(str(block.get("code") or ""))
- if tree is None or not _code_contains_get_by_text_click(tree):
- continue
- missing_interaction = missing_interactions[0]
- return CodeAuthoringRepairContext(
- block_label=label,
- reason_code="select_option_interaction_mismatch",
- selector=missing_interaction.selector,
- source_url=missing_interaction.source_origin or None,
- current_origin=missing_interaction.source_origin or None,
- binding_candidates=[
- "expected_tool:select_option",
- "authored_tool:get_by_text_click",
- f"value_present:{missing_interaction.value_present}",
- f"label_present:{missing_interaction.label_present}",
- ],
- repair_instruction=(
- "Use the captured select element API for this interaction, for example "
- "page.locator(selector).select_option(...), instead of clicking option text."
- ),
- )
- return None
-
-
-def _workflow_select_option_call_selectors(code_blocks: Iterable[Mapping[str, Any]]) -> set[str | None]:
- selectors: set[str | None] = set()
- for block in code_blocks:
- tree = _parsed_code_tree(str(block.get("code") or ""))
- if tree is not None:
- selectors.update(_select_option_call_selectors(tree))
- return selectors
-
-
-def _parsed_code_tree(code: str) -> ast.AST | None:
- try:
- return ast.parse(code)
- except SyntaxError:
- return None
-
-
-def _code_contains_get_by_text_click(tree: ast.AST) -> bool:
- for node in ast.walk(tree):
- if (
- isinstance(node, ast.Call)
- and isinstance(node.func, ast.Attribute)
- and node.func.attr == "click"
- and _call_chain_contains_method(node.func.value, "get_by_text")
- ):
- return True
- return False
-
-
-def _select_option_call_selectors(tree: ast.AST) -> set[str | None]:
- locator_aliases = _locator_alias_selectors(tree)
- selectors: set[str | None] = set()
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
- continue
- if node.func.attr != "select_option":
+ for item in node.elts:
+ array_paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
+ return array_paths
+ if not isinstance(node, ast.Dict):
+ return set()
+ paths: set[str] = set()
+ for key_node, value_node in zip(node.keys, node.values):
+ if not (isinstance(key_node, ast.Constant) and isinstance(key_node.value, str) and key_node.value.strip()):
continue
- resolved = _locator_receiver_selectors(node.func.value, locator_aliases)
- if resolved:
- selectors.update(resolved)
- else:
- selectors.add(None)
- return selectors
+ path = f"{prefix}.{key_node.value.strip()}" if prefix else key_node.value.strip()
+ paths.add(path)
+ if isinstance(value_node, ast.Dict):
+ paths.update(_dict_literal_string_key_paths(value_node, dict_assignments, prefix=path))
+ elif isinstance(value_node, ast.List):
+ array_prefix = f"{path}[]"
+ for item in value_node.elts:
+ paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
+ elif isinstance(value_node, ast.Name):
+ for child_path in dict_assignments.get(value_node.id, set()):
+ paths.add(f"{path}{child_path}" if child_path.startswith("[]") else f"{path}.{child_path}")
+ return paths
def _locator_alias_selectors(tree: ast.AST) -> dict[str, set[str]]:
@@ -8243,11 +5525,6 @@ def _call_chain_contains_method(node: ast.AST, method_name: str) -> bool:
return False
-def _code_block_has_top_level_return(tree: ast.AST) -> bool:
- body = tree.body if isinstance(tree, ast.Module) else []
- return any(isinstance(statement, ast.Return) for statement in _iter_top_level_scope(body))
-
-
def _return_value_is_meaningful(
node: ast.expr | None,
helper_return_roots: Mapping[str, _ProducedOutputRoots],
@@ -8324,124 +5601,10 @@ def _raw_workflow_yaml_conflict_marker_error(workflow_yaml: str) -> str | None:
return None
-def _declared_workflow_parameter_keys(parsed: dict[str, Any]) -> set[str] | None:
- workflow_definition = parsed.get("workflow_definition")
- if not isinstance(workflow_definition, dict):
- return set()
- parameters = workflow_definition.get("parameters")
- if parameters is None:
- return set()
- if not isinstance(parameters, list):
- return None
- return {
- str(parameter.get("key") or "").strip()
- for parameter in parameters
- if isinstance(parameter, Mapping) and str(parameter.get("key") or "").strip()
- }
-
-
_ORDERED_CHILD_BLOCK_LIST_KEYS = ("loop_blocks", "blocks")
_ORDERED_BRANCH_LIST_KEYS = ("branch_conditions", "branches", "ordered_branches")
-def _code_block_parameter_contract_error(workflow_yaml: str) -> str | None:
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return None
- declared_parameter_keys = _declared_workflow_parameter_keys(parsed)
- if declared_parameter_keys is None:
- return "Unable to validate code-block parameter keys: workflow_definition.parameters must be a list."
- registered_download_keys = set(REGISTERED_DOWNLOAD_OUTPUT_KEYS)
-
- errors: list[str] = []
-
- def block_output_key(block: Mapping[str, Any]) -> str | None:
- label = str(block.get("label") or "").strip()
- return f"{label}_output" if label else None
-
- def check_code_block(block: Mapping[str, Any], available_parameter_keys: set[str]) -> None:
- label = str(block.get("label") or "").strip() or "unlabeled code block"
- code = str(block.get("code") or "")
- for line_number, line in enumerate(code.splitlines(), start=1):
- marker = _conflict_marker_for_line(line)
- if marker is not None:
- errors.append(
- f"Code block `{label}` contains unresolved conflict marker `{marker}` on code line "
- f"{line_number}. Remove every git conflict marker line before retrying."
- )
- break
- parameter_keys = _code_block_parameter_keys(block)
- registered_keys = sorted(parameter_keys & registered_download_keys)
- if registered_keys:
- joined = ", ".join(f"`{key}`" for key in registered_keys)
- errors.append(
- f"Code block `{label}` lists registered download output key(s) {joined} in `parameter_keys`, "
- "but the execution layer injects registered download output keys only after a browser download "
- "fires. Remove them from `parameter_keys` and return a small descriptor instead."
- )
- undeclared_keys = sorted(parameter_keys - available_parameter_keys - registered_download_keys)
- if undeclared_keys:
- joined = ", ".join(f"`{key}`" for key in undeclared_keys)
- errors.append(
- f"Code block `{label}` lists undeclared `parameter_keys`: {joined}. Declare each workflow input "
- "under `workflow_definition.parameters`, use a prior block output key such as "
- "`<block_label>_output`, or remove stale `parameter_keys` entries before retrying."
- )
-
- def visit_branch(branch: Mapping[str, Any], available_parameter_keys: set[str]) -> None:
- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
- # Child scopes inherit known keys without leaking their outputs back into sibling branches.
- visit_blocks(branch.get(key), set(available_parameter_keys))
- for branch_key in _ORDERED_BRANCH_LIST_KEYS:
- branches = branch.get(branch_key)
- if not isinstance(branches, list):
- continue
- for nested_branch in branches:
- if isinstance(nested_branch, Mapping):
- # Branch scopes intentionally isolate output keys from their parent branch.
- visit_branch(nested_branch, set(available_parameter_keys))
-
- def visit_blocks(blocks: Any, available_parameter_keys: set[str]) -> set[str]:
- if not isinstance(blocks, list):
- return available_parameter_keys
- for block in blocks:
- if not isinstance(block, Mapping):
- continue
- if _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
- check_code_block(block, available_parameter_keys)
- for key in _ORDERED_CHILD_BLOCK_LIST_KEYS:
- visit_blocks(block.get(key), set(available_parameter_keys))
- for branch_key in _ORDERED_BRANCH_LIST_KEYS:
- branches = block.get(branch_key)
- if not isinstance(branches, list):
- continue
- for branch in branches:
- if isinstance(branch, Mapping):
- visit_branch(branch, set(available_parameter_keys))
- output_key = block_output_key(block)
- if output_key:
- available_parameter_keys.add(output_key)
- return available_parameter_keys
-
- workflow_definition = parsed.get("workflow_definition")
- blocks = workflow_definition.get("blocks") if isinstance(workflow_definition, dict) else None
- try:
- visit_blocks(blocks, set(declared_parameter_keys))
- except RecursionError:
- return "Workflow YAML nesting is too deep to validate."
- return "\n".join(errors) if errors else None
-
-
-class _NeverCapturedObligationCandidate(NamedTuple):
- block_label: str
- site: str
- method: str
- receiver: str
- call_shape: str
- expected_tool_name: str
- expected_argument_literal: str | None
-
-
def _code_repair_progress_data(
repair_context: CodeAuthoringRepairContext | None = None,
*,
@@ -8480,8 +5643,6 @@ class _SynthesizedCodeImpositionResult:
selected_extraction_metadata_disposition: SelectedExtractionMetadataDisposition = "none"
minted_parameter_keys: list[str] = dataclass_field(default_factory=list)
metadata_repair_contract: dict[str, object] | None = None
- ablation_gate_id: str | None = None
- never_captured_candidate: _NeverCapturedObligationCandidate | None = None
omission_digest: str | None = None
@@ -8673,19 +5834,6 @@ def _missing_code_artifact_metadata_labels(workflow_yaml: str, ctx: AgentContext
]
-def _missing_code_artifact_metadata_error(workflow_yaml: str, ctx: AgentContext, raw_metadata: Any) -> str | None:
- missing = _missing_code_artifact_metadata_labels(workflow_yaml, ctx, raw_metadata)
- if not missing:
- return None
- joined = ", ".join(f"`{label}`" for label in missing)
- return (
- f"Code-only browser workflow output block(s) {joined} must pass `code_artifact_metadata` with concrete "
- "`goal_value_paths` before saving or testing. Include an `extraction_schema` when the user requested "
- "structured fields, and make the code return a keyed dict/list or expose top-level locals matching those "
- "goal path roots."
- )
-
-
def _submitted_code_block_changed(block: Mapping[str, Any], prior_yaml: str | None) -> bool:
label = str(block.get("label") or "").strip()
if not label or not prior_yaml:
@@ -8706,7 +5854,6 @@ def _should_impose_after_update_attempt(ctx: AgentContext, *, repeated_identical
(isinstance(target, ReachedDownloadTarget) and not target.already_registered and bool(target.selector.strip()))
or synthesized_persistence_reopened_after_failed_run(ctx)
or ctx.synthesized_block_reopened_for_credential_scout
- or ctx.synthesized_block_reopened_for_capture_obligation
or repeated_identical_omission
or _author_time_reject_reopens_synthesized_imposition(ctx)
)
@@ -9304,90 +6451,6 @@ _NEVER_CAPTURED_SCOUT_TOOL_BY_METHOD = {
}
-def _never_captured_obligation_candidate(
- provenance: Iterable[_BrowserSurfaceRejectionProvenance],
-) -> _NeverCapturedObligationCandidate | None:
- for record in provenance:
- mutation = record.mutation
- if record.kind != "never_captured" or mutation is None:
- continue
- expected_tool_name = _NEVER_CAPTURED_SCOUT_TOOL_BY_METHOD.get(mutation.method)
- if expected_tool_name is None or not mutation.generator_compatible:
- continue
- if mutation.method == "press" and mutation.argument_literal is None:
- continue
- return _NeverCapturedObligationCandidate(
- block_label=record.block_label,
- site=record.site,
- method=mutation.method,
- receiver=normalized_locator_expr(mutation.receiver),
- call_shape=mutation.call_shape,
- expected_tool_name=expected_tool_name,
- expected_argument_literal=mutation.argument_literal,
- )
- return None
-
-
-def _arm_never_captured_obligation(
- ctx: AgentContext,
- workflow_yaml: str,
- candidate: _NeverCapturedObligationCandidate | None,
- replay_payload: NeverCapturedReplayPayload,
-) -> None:
- if candidate is None:
- return
- turn_id = str(getattr(ctx, "turn_id", ""))
- draft_fingerprint = hashlib.sha256(workflow_yaml.encode()).hexdigest()
- call_shape_digest = hashlib.sha256(candidate.call_shape.encode()).hexdigest()
- identity_payload = "\0".join(
- (
- turn_id,
- draft_fingerprint,
- candidate.block_label,
- candidate.site,
- candidate.method,
- candidate.receiver,
- call_shape_digest,
- )
- )
- identity_digest = hashlib.sha256(identity_payload.encode()).hexdigest()
- if identity_digest in ctx.never_captured_obligation_identity_history:
- return
- armed_after_trajectory_index = max(
- (index for item in ctx.scout_trajectory if isinstance((index := item.get("trajectory_index")), int)),
- default=-1,
- )
- ctx.never_captured_obligation = NeverCapturedObligation(
- identity_digest=identity_digest,
- turn_id=turn_id,
- draft_fingerprint=draft_fingerprint,
- block_label=candidate.block_label,
- site=candidate.site,
- method=candidate.method,
- normalized_receiver=candidate.receiver,
- call_shape_digest=call_shape_digest,
- expected_tool_name=candidate.expected_tool_name,
- armed_after_trajectory_index=armed_after_trajectory_index,
- expected_argument_literal=candidate.expected_argument_literal,
- replay_payload=replay_payload,
- )
- ctx.never_captured_obligation_identity_history.add(identity_digest)
- LOG.info(
- "copilot_never_captured_obligation_armed",
- identity_digest=identity_digest,
- turn_id=turn_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- draft_fingerprint=draft_fingerprint,
- block_label=candidate.block_label,
- site=candidate.site,
- method=candidate.method,
- receiver=candidate.receiver,
- call_shape_digest=call_shape_digest,
- expected_tool_name=candidate.expected_tool_name,
- armed_after_trajectory_index=armed_after_trajectory_index,
- )
-
-
class _BrowserBindings(NamedTuple):
page_aliases: set[str]
locator_aliases: set[str]
@@ -10099,15 +7162,6 @@ def _synthesized_resubmission_credential_scout_requirements(
return _credentialed_code_block_scout_gate_errors(probe_yaml, ctx)
-def _record_scouted_spine_omission_reject(ctx: AgentContext, digest: str | None) -> None:
- if digest is not None and digest == ctx.scouted_spine_previous_omission_digest:
- ctx.scouted_spine_repeated_identical_missing_steps = True
- LOG.info("copilot_scouted_spine_repeated_identical_omission", omission_digest=digest)
- else:
- ctx.scouted_spine_repeated_identical_missing_steps = False
- ctx.scouted_spine_previous_omission_digest = digest
-
-
def _scouted_spine_omission_digest(records: Sequence[Mapping[str, Any]]) -> str:
items = sorted(
(
@@ -10197,234 +7251,44 @@ def _scouted_spine_under_build_result(
]
if not partition_findings:
return None
- return _scouted_spine_partition_under_build_result(
- workflow_yaml, findings=partition_findings, block_label=block_label, site=site
- )
-
-
-def _scouted_spine_partition_under_build_result(
- workflow_yaml: str,
- *,
- findings: list[ObligationFinding],
- block_label: str,
- site: str,
-) -> _SynthesizedCodeImpositionResult:
- first = findings[0]
- reason_code = obligation_finding_reason_code(first)
- summary = render_obligation_findings(findings)
- LOG.info(
- "copilot_scouted_spine_under_build",
- block_label=block_label,
- site=site,
- reason_code=reason_code,
- finding_count=len(findings),
- finding_summary=summary,
- )
- violation = (
- f"Unable to impose synthesized code block: `{block_label}` under-builds the scouted spine "
- f"({reason_code}): {summary}. Resubmit the code block reusing the synthesized rung source verbatim so "
- "every captured interaction is replayed or carries a forgiven-drop reason."
- )
- return _SynthesizedCodeImpositionResult(
- workflow_yaml=workflow_yaml,
- violations=[violation],
- repair_context=CodeAuthoringRepairContext(
- block_label=block_label,
- reason_code=reason_code,
- selector=obligation_finding_selector(first),
- ),
- )
-
-
-def _pre_persist_scouted_spine_result(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult | None:
- """Last author-time gate before a durable persist: whatever path produced the final yaml, a draft
- whose browser calls leave the scouted spine partition open is rejected instead of persisted."""
- if not ctx.impose_synthesized_code_block:
- return None
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- scout_trajectory = ctx.scout_trajectory
- if not scout_trajectory:
- return None
- if not str(scout_trajectory[0].get("source_url") or "").strip():
- return None
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return None
- synthesized = ctx.imposition_synthesized_block
- if synthesized is None:
- # Deliberate fallback: when no imposition pass synthesized this attempt, grade a fresh strict
- # synthesis of the raw trajectory, without imposition's grounded-outcome reconciliation.
- synthesized = synthesize_code_block(
- scout_trajectory,
- strict_selectors=True,
- reached_download_target=ctx.reached_download_target,
- )
- if synthesized is None:
- return None
- code_blocks = _workflow_code_blocks(parsed)
- # A submission with zero code blocks still holds the open spine obligation: empty draft calls
- # leave every required rung uncovered rather than slipping the seam.
- return _scouted_spine_under_build_result(
- workflow_yaml,
- ctx=ctx,
- synthesized=synthesized,
- draft_codes=[str(block.get("code") or "") for block in code_blocks],
- block_label=", ".join(_code_block_label(block) for block in code_blocks) or _SYNTHESIZED_BLOCK_LABEL,
- site="pre_persist",
- draft_repairable_only=True,
- )
-
-
-_FREEHAND_UNRESOLVABLE_SELECTOR_REASON_CODE = "freehand_unresolvable_selector"
-_FREEHAND_UNGUARDED_CREDENTIAL_REASON_CODE = "freehand_unguarded_credential_fill"
-
-
-def _scouted_selector_forms(scout_trajectory: list[ScoutedInteraction]) -> set[str]:
- return {
- normalized_scout_selector(selector)
- for interaction in scout_trajectory
- if (selector := str(interaction.get("selector") or "").strip())
- }
-
-
-def _receiver_is_self_validating_get_by_role(receiver: str) -> bool:
- try:
- node = ast.parse(receiver, mode="eval").body
- except SyntaxError:
- return False
- if not (
- isinstance(node, ast.Call)
- and isinstance(node.func, ast.Attribute)
- and node.func.attr == "get_by_role"
- and isinstance(node.func.value, ast.Name)
- and node.func.value.id == "page"
- ):
- return False
- if len(node.args) != 1:
- return False
- role = node.args[0]
- if not (isinstance(role, ast.Constant) and isinstance(role.value, str) and role.value):
- return False
- if any(keyword.arg is None or not isinstance(keyword.value, ast.Constant) for keyword in node.keywords):
- return False
- return any(
- keyword.arg == "name"
- and isinstance(keyword.value, ast.Constant)
- and isinstance(keyword.value.value, str)
- and keyword.value.value
- for keyword in node.keywords
- )
-
-
-def _fragment_scout_freehand_validation(
- *,
- code_blocks: list[dict[str, Any]],
- exempt_block: dict[str, Any] | None,
- prior_yaml: str | None,
- scout_trajectory: list[ScoutedInteraction],
- synthesized_code: str,
- synthesized_diagnostics: SynthesisDiagnostics | None,
-) -> _BrowserSurfaceValidation:
- """A fragment scout is no authoritative full-spine reference, but every durable mutation must still
- resolve to a scout-captured form (captured selector or synthesized receiver) or be rejected; only
- selectorless page navigation and self-validating literal role+name receivers are admitted unscouted."""
- scouted_forms = _scouted_selector_forms(scout_trajectory)
- scouted_mutations, _, _ = _browser_surface_for_code(synthesized_code)
- admitted_receivers = {
- (mutation.method, normalized_locator_expr(mutation.receiver)) for mutation in scouted_mutations
- }
- violations: list[str] = []
- provenance: list[_BrowserSurfaceRejectionProvenance] = []
- for block in code_blocks:
- if block is exempt_block:
- continue
- if prior_yaml is not None and not _submitted_code_block_changed(block, prior_yaml):
- continue
- label = _code_block_label(block)
- block_mutations, _, block_ambiguous = _browser_surface_for_code(str(block.get("code") or ""))
- freehand_mutations: list[_BrowserMutationSignature] = []
- for mutation in block_mutations:
- if mutation.receiver == "page":
- continue
- if (mutation.method, normalized_locator_expr(mutation.receiver)) in admitted_receivers:
- continue
- if _receiver_is_self_validating_get_by_role(mutation.receiver):
- continue
- selector = _bare_locator_call_selector(mutation.receiver)
- if selector is not None and normalized_scout_selector(selector) in scouted_forms:
- continue
- freehand_mutations.append(mutation)
- if freehand_mutations:
- action_text = ", ".join(f"{mutation.receiver}.{mutation.method}" for mutation in sorted(freehand_mutations))
- block_provenance = [
- _classify_unscouted_mutation(
- mutation,
- scouted_mutations=scouted_mutations,
- diagnostics=synthesized_diagnostics,
- site="fragment_scout",
- block_label=label,
- )
- for mutation in sorted(freehand_mutations)
- ]
- provenance.extend(block_provenance)
- violations.append(
- f"Unable to impose synthesized code block: `{label}` contains unscouted browser action(s): "
- f"{action_text}.{_provenance_suffix_text(block_provenance)}"
- )
- if block_ambiguous:
- ambiguous_provenance = [
- _ambiguous_browser_action_provenance(action, site="fragment_scout", block_label=label)
- for action in block_ambiguous
- ]
- provenance.extend(ambiguous_provenance)
- violations.append(
- f"Unable to impose synthesized code block: `{label}` contains ambiguous browser action(s): "
- + ", ".join(block_ambiguous)
- + "."
- + _provenance_suffix_text(ambiguous_provenance)
- )
- _log_browser_surface_rejection_provenance(provenance)
- return _BrowserSurfaceValidation(violations, provenance)
-
-
-def _workflow_credential_parameter_keys(parsed: Mapping[str, Any]) -> set[str]:
- workflow_definition = parsed.get("workflow_definition")
- parameters = workflow_definition.get("parameters") if isinstance(workflow_definition, dict) else None
- if not isinstance(parameters, list):
- return set()
- keys: set[str] = set()
- for parameter in parameters:
- if not isinstance(parameter, Mapping) or not _is_credential_parameter(parameter):
- continue
- key = str(parameter.get("key") or "").strip()
- if key:
- keys.add(key)
- return keys
+ return _scouted_spine_partition_under_build_result(
+ workflow_yaml, findings=partition_findings, block_label=block_label, site=site
+ )
-def _freehand_surface_reject(
- workflow_yaml: str, code_blocks: list[dict[str, Any]], validation: _BrowserSurfaceValidation
+def _scouted_spine_partition_under_build_result(
+ workflow_yaml: str,
+ *,
+ findings: list[ObligationFinding],
+ block_label: str,
+ site: str,
) -> _SynthesizedCodeImpositionResult:
- first = validation.provenance[0] if validation.provenance else None
- repair_context = CodeAuthoringRepairContext(
- block_label=first.block_label if first is not None else _code_block_label(code_blocks[0]),
- reason_code=_FREEHAND_UNRESOLVABLE_SELECTOR_REASON_CODE,
- selector=first.nearest_selector if first is not None else None,
+ first = findings[0]
+ reason_code = obligation_finding_reason_code(first)
+ summary = render_obligation_findings(findings)
+ LOG.info(
+ "copilot_scouted_spine_under_build",
+ block_label=block_label,
+ site=site,
+ reason_code=reason_code,
+ finding_count=len(findings),
+ finding_summary=summary,
)
+ violation = f"`{block_label}` does not replay every scouted interaction ({reason_code}): {summary}."
return _SynthesizedCodeImpositionResult(
workflow_yaml=workflow_yaml,
- violations=validation.violations,
- repair_context=repair_context,
- never_captured_candidate=_never_captured_obligation_candidate(validation.provenance),
+ violations=[violation],
+ repair_context=CodeAuthoringRepairContext(
+ block_label=block_label,
+ reason_code=reason_code,
+ selector=obligation_finding_selector(first),
+ ),
)
-def _persist_seam_freehand_surface_result(
- workflow_yaml: str, ctx: AgentContext
-) -> _SynthesizedCodeImpositionResult | None:
- # Imposition validates only the spine it rewrites; this leg closes the persist lanes where a changed
- # freehand browser block never reached that surface gate, applying the same admissibility to siblings.
+def _pre_persist_scouted_spine_result(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult | None:
+ """Last author-time gate before a durable persist: whatever path produced the final yaml, a draft
+ whose browser calls leave the scouted spine partition open is rejected instead of persisted."""
if not ctx.impose_synthesized_code_block:
return None
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
@@ -10437,87 +7301,32 @@ def _persist_seam_freehand_surface_result(
parsed = parse_workflow_yaml(workflow_yaml)
if not isinstance(parsed, dict):
return None
- code_blocks = _workflow_code_blocks(parsed)
- if not code_blocks:
- return None
- _, prior_yaml = _prior_yaml_source(ctx)
- carrier_label = ctx.spine_imposition_carrier_label
- exempt_block = (
- next((block for block in code_blocks if _code_block_label(block) == carrier_label), None)
- if carrier_label
- else None
- )
- credential_parameter_keys = _workflow_credential_parameter_keys(parsed)
- for block in code_blocks:
- if block is exempt_block:
- continue
- if prior_yaml is not None and not _submitted_code_block_changed(block, prior_yaml):
- continue
- if not _block_has_unguarded_credential_fill(str(block.get("code") or ""), credential_parameter_keys):
- continue
- label = _code_block_label(block)
- LOG.info("copilot_freehand_unguarded_credential_fill", block_label=label)
- violation = (
- f"Unable to persist code block: `{label}` fills a credential field without a presence guard, so it "
- "re-fills and times out when the workflow replays on an already-authenticated page. Reuse the "
- "synthesized entry rung, which fills only when the login form is still present."
- )
- return _SynthesizedCodeImpositionResult(
- workflow_yaml=workflow_yaml,
- violations=[violation],
- repair_context=CodeAuthoringRepairContext(
- block_label=label, reason_code=_FREEHAND_UNGUARDED_CREDENTIAL_REASON_CODE
- ),
+ synthesized = ctx.imposition_synthesized_block
+ if synthesized is None:
+ # Deliberate fallback: when no imposition pass synthesized this attempt, grade a fresh strict
+ # synthesis of the raw trajectory, without imposition's grounded-outcome reconciliation.
+ synthesized = synthesize_code_block(
+ scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
)
- synthesized = synthesize_code_block(
- scout_trajectory, strict_selectors=True, reached_download_target=ctx.reached_download_target
- )
- synthesized_code = "" if synthesized is None else (synthesized.interaction_code or synthesized.code)
- synthesized_diagnostics = None if synthesized is None else synthesized.diagnostics
- sentinel_selected: dict[str, Any] = {}
- selected_code_block = exempt_block if exempt_block is not None else sentinel_selected
- # get_by_label is never a synthesized selector, so an unresolvable one is rejected regardless of whether
- # the scout reached its goal; a fragment scout still fails this class closed to the scout-the-step route.
- factory_validation = _whole_trajectory_browser_surface_violations(
- code_blocks=code_blocks,
- selected_code_block=selected_code_block,
- submitted_selected_code="",
- synthesized_code=synthesized_code,
- prior_yaml=prior_yaml,
- synthesized_diagnostics=synthesized_diagnostics,
- admit_on_receiver=True,
- restrict_to_durable_factory=True,
- )
- if factory_validation.violations:
- return _freehand_surface_reject(workflow_yaml, code_blocks, factory_validation)
- # The full synthesized spine is an authoritative admissibility reference only when the scout captured a
- # durable entry through a commit; a fragment scout would false-reject legitimate multi-block authoring.
- if synthesized is None or not synthesized_trajectory_reaches_goal(ctx):
- # scouted_interactions joins the admitted set because the trajectory is cap-evicted; both
- # collections carry only scout-captured selectors.
- fragment_validation = _fragment_scout_freehand_validation(
- code_blocks=code_blocks,
- exempt_block=exempt_block,
- prior_yaml=prior_yaml,
- scout_trajectory=[*scout_trajectory, *ctx.scouted_interactions],
- synthesized_code=synthesized_code,
- synthesized_diagnostics=synthesized_diagnostics,
- )
- if fragment_validation.violations:
- return _freehand_surface_reject(workflow_yaml, code_blocks, fragment_validation)
+ if synthesized is None:
return None
- validation = _whole_trajectory_browser_surface_violations(
- code_blocks=code_blocks,
- selected_code_block=selected_code_block,
- submitted_selected_code="",
- synthesized_code=synthesized_code,
- prior_yaml=prior_yaml,
- synthesized_diagnostics=synthesized_diagnostics,
- admit_on_receiver=True,
+ code_blocks = _workflow_code_blocks(parsed)
+ # A submission with zero code blocks still holds the open spine obligation: empty draft calls
+ # leave every required rung uncovered rather than slipping the seam.
+ return _scouted_spine_under_build_result(
+ workflow_yaml,
+ ctx=ctx,
+ synthesized=synthesized,
+ draft_codes=[str(block.get("code") or "") for block in code_blocks],
+ block_label=", ".join(_code_block_label(block) for block in code_blocks) or _SYNTHESIZED_BLOCK_LABEL,
+ site="pre_persist",
+ draft_repairable_only=True,
)
- if not validation.violations:
- return None
- return _freehand_surface_reject(workflow_yaml, code_blocks, validation)
+
+
+_FREEHAND_UNRESOLVABLE_SELECTOR_REASON_CODE = "freehand_unresolvable_selector"
def _workflow_yaml_browser_call_pairs(workflow_yaml: str) -> list[tuple[str, str]]:
@@ -10757,11 +7566,6 @@ def _submitted_direct_fill_type_usage(code: str, parameter_key: str) -> _DirectF
return _DirectFillTypeUsage(matched, mismatched)
-def _submitted_uses_parameter_in_direct_fill_type(code: str, parameter_key: str) -> bool:
- usage = _submitted_direct_fill_type_usage(code, parameter_key)
- return usage.matched and not usage.mismatched
-
-
def _safe_singleton_literal_for_parameter(
code: str, parameter_key: str, typed_length: int | None
) -> tuple[str | None, str | None]:
@@ -10960,11 +7764,6 @@ def _strip_redundant_sandbox_imports_in_yaml(workflow_yaml: str) -> tuple[str, l
return yaml.safe_dump(parsed, sort_keys=False), stripped_modules
-def _code_block_has_load_bearing_mutation(code: str) -> bool:
- mutations, _reads, _ambiguous = _browser_surface_for_code(code)
- return any(mutation.method in _LOCATOR_MUTATION_METHODS for mutation in mutations)
-
-
def _iter_workflow_blocks(blocks: object) -> Iterator[dict[str, Any]]:
if not isinstance(blocks, list):
return
@@ -11011,102 +7810,6 @@ def _dangling_next_block_label_violation(parsed: Mapping[str, Any]) -> str | Non
)
-def _scaffolding_only_body_violation(ctx: AgentContext, parsed: Mapping[str, Any]) -> str | None:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- scout_trajectory = ctx.scout_trajectory
- if not isinstance(scout_trajectory, list) or not scout_trajectory:
- return None
- synthesized = synthesize_code_block(
- scout_trajectory, reached_download_target=getattr(ctx, "reached_download_target", None)
- )
- if synthesized is None:
- return None
- required = [record for record in synthesized.diagnostics.emitted_interactions if not str(record.get("lane") or "")]
- if not required:
- return None
- code_blocks = _workflow_code_blocks(parsed) if isinstance(parsed, dict) else []
- if not code_blocks:
- return None
- if any(_code_block_has_load_bearing_mutation(str(block.get("code") or "")) for block in code_blocks):
- return None
- return (
- "Unable to persist workflow: the persisted code block(s) commit no load-bearing browser interaction "
- f"({SCOUTED_SPINE_UNDER_BUILD_REASON_CODE}), though the scout demonstrated {len(required)} rung(s). "
- "Reuse the synthesized code block verbatim so the captured interactions are replayed."
- )
-
-
-def _normalize_prompt_text(text: str) -> str:
- return " ".join(text.split()).casefold()
-
-
-def _stub_with_raw_prompt_violation(ctx: AgentContext, parsed: Mapping[str, Any]) -> str | None:
- packet = getattr(ctx, "turn_context_packet", None)
- if packet is None:
- return None
- anchor = _normalize_prompt_text(str(packet.transcript_context.earliest_user_turn or ""))
- if not anchor:
- return None
- if _scaffolding_only_body_violation(ctx, parsed) is None:
- return None
- for block in _iter_workflow_blocks(parsed.get("workflow_definition", {}).get("blocks")):
- for field_name in ("prompt", "description", "title"):
- if _normalize_prompt_text(str(block.get(field_name) or "")) == anchor:
- return (
- "Unable to persist workflow: a block reproduces the raw user request as its "
- f"{field_name} while committing no load-bearing browser interaction "
- f"({SCOUTED_SPINE_UNDER_BUILD_REASON_CODE})."
- )
- return None
-
-
-def _code_references_parameter(code: str, key: str) -> bool:
- sources = _code_runtime_parameter_sources(code, {key})
- return sources is not None and key in sources
-
-
-def _orphan_minted_parameter_violation(parsed: Mapping[str, Any], minted_keys: list[str]) -> str | None:
- if not minted_keys or not isinstance(parsed, dict):
- return None
- codes = [str(block.get("code") or "") for block in _workflow_code_blocks(parsed)]
- orphans = sorted(
- {key for key in minted_keys if key and not any(_code_references_parameter(code, key) for code in codes)}
- )
- if not orphans:
- return None
- return (
- "Unable to persist workflow: minted workflow parameter(s) are not referenced by any generated code: "
- + ", ".join(f"`{key}`" for key in orphans)
- + "."
- )
-
-
-def _final_yaml_structural_violations(
- ctx: AgentContext,
- workflow_yaml: str,
- *,
- minted_parameter_keys: list[str],
- promoted_parameter_keys: list[str],
- carried_by_imposition: bool,
-) -> list[str]:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return []
- parsed = parse_workflow_yaml(workflow_yaml)
- if not isinstance(parsed, dict):
- return []
- candidates: list[str | None] = [
- _dangling_next_block_label_violation(parsed),
- _orphan_minted_parameter_violation(parsed, [*minted_parameter_keys, *promoted_parameter_keys]),
- ]
- # The spine-coverage seam runs only when imposition made no substitution; a successful imposition
- # skips it, so the body-shape predicates re-validate exactly that bypassed window.
- if carried_by_imposition:
- candidates.insert(0, _scaffolding_only_body_violation(ctx, parsed))
- candidates.insert(1, _stub_with_raw_prompt_violation(ctx, parsed))
- return [violation for violation in candidates if violation is not None]
-
-
def _apply_scouted_typed_default_promotions(workflow_yaml: str, ctx: AgentContext) -> tuple[str, list[str], list[str]]:
if not getattr(ctx, "impose_synthesized_code_block", False):
return workflow_yaml, [], []
@@ -11359,31 +8062,6 @@ def _apply_parameter_reconciliation_to_code(code: str, reconciliation: _Synthesi
_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE = "synthesized_parameter_binding_ambiguous"
-def _record_synthesized_parameter_binding_ablation_event(
- ctx: AgentContext,
- workflow_yaml: str,
- repair_context: CodeAuthoringRepairContext,
-) -> bool:
- fingerprint = authored_structure_signature_from_workflow(workflow_yaml, ctx.raw_code_artifact_metadata)
- if fingerprint is None:
- return False
- payload: AuthorTimeGateAblationPayload = {
- "block_label": repair_context.block_label,
- "unresolved_names": list(repair_context.unresolved_names),
- "parameter_keys": list(repair_context.parameter_keys),
- "available_parameter_keys": list(repair_context.available_parameter_keys),
- "binding_candidates": list(repair_context.binding_candidates),
- }
- return record_author_time_gate_ablation_event(
- ctx,
- gate_id=SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- reason_code=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
- fingerprint=fingerprint,
- blocked_tool="update_workflow",
- payload=payload,
- )
-
-
def _synthesized_parameter_binding_repair_context(
*,
parsed: Mapping[str, Any],
@@ -11844,7 +8522,6 @@ def _maybe_impose_synthesized_code_block(
if not isinstance(parsed, dict):
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
requirement = ctx.recorded_outcome_grounding_requirement
- constraint = ctx.recorded_outcome_binding_constraint
code_blocks = _workflow_code_blocks(parsed)
prior_source, prior_yaml = _prior_yaml_source(ctx)
code_block = _select_synthesized_imposition_code_block(
@@ -11939,20 +8616,6 @@ def _maybe_impose_synthesized_code_block(
violations=["Unable to impose synthesized code block: current-page parameter binding is ambiguous."],
repair_context=binding_repair_context,
)
- grounding_repair_active = (
- binding_snapshot is None
- and file_match_transform is None
- and bool(declared_keys)
- and isinstance(requirement, RecordedOutcomeGroundingRequirement)
- and requirement.satisfied
- and isinstance(constraint, RecordedOutcomeBindingConstraint)
- and constraint.repeated_structural_key == requirement.structural_key
- )
- if grounding_repair_active and _stash_unresolved_recorded_outcome_grounding_halt(ctx, declared_keys):
- return _SynthesizedCodeImpositionResult(
- workflow_yaml=workflow_yaml,
- violations=["Unable to impose synthesized code block: current-page parameter binding is unresolved."],
- )
synthesis_values = _runtime_declared_string_parameter_values(parsed, runtime_parameters)
synthesis_trajectory: list[Mapping[str, Any]] = []
for interaction in scout_trajectory:
@@ -12186,7 +8849,6 @@ def _maybe_impose_synthesized_code_block(
scout_trajectory=scout_trajectory,
parameter_binding_snapshot=binding_snapshot,
)
- parameter_binding_is_only_violation = not violations and bool(parameter_reconciliation.violations)
violations.extend(parameter_reconciliation.violations)
if repair_context is None:
repair_context = parameter_reconciliation.repair_context
@@ -12203,14 +8865,6 @@ def _maybe_impose_synthesized_code_block(
violations=violations,
repair_context=repair_context,
metadata_repair_contract=_co_computed_metadata_repair_contract(ctx, workflow_yaml, raw_metadata),
- never_captured_candidate=_never_captured_obligation_candidate(rejection_provenance),
- ablation_gate_id=(
- SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID
- if parameter_binding_is_only_violation
- and repair_context is not None
- and repair_context.reason_code == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
- else None
- ),
)
metadata_declares_goal_values = bool(raw_metadata) and _raw_metadata_declares_goal_values_for_block(
@@ -13021,6 +9675,38 @@ def _code_uses_expect_download(code: str) -> bool:
return False
+_DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET = frozenset({"downloaded_file_path", "download_url"})
+
+
+def _download_descriptor_leak_finding(label: str, code: str) -> str | None:
+ """A run cannot reveal this: it succeeds, and a local filesystem path or raw download URL
+ lands in workflow output. The execution layer owns those keys."""
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return None
+ leak_key_locals: set[str] = set()
+ leaked = False
+ for node in _iter_top_level_scope(tree.body):
+ if isinstance(node, ast.Return) and node.value is not None:
+ if _dict_keys(node.value) & _DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET or (
+ isinstance(node.value, ast.Name) and node.value.id in leak_key_locals
+ ):
+ leaked = True
+ break
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ if _dict_keys(node.value) & _DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET:
+ leak_key_locals.add(node.targets[0].id)
+ if not leaked:
+ return None
+ return (
+ f"Code block `{label}` returns the raw descriptor keys "
+ f"({', '.join(sorted(_DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET))}), which put a local filesystem path or download "
+ "URL into workflow output. The execution layer owns those; return a small descriptor such as "
+ '`{"saved_as": dl_info.value.suggested_filename}` instead.'
+ )
+
+
def _dict_keys(node: ast.expr) -> set[str]:
if isinstance(node, ast.Await):
return _dict_keys(node.value)
@@ -13032,7 +9718,6 @@ def _dict_keys(node: ast.expr) -> set[str]:
_REGISTERED_DOWNLOAD_OUTPUT_KEY_SET = frozenset(REGISTERED_DOWNLOAD_OUTPUT_KEYS)
-_DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET = frozenset({"downloaded_file_path", "download_url"})
def _code_returns_registration_keys(code: str) -> bool:
@@ -13053,26 +9738,6 @@ def _code_returns_registration_keys(code: str) -> bool:
return False
-def _code_returns_download_descriptor_leak_keys(code: str) -> bool:
- """Detect simple top-level descriptor returns; execution-registered artifacts remain
- the authoritative download proof."""
- try:
- tree = ast.parse(textwrap.dedent(code).strip() or "pass")
- except SyntaxError:
- return False
- leak_key_locals: set[str] = set()
- for node in _iter_top_level_scope(tree.body):
- if isinstance(node, ast.Return) and node.value is not None:
- if _dict_keys(node.value) & _DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET:
- return True
- if isinstance(node.value, ast.Name) and node.value.id in leak_key_locals:
- return True
- if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
- if _dict_keys(node.value) & _DOWNLOAD_DESCRIPTOR_LEAK_KEY_SET:
- leak_key_locals.add(node.targets[0].id)
- return False
-
-
def _artifact_declares_registration_keys(artifact: Mapping[str, Any]) -> bool:
for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
for row in _artifact_rows(artifact.get(field_name)):
@@ -13096,36 +9761,6 @@ def _is_download_intent(artifact: Mapping[str, Any], code: str) -> bool:
)
-def _download_return_shape_error(label: str, artifact: Mapping[str, Any], code: str) -> str | None:
- """Fail closed on a download-intent code block that cannot register a real download:
- one lacking the `page.expect_download` idiom (no browser download ever fires), or one
- whose return writes the execution-layer-owned registration keys (self-certification)."""
- if not _is_download_intent(artifact, code):
- return None
- if not _code_uses_expect_download(code):
- return (
- f"Code block `{label}` is a download block but does not fire the browser download with the "
- "`page.expect_download` idiom (async with page.expect_download() as dl_info: await "
- "page.click(<unique selector>)). A static fetch or a plain click registers no file. Author the "
- "expect_download idiom against a unique target so the runtime registers the file into the workflow "
- "output (downloaded_files); never place file bytes or URLs in the chat reply."
- )
- if _code_returns_registration_keys(code):
- return (
- f"Code block `{label}` `return`s the download registration keys "
- f"({', '.join(REGISTERED_DOWNLOAD_OUTPUT_KEYS)}) itself. The execution layer injects those keys when a "
- "browser download fires; writing them self-certifies a download that may never have happened. Return a "
- 'small keyed descriptor instead (for example `return {"saved_as": dl_info.value.suggested_filename}`) and '
- "let the expect_download idiom register the file."
- )
- if _code_returns_download_descriptor_leak_keys(code):
- return (
- f"Code block `{label}` returns raw download path/URL descriptor keys. Return only non-sensitive summary "
- "fields such as the suggested filename; the execution layer injects artifact URLs and IDs."
- )
- return None
-
-
def _code_artifact_metadata_shape_errors(
label: str,
artifact: Mapping[str, Any],
@@ -13424,34 +10059,59 @@ def _credential_scout_reject_error_text(ctx: AgentContext, credential_scout_erro
return error_text
-def _reject_schema_incompatibility(
- ctx: AgentContext,
- incompatibility: SchemaIncompatibility,
- reject: Callable[..., dict[str, Any]],
-) -> dict[str, Any]:
- """Emit the typed, non-repairable schema-incompatibility outcome: stash the terminal
- blocker + turn halt so enforcement renders a product-language reply (instead of
- falling through to repair churn), record it on the context, and reject without
- persisting the incompatible draft."""
- signal = build_schema_incompatibility_blocker_signal(incompatibility)
- _record_author_time_reject_outcome(
- ctx,
- reason_code="schema_incompatibility",
- summary=signal.user_facing_reason,
- structural_payload=incompatibility.to_summary_dict(),
- block_labels=[incompatibility.block_label],
- )
- stash_blocker_signal(ctx, signal)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
- ctx.latest_schema_incompatibility = incompatibility
- return reject(
- error=render_schema_incompatibility_agent_steer(incompatibility),
- user_facing_summary=signal.user_facing_reason,
- data={
- "failure_type": SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,
- "schema_incompatibility": incompatibility.to_summary_dict(),
- },
- )
+def carry_author_time_findings(update_result: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]:
+ """The combined update-and-run tool returns the run's result, not the update's, so findings
+ labelled on the persisted draft reach the model only if they are carried across."""
+ update_data = update_result.get("data")
+ findings = update_data.get("findings") if isinstance(update_data, dict) else None
+ if not findings:
+ return result
+ data = result.get("data")
+ if not isinstance(data, dict):
+ data = {}
+ result["data"] = data
+ data.setdefault("findings", findings)
+ return result
+
+
+def _author_time_findings(
+ *,
+ schema_incompatibility: SchemaIncompatibility | None,
+ metadata_violations: Sequence[str],
+ imposition_violations: Sequence[str],
+) -> list[dict[str, Any]]:
+ """Non-blocking labels on a draft that persisted anyway. Each entry needs a reason a
+ test-run would not surface it; anything a run reveals belongs in the run, not here."""
+ findings: list[dict[str, Any]] = []
+ # An edited schema field that binds to nothing does not fail the run: the run succeeds and
+ # silently omits the field, so only the known output contract can say it can never bind.
+ if schema_incompatibility is not None:
+ findings.append(
+ {
+ "reason_code": SCHEMA_INCOMPATIBILITY_REASON_CODE,
+ "summary": render_schema_incompatibility_user_reason(schema_incompatibility),
+ "schema_incompatibility": schema_incompatibility.to_summary_dict(),
+ }
+ )
+ # The normalizer rewrites what it can; the residue names metadata the authored artifact
+ # never declared, which a run cannot report because it never had the field to produce.
+ if metadata_violations:
+ findings.append(
+ {
+ "reason_code": "code_artifact_metadata_incomplete",
+ "summary": "\n".join(str(violation) for violation in metadata_violations),
+ }
+ )
+ # An imposition that could not apply leaves the model's own YAML persisted; naming why
+ # is not observable from the run, which only ever sees the un-imposed draft.
+ if imposition_violations:
+ findings.append(
+ {
+ "reason_code": "synthesized_code_block_not_imposed",
+ "summary": "\n".join(str(violation) for violation in imposition_violations),
+ }
+ )
+ return findings
def _declared_string_parameter_values(workflow_yaml: str) -> dict[str, str]:
@@ -13509,11 +10169,9 @@ async def _update_workflow(
output_contract_imposed_signature = ""
- def reject(
+ def _blocked(
+ block: AuthorTimeBlock,
*,
- error: str,
- user_facing_summary: str | None = None,
- data: dict[str, Any] | None = None,
repair_context: CodeAuthoringRepairContext | None = None,
record_repair_context_outcome: bool = True,
) -> dict[str, Any]:
@@ -13526,291 +10184,56 @@ async def _update_workflow(
_set_code_authoring_repair_context(ctx, repair_context)
else:
ctx.last_code_authoring_repair_context = repair_context
+ _record_code_authoring_guardrail_reject(
+ ctx,
+ defer_churn_stop=block.block_id == CREDENTIAL_SCOUT_BLOCK_ID,
+ )
+ result: dict[str, Any] = {"ok": False, "error": block.error, "block_id": block.block_id}
+ if block.user_facing_summary is not None:
+ result["user_facing_summary"] = block.user_facing_summary
+ if block.data:
+ result["data"] = block.data
+ return result
+
+ def _tool_error(error: str, *, user_facing_summary: str | None = None) -> dict[str, Any]:
+ # The submission cannot become a Workflow, so there is no authored artifact to refuse:
+ # report it honestly without a block identity, a turn halt, or a churn increment.
result: dict[str, Any] = {"ok": False, "error": error}
if user_facing_summary is not None:
result["user_facing_summary"] = user_facing_summary
- if data is not None:
- result["data"] = data
return result
authority_error = _authority_tool_error(ctx, "update_workflow")
if authority_error is not None:
- return reject(error=authority_error)
+ return _tool_error(authority_error)
workflow_yaml = params["workflow_yaml"]
- never_captured_replay_payload = NeverCapturedReplayPayload(
- params=copy.deepcopy(params),
- allow_missing_credentials=allow_missing_credentials,
- allow_static_output_uncertainty=allow_static_output_uncertainty,
- formation_prepared=formation_prepared,
- )
raw_conflict_marker_error = _raw_workflow_yaml_conflict_marker_error(workflow_yaml)
if raw_conflict_marker_error is not None:
- return reject(
- error=raw_conflict_marker_error,
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
- # Tool wrappers run authority/loop guards before calling here. The composition
- # gate below consumes these refs, so they must be visible before validation.
+ return _tool_error(raw_conflict_marker_error, user_facing_summary=_compiled_authoring_user_summary())
ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(params.get("code_artifact_metadata"))
- # Imposition reconciles synthesized aliases/parameters before the persisted YAML contract is checked.
- _enrich_scout_trajectory_input_correspondences(workflow_yaml, ctx)
- runtime_parameters = params.get("parameters")
- capture_obligation = ctx.never_captured_obligation
- if (
- capture_obligation is not None
- and capture_obligation.state == "captured"
- and capture_obligation.draft_fingerprint != hashlib.sha256(workflow_yaml.encode()).hexdigest()
- ):
- ctx.never_captured_obligation = replace(capture_obligation, state="consumed")
- ctx.synthesized_block_reopened_for_capture_obligation = False
- LOG.info(
- "copilot_never_captured_obligation_expired_for_different_draft",
- identity_digest=capture_obligation.identity_digest,
- )
- imposition = _maybe_impose_synthesized_code_block(
- workflow_yaml,
- ctx,
- runtime_parameters if isinstance(runtime_parameters, Mapping) else None,
- )
- capture_obligation = ctx.never_captured_obligation
- if capture_obligation is not None and capture_obligation.state == "captured":
- ctx.never_captured_obligation = replace(capture_obligation, state="consumed")
- ctx.synthesized_block_reopened_for_capture_obligation = False
- # Consume the one-shot credential-scout reopen before the gate below so a fresh reject can re-arm it.
- ctx.synthesized_block_reopened_for_credential_scout = False
- if imposition.violations:
- if (
- imposition.ablation_gate_id == SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID
- and imposition.repair_context is not None
- and _record_synthesized_parameter_binding_ablation_event(
- ctx,
- workflow_yaml,
- imposition.repair_context,
- )
- ):
- imposition = _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
- else:
- _arm_never_captured_obligation(
- ctx,
- workflow_yaml,
- imposition.never_captured_candidate,
- never_captured_replay_payload,
- )
- if (
- imposition.repair_context is not None
- and imposition.repair_context.reason_code == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
- ):
- required_paths = _output_contract_required_paths_source(ctx).union
- _record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
- )
- if (
- imposition.repair_context is not None
- and imposition.repair_context.reason_code in _SCOUTED_SPINE_REASON_CODES
- ):
- _set_code_authoring_repair_context(ctx, imposition.repair_context)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error="\n".join(imposition.violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(imposition.repair_context),
- repair_context=imposition.repair_context,
- record_repair_context_outcome=False,
- )
- return reject(
- error="\n".join(imposition.violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(
- imposition.repair_context,
- metadata_repair_contract=imposition.metadata_repair_contract,
- ),
- repair_context=imposition.repair_context,
- )
- workflow_yaml = imposition.workflow_yaml
- if imposition.substitutions is None:
- freehand_surface = _persist_seam_freehand_surface_result(workflow_yaml, ctx)
- if freehand_surface is not None:
- _arm_never_captured_obligation(
- ctx,
- workflow_yaml,
- freehand_surface.never_captured_candidate,
- never_captured_replay_payload,
- )
- _set_code_authoring_repair_context(ctx, freehand_surface.repair_context)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error="\n".join(freehand_surface.violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(freehand_surface.repair_context),
- repair_context=freehand_surface.repair_context,
- record_repair_context_outcome=False,
- )
+ _enrich_scout_trajectory_input_correspondences(workflow_yaml, ctx)
+ runtime_parameters = params.get("parameters")
+ imposition = _maybe_impose_synthesized_code_block(
+ workflow_yaml,
+ ctx,
+ runtime_parameters if isinstance(runtime_parameters, Mapping) else None,
+ )
+ ctx.synthesized_block_reopened_for_credential_scout = False
+ if not imposition.violations:
+ workflow_yaml = imposition.workflow_yaml
stripped_sandbox_imports: list[str] = []
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
workflow_yaml, stripped_sandbox_imports = _strip_redundant_sandbox_imports_in_yaml(workflow_yaml)
- workflow_yaml, typed_default_violations, promoted_parameter_keys = _apply_scouted_typed_default_promotions(
+ workflow_yaml, _typed_default_violations, _promoted_parameter_keys = _apply_scouted_typed_default_promotions(
workflow_yaml, ctx
)
- if typed_default_violations:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error="\n".join(typed_default_violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
workflow_yaml = _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
- final_structural_violations = _final_yaml_structural_violations(
- ctx,
- workflow_yaml,
- minted_parameter_keys=imposition.minted_parameter_keys,
- promoted_parameter_keys=promoted_parameter_keys,
- carried_by_imposition=imposition.substitutions is not None,
- )
- if final_structural_violations:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error="\n".join(final_structural_violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
- business_input_floor_reject = _synthesized_business_input_floor_reject(ctx, workflow_yaml, imposition)
- if business_input_floor_reject is not None:
- unreferenced_keys = list(business_input_floor_reject.unreferenced_parameter_keys)
- ctx.synthesized_business_required_parameter_keys.update(unreferenced_keys)
- _record_author_time_reject_outcome(
- ctx,
- reason_code="required_input_unbound",
- summary=_definition_plane_reject_error(business_input_floor_reject),
- structural_payload={
- "reason_code": SYNTHESIZED_BUSINESS_INPUT_FLOOR_REASON_CODE,
- "unreferenced_parameter_keys": unreferenced_keys,
- },
- block_labels=[
- _code_block_label(block) for block in _workflow_code_blocks(parse_workflow_yaml(workflow_yaml) or {})
- ],
- )
- _record_code_authoring_guardrail_reject(ctx)
- _reopen_scout_after_synthesized_business_input_floor(ctx)
- return reject(
- error=(
- _definition_plane_reject_error(business_input_floor_reject)
- + " Continue scouting the business-value fields before saving or running this synthesized block."
- ),
- user_facing_summary=_compiled_authoring_user_summary(),
- data={
- "reason_code": SYNTHESIZED_BUSINESS_INPUT_FLOOR_REASON_CODE,
- "unreferenced_parameter_keys": unreferenced_keys,
- },
- )
- definition_reject = _definition_plane_preflight_reject(ctx, workflow_yaml)
- if definition_reject is not None:
- definition_corrective = (
- None
- if copilot_author_time_gate_log_only_enabled(ctx, DEFINITION_CONTRACT_UNSATISFIED_GATE_ID)
- else _value_bearing_definition_reject_corrective(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=ctx.raw_code_artifact_metadata,
- )
- )
- definition_data = {
- "reason_code": "definition_contract_unsatisfied",
- "definition_criterion_ids": list(definition_reject.criterion_ids),
- "definition_reason_codes": list(definition_reject.reason_codes),
- "unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),
- **(definition_corrective or {}),
- }
- halted = _stash_unresolved_recorded_outcome_grounding_halt(
- ctx,
- definition_reject.unreferenced_parameter_keys,
- )
- if halted:
- return reject(
- error=_definition_plane_reject_error(definition_reject),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=definition_data,
- )
- if not _record_definition_plane_ablation_event(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=ctx.raw_code_artifact_metadata,
- ):
- _record_definition_plane_reject(
- ctx,
- workflow_yaml,
- definition_reject,
- code_artifact_metadata=ctx.raw_code_artifact_metadata,
- )
- return reject(
- error=_definition_plane_reject_error(definition_reject),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=definition_data,
- )
- if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- unrenderable = _code_block_render_reject(workflow_yaml, getattr(ctx, "workflow_yaml", None))
- if unrenderable is not None:
- unrenderable_label, render_diagnostic = unrenderable
- render_error = (
- f"Code block `{unrenderable_label}` contains a Jinja expression the workflow runtime "
- f"cannot render. {render_diagnostic.message}"
- )
- LOG.info(
- "copilot code block render preflight reject",
- reason_code="code_block_unrenderable",
- block_label=unrenderable_label,
- failing_expression=render_diagnostic.failing_expression,
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="code_block_unrenderable",
- summary=render_error,
- structural_payload={
- "reason_code": "code_block_unrenderable",
- "block_label": unrenderable_label,
- "failing_expression": render_diagnostic.failing_expression,
- },
- block_labels=[unrenderable_label],
- )
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error=render_error,
- user_facing_summary=_compiled_authoring_user_summary(),
- data={
- "reason_code": "code_block_unrenderable",
- "block_label": unrenderable_label,
- "failing_expression": render_diagnostic.failing_expression,
- },
- )
params["workflow_yaml"] = workflow_yaml
- metadata_scrubbed_by_imposition = False
- if (
- imposition.selected_extraction_metadata_disposition == "browser_spine_replaced_metadata_stale"
- and imposition.scrubbed_selected_metadata_label
- ):
- scrubbed_metadata = _downgrade_stale_selected_goal_value_paths(
- params.get("code_artifact_metadata"),
- imposition.scrubbed_selected_metadata_label,
- )
- params["code_artifact_metadata"] = scrubbed_metadata
- ctx.raw_code_artifact_metadata = scrubbed_metadata
- metadata_scrubbed_by_imposition = True
- parameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)
- if parameter_contract_error is not None:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error=parameter_contract_error,
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
workflow_yaml, imposed_metadata, envelope_imposed = (
(workflow_yaml, params.get("code_artifact_metadata"), False)
if formation_prepared and imposition.substitutions is None
@@ -13827,7 +10250,7 @@ async def _update_workflow(
ctx.raw_code_artifact_metadata = imposed_metadata
scaffolded_metadata, scaffold_applied = (
(params.get("code_artifact_metadata"), False)
- if formation_prepared and not metadata_scrubbed_by_imposition
+ if formation_prepared
else _scaffold_metadata_contract_for_update(
ctx,
workflow_yaml,
@@ -13838,11 +10261,6 @@ async def _update_workflow(
params["code_artifact_metadata"] = scaffolded_metadata
params["raw_code_artifact_metadata"] = scaffolded_metadata
ctx.raw_code_artifact_metadata = scaffolded_metadata
- (
- required_child_output_paths,
- output_path_coverage_source,
- output_path_coverage_reason_code,
- ) = _required_child_output_paths_for_authoring(ctx)
output_contract_evaluation = _evaluate_output_contract_for_code_block(
ctx,
workflow_yaml,
@@ -13850,265 +10268,10 @@ async def _update_workflow(
allow_static_return_advisory=allow_static_output_uncertainty,
enforce_value_bearing_liveness=True,
)
- if (
- output_contract_evaluation is not None
- and _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in output_contract_evaluation.shape_violations
- ):
- _revoke_output_contract_advisory_run(ctx, output_contract_evaluation.canonical_signature)
output_contract_static_advisory_allowed = (
output_contract_evaluation is not None and output_contract_evaluation.can_attempt_run
)
- output_contract_deficient = (
- output_contract_evaluation is not None
- and output_contract_evaluation.has_deficiencies
- and not output_contract_static_advisory_allowed
- )
- output_contract_ceiling_resolution: _OutputContractCeilingResolution | None = None
- if output_contract_deficient:
- assert output_contract_evaluation is not None
- authored_fingerprint = _output_contract_structural_fingerprint(
- workflow_yaml, output_contract_evaluation.canonical_signature
- )
- # Recorded before the pass path resolves so the gate's would-block population still
- # counts the submissions credit-or-impose goes on to handle.
- output_contract_ablated = _record_output_contract_ablation_event(
- ctx,
- output_contract_evaluation,
- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
- blocked_tool="update_workflow",
- fingerprint=authored_fingerprint,
- )
- output_contract_ceiling_resolution = _credit_or_impose_output_contract(
- ctx,
- output_contract_evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- )
- if output_contract_ceiling_resolution is None:
- output_contract_ceiling_resolution = _guardrail_ceiling_actuation_resolution(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- )
- if output_contract_ceiling_resolution is not None:
- assert output_contract_evaluation is not None
- workflow_yaml = output_contract_ceiling_resolution.workflow_yaml
- params["workflow_yaml"] = workflow_yaml
- params["code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadata
- params["raw_code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadata
- ctx.raw_code_artifact_metadata = output_contract_ceiling_resolution.code_artifact_metadata
- # The pass path inherits the advisory lane's adjudication: the draft persists now, and a
- # run that binds nothing reopens the ladder instead of shipping an unverified workflow.
- _arm_pending_run_evidence(
- ctx,
- output_contract_evaluation.canonical_signature,
- set(output_contract_evaluation.observation_paths),
- )
- if output_contract_ceiling_resolution.mode == "imposed":
- _mark_output_contract_imposed(ctx, output_contract_evaluation.canonical_signature)
- output_contract_imposed_signature = output_contract_evaluation.canonical_signature
- output_contract_evaluation = output_contract_ceiling_resolution.evaluation
- elif output_contract_deficient:
- assert output_contract_evaluation is not None
- if not output_contract_ablated:
- payload = _record_output_contract_reject(
- ctx,
- output_contract_evaluation,
- summary="Submitted workflow does not satisfy the requested output contract.",
- authored_structural_fingerprint=authored_fingerprint,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- )
- if allow_static_output_uncertainty and _output_contract_advisory_granted(
- ctx, output_contract_evaluation.canonical_signature
- ):
- _arm_pending_run_evidence(
- ctx,
- output_contract_evaluation.canonical_signature,
- set(output_contract_evaluation.observation_paths),
- )
- else:
- reject_result = _output_contract_reject_result(
- output_contract_evaluation,
- payload=payload,
- tool_name="update_workflow",
- )
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error=str(reject_result["error"]),
- user_facing_summary=str(
- reject_result.get("user_facing_summary") or _compiled_authoring_user_summary()
- ),
- data=reject_result.get("data") if isinstance(reject_result.get("data"), dict) else None,
- repair_context=output_contract_evaluation.repair_context,
- record_repair_context_outcome=False,
- )
- typed_output_owner_contract_complete = (
- formation_prepared
- and output_contract_evaluation is not None
- and not output_contract_evaluation.has_deficiencies
- )
- # update_and_run_blocks already resolved the exact output owner and validated its typed
- # contract. Do not let the legacy output-intent scan reclassify a non-owner browser block
- # as another output owner. Save-only update_workflow retains the conservative label gate.
- missing_metadata_error = (
- None
- if typed_output_owner_contract_complete
- else _missing_code_artifact_metadata_error(
- workflow_yaml,
- ctx,
- params.get("code_artifact_metadata"),
- )
- )
- missing_labels = (
- []
- if missing_metadata_error is None
- else _missing_code_artifact_metadata_labels(workflow_yaml, ctx, params.get("code_artifact_metadata"))
- )
- metadata_reject_ablated = False
- if missing_metadata_error is not None and _record_output_contract_actuation_ablation_event(
- ctx,
- workflow_yaml,
- reason_code=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
- signature=(output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""),
- block_labels=missing_labels,
- required_paths=required_child_output_paths,
- ):
- missing_metadata_error = None
- metadata_reject_ablated = True
- if missing_metadata_error is not None:
- missing_metadata_output_facts = _missing_requested_output_facts(
- required_child_output_paths,
- reason_code=output_path_coverage_reason_code,
- )
- if required_child_output_paths:
- missing_metadata_error = (
- f"{missing_metadata_error}\nRequired requested output paths: "
- f"{', '.join(sorted(required_child_output_paths))}"
- )
- metadata_repair_context = _metadata_output_repair_context(
- block_labels=missing_labels,
- required_paths=required_child_output_paths,
- coverage_reason_code=output_path_coverage_reason_code,
- source=output_path_coverage_source,
- summary=missing_metadata_error,
- )
- metadata_repair_contract = _metadata_repair_contract(
- block_labels=missing_labels,
- required_paths=required_child_output_paths,
- source=output_path_coverage_source,
- reason_code=output_path_coverage_reason_code,
- )
- metadata_reject_payload = _code_artifact_metadata_reject_payload(
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- offending_labels=[],
- missing_labels=missing_labels,
- violation_categories=["missing_code_artifact_metadata"],
- )
- candidate_outcome = _build_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=missing_metadata_error,
- structural_payload=metadata_reject_payload,
- block_labels=missing_labels,
- missing_requested_output_facts=missing_metadata_output_facts,
- )
- missing_fields_by_label = _metadata_missing_required_fields_by_label(
- params.get("code_artifact_metadata"),
- labels=missing_labels,
- missing_labels=missing_labels,
- )
- if len(missing_labels) != 1:
- _reset_metadata_reject_ladder_state(
- ctx,
- superseding_reject_family="multi_label_missing_code_artifact_metadata",
- )
- metadata_ladder_decision = (
- None
- if len(missing_labels) != 1
- or copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID)
- else _metadata_reject_ladder_decision(
- ctx,
- candidate_outcome=candidate_outcome,
- reject_family="missing_code_artifact_metadata",
- missing_fields_by_label=missing_fields_by_label,
- )
- )
- if metadata_ladder_decision is not None and metadata_ladder_decision.action == "terminal":
- assert ctx.blocker_signal is not None
- return reject(
- error=ctx.blocker_signal.user_facing_reason,
- user_facing_summary=ctx.blocker_signal.user_facing_reason,
- data=_metadata_reject_terminal_data(metadata_ladder_decision),
- repair_context=metadata_repair_context,
- record_repair_context_outcome=False,
- )
- missing_metadata_reject_count = _record_output_contract_family_reject(
- ctx,
- required_child_output_paths,
- reject_family=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
- )
- record_build_test_outcome(ctx, candidate_outcome)
- _capture_rejected_code_artifact_metadata(ctx)
- credential_scout_errors = (
- []
- if _request_policy_allows_untested_code_block_draft(ctx)
- else _credentialed_code_block_scout_gate_errors(
- workflow_yaml,
- ctx,
- block_labels=params.get("block_labels"),
- )
- )
- budget_terminal = _adjudicate_output_contract_budget(
- ctx,
- required_child_output_paths,
- count=missing_metadata_reject_count,
- block_label=missing_labels[0] if len(missing_labels) == 1 else "",
- )
- if credential_scout_errors:
- _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
- elif missing_metadata_reject_count < 1 and not budget_terminal:
- _record_code_authoring_guardrail_reject(ctx)
- metadata_signature = _output_contract_signature(ctx=ctx, required_paths=required_child_output_paths)
- if metadata_ladder_decision is not None and metadata_ladder_decision.rung is not None:
- _record_armed_directive_fingerprint(
- ctx,
- metadata_signature,
- _metadata_reject_seam_fingerprint(
- workflow_yaml,
- metadata_signature,
- missing_fields_by_label,
- ),
- )
- metadata_convergence_directive = (
- _emit_metadata_convergence_directive(
- signature=metadata_signature,
- block_label=missing_labels[0],
- missing_fields_by_label=missing_fields_by_label,
- required_paths=required_child_output_paths,
- escalate=metadata_ladder_decision.rung == 2,
- )
- if metadata_ladder_decision is not None
- and metadata_ladder_decision.rung is not None
- and len(missing_labels) == 1
- else None
- )
- metadata_reject_data = _code_repair_progress_data(
- metadata_repair_context,
- missing_requested_output_facts=missing_metadata_output_facts,
- metadata_repair_contract=metadata_repair_contract,
- )
- if metadata_convergence_directive is not None:
- metadata_reject_data["metadata_convergence_directive"] = metadata_convergence_directive
- return reject(
- error=missing_metadata_error,
- user_facing_summary=_compiled_authoring_user_summary(),
- data=metadata_reject_data,
- repair_context=metadata_repair_context,
- record_repair_context_outcome=False,
- )
- scout_trajectory = getattr(ctx, "scout_trajectory", None)
+ scout_trajectory = ctx.scout_trajectory
normalization = _normalize_code_artifact_metadata_detailed(
params.get("code_artifact_metadata"),
workflow_yaml,
@@ -14117,73 +10280,14 @@ async def _update_workflow(
verified_runtime_output_paths_by_label=_verified_runtime_output_contract_paths_by_label(ctx, workflow_yaml),
advisory_declared_output_return_shape_labels=(
{output_contract_evaluation.block_label}
- if output_contract_evaluation is not None
- and (output_contract_static_advisory_allowed or output_contract_ceiling_resolution is not None)
+ if output_contract_evaluation is not None and output_contract_static_advisory_allowed
else None
),
)
code_artifact_metadata = normalization.normalized
- code_artifact_metadata_error = normalization.error
- # A schema incompatibility rejects below regardless, so suppressing here would emit an
- # ablation event for a save that never lands and overstate what the flag let through.
- if (
- code_artifact_metadata_error is not None
- and not normalization.schema_incompatibilities
- and (
- metadata_reject_ablated
- or _record_output_contract_actuation_ablation_event(
- ctx,
- workflow_yaml,
- reason_code=_METADATA_NORMALIZATION_REJECT_FAMILY,
- signature=(
- output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""
- ),
- block_labels=normalization.offending_labels,
- required_paths=required_child_output_paths,
- )
- )
- ):
- code_artifact_metadata_error = None
- if code_artifact_metadata_error is not None:
- _reset_metadata_reject_ladder_state(
- ctx,
- superseding_reject_family=_METADATA_NORMALIZATION_REJECT_FAMILY,
- )
- record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
- normalization_reject_count = _record_output_contract_family_reject(
- ctx,
- required_child_output_paths,
- reject_family=_METADATA_NORMALIZATION_REJECT_FAMILY,
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=code_artifact_metadata_error,
- structural_payload=_output_contract_author_time_structural_payload(
- ctx,
- required_child_output_paths,
- block_label=normalization.offending_labels[0] if len(normalization.offending_labels) == 1 else "",
- )
- or _code_artifact_metadata_reject_payload(
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- offending_labels=normalization.offending_labels,
- violation_categories=_metadata_violation_categories(normalization.violations),
- ),
- block_labels=normalization.offending_labels,
- )
- _capture_rejected_code_artifact_metadata(ctx)
- _adjudicate_output_contract_budget(
- ctx,
- required_child_output_paths,
- count=normalization_reject_count,
- block_label=normalization.offending_labels[0] if len(normalization.offending_labels) == 1 else "",
- )
- if normalization.schema_incompatibilities:
- incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)
- if incompatibility is not None:
- return _reject_schema_incompatibility(ctx, incompatibility, reject)
- prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)
+ schema_incompatibility_finding = merge_schema_incompatibilities(normalization.schema_incompatibilities)
+
+ prior_workflow_yaml = ctx.workflow_yaml
code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
code_authoring_repair_context = (
_code_block_authoring_repair_context(workflow_yaml, prior_workflow_yaml)
@@ -14195,10 +10299,6 @@ async def _update_workflow(
if _request_policy_allows_untested_code_block_draft(ctx)
else _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx, block_labels=params.get("block_labels"))
)
- unresolved_symbol_priority_reject = _is_unresolved_symbol_repair_context(code_authoring_repair_context)
- credential_priority_reject = (
- bool(credential_scout_errors) and code_artifact_metadata_error is None and not unresolved_symbol_priority_reject
- )
if code_safety_errors:
_set_code_authoring_repair_context(ctx, code_authoring_repair_context)
if code_authoring_repair_context is None:
@@ -14208,13 +10308,12 @@ async def _update_workflow(
summary="Code authoring guardrail rejected the submitted code block.",
structural_payload=_code_safety_reject_payload(code_safety_errors),
)
- _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=credential_priority_reject)
# Per-label salvage keeps conforming metadata across a rejection; a
# rejected code block keeps nothing, since its yaml never becomes the
# draft. Prior-draft labels survive every rejection gate below — the
# accept path prunes to the submitted blocks once the draft switches.
if code_artifact_metadata and not code_safety_errors:
- existing_metadata = getattr(ctx, "code_artifact_metadata", None)
+ existing_metadata = ctx.code_artifact_metadata
merged_metadata = {
**(existing_metadata if isinstance(existing_metadata, dict) else {}),
**code_artifact_metadata,
@@ -14229,53 +10328,50 @@ async def _update_workflow(
params["code_artifact_metadata"] = merged_metadata
workflow_yaml = _apply_metadata_contract_schema_to_workflow_yaml(ctx, workflow_yaml, merged_metadata)
params["workflow_yaml"] = workflow_yaml
- if (
- credential_scout_errors
- and code_safety_errors
- and code_artifact_metadata_error is None
- and unresolved_symbol_priority_reject
- ):
- return reject(
- error="\n".join(str(error) for error in code_safety_errors if error),
- user_facing_summary=_code_seam_rejection_user_summary(
- metadata_rejected=False,
- code_rejected=True,
+ if credential_scout_errors and code_safety_errors:
+ if _is_unresolved_symbol_repair_context(code_authoring_repair_context):
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=CODE_SAFETY_BLOCK_ID,
+ error="\n".join(str(error) for error in code_safety_errors if error),
+ user_facing_summary=_code_seam_rejection_user_summary(
+ metadata_rejected=False,
+ code_rejected=True,
+ ),
+ data=_code_repair_progress_data(code_authoring_repair_context),
+ ),
+ repair_context=code_authoring_repair_context,
+ )
+ arm_credential_scout_reopen(ctx, _credential_scout_reopen_identity_digest(workflow_yaml))
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=CREDENTIAL_SCOUT_BLOCK_ID,
+ error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
+ user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
+ data={
+ "failure_type": "missing_credential_or_init",
+ "diagnostic_code_safety_errors": code_safety_errors,
+ **(
+ {"authoring_repair_context": code_authoring_repair_context.model_dump(mode="json")}
+ if code_authoring_repair_context is not None
+ else {}
+ ),
+ },
),
- data=_code_repair_progress_data(code_authoring_repair_context),
repair_context=code_authoring_repair_context,
)
- if credential_scout_errors and code_safety_errors and code_artifact_metadata_error is None:
- arm_credential_scout_reopen(ctx, _credential_scout_reopen_identity_digest(workflow_yaml))
- return reject(
- error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
- user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
- data={
- "failure_type": "missing_credential_or_init",
- "diagnostic_code_safety_errors": code_safety_errors,
- **(
- {"authoring_repair_context": code_authoring_repair_context.model_dump(mode="json")}
- if code_authoring_repair_context is not None
- else {}
+ human_facing_code_safety_errors = _human_facing_code_safety_errors(code_safety_errors)
+ if human_facing_code_safety_errors:
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=CODE_SAFETY_BLOCK_ID,
+ error="\n".join(human_facing_code_safety_errors),
+ user_facing_summary=_code_seam_rejection_user_summary(
+ metadata_rejected=False,
+ code_rejected=True,
),
- },
- repair_context=code_authoring_repair_context,
- )
- seam_errors = [
- error
- for error in (
- code_artifact_metadata_error,
- *_human_facing_code_safety_errors(code_safety_errors),
- )
- if error
- ]
- if seam_errors:
- return reject(
- error="\n".join(seam_errors),
- user_facing_summary=_code_seam_rejection_user_summary(
- metadata_rejected=code_artifact_metadata_error is not None,
- code_rejected=bool(code_safety_errors),
+ data=_code_repair_progress_data(code_authoring_repair_context),
),
- data=_code_repair_progress_data(code_authoring_repair_context),
repair_context=code_authoring_repair_context,
)
if credential_scout_errors:
@@ -14285,19 +10381,21 @@ async def _update_workflow(
summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
structural_payload=_credential_scout_reject_payload(workflow_yaml),
)
- _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
arm_credential_scout_reopen(ctx, _credential_scout_reopen_identity_digest(workflow_yaml))
- return reject(
- error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
- user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
- data={"failure_type": "missing_credential_or_init"},
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=CREDENTIAL_SCOUT_BLOCK_ID,
+ error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
+ user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
+ data={"failure_type": "missing_credential_or_init"},
+ )
)
if allow_missing_credentials is None:
- allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
+ allow_missing_credentials = ctx.allow_untested_workflow_draft is True
if not allow_missing_credentials:
credential_error = await _credential_reference_validation_error(workflow_yaml, ctx)
if credential_error is not None:
- return reject(error=credential_error)
+ return _blocked(AuthorTimeBlock(block_id=CREDENTIAL_SCOUT_BLOCK_ID, error=credential_error))
misbinding_findings = _credential_id_misbinding_findings(workflow_yaml)
if misbinding_findings:
@@ -14307,19 +10405,30 @@ async def _update_workflow(
workflow_id=ctx.workflow_id,
findings=misbinding_findings,
)
- return reject(error=_credential_id_misbinding_error_message(misbinding_findings))
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=CREDENTIAL_SCOUT_BLOCK_ID,
+ error=_credential_id_misbinding_error_message(misbinding_findings),
+ )
+ )
output_policy_verdict = evaluate_output_policy(
- request_policy=getattr(ctx, "request_policy", None),
+ request_policy=ctx.request_policy,
workflow_yaml=workflow_yaml,
tool_arguments=params,
)
+ output_policy_steered_reasons = demote_author_time_steer_reasons(output_policy_verdict)
if not output_policy_verdict.allowed:
output_policy_trace_data = output_policy_verdict_to_trace_data(
output_policy_verdict,
surface="tool_body",
tool_name="update_workflow",
)
+ if output_policy_steered_reasons:
+ output_policy_trace_data = {
+ **output_policy_trace_data,
+ "steered_reason_codes": [reason.value for reason in output_policy_steered_reasons],
+ }
output_policy_error = format_output_policy_tool_error(output_policy_verdict)
_record_code_only_raw_secret_reject_span(ctx, output_policy_verdict)
LOG.info(
@@ -14332,28 +10441,13 @@ async def _update_workflow(
summary=output_policy_error,
structural_payload=output_policy_trace_data,
)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=output_policy_error)
+ return _blocked(AuthorTimeBlock(block_id=CREDENTIAL_SCOUT_BLOCK_ID, error=output_policy_error))
# Prefer the most-recent in-turn emission so cross-path flows (inline
# REPLACE_WORKFLOW followed by update_workflow) compare against what the
# LLM actually saw, not the turn-start persisted state.
- last_yaml = getattr(ctx, "last_workflow_yaml", None)
+ last_yaml = ctx.last_workflow_yaml
prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml
- stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)
- if stale_metadata:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=_stale_block_metadata_message(stale_metadata))
-
- wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
- if wait_block_error:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=wait_block_error)
-
- challenge_http_error = _challenge_http_request_reject_message(ctx, workflow_yaml, ctx.workflow_yaml)
- if challenge_http_error:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=challenge_http_error)
# Post-emission reject of copilot-v2 writes that introduce a banned
# block type. The schema pre_hook only fires when the LLM consults the
@@ -14366,260 +10460,21 @@ async def _update_workflow(
)
if banned_items:
_record_banned_block_reject_span("_update_workflow", banned_items)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=_banned_block_reject_message(banned_items, ctx))
-
- download_scout_error = _download_scout_required_error(ctx, workflow_yaml)
- if download_scout_error:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=download_scout_error)
-
- download_binding_error = _download_binding_required_error(ctx, workflow_yaml)
- if download_binding_error:
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=download_binding_error)
-
- composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
- if composition_evidence_error:
- LOG.info(
- "copilot composition page evidence rejected workflow",
- workflow_permanent_id=ctx.workflow_permanent_id,
- target_url=workflow_target_url(workflow_yaml),
+ return _blocked(
+ AuthorTimeBlock(
+ block_id=BANNED_BLOCKS_BLOCK_ID,
+ error=_banned_block_reject_message(banned_items, ctx),
+ )
)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(error=composition_evidence_error)
# New data-write blocks default to surfacing failures rather than swallowing them.
workflow_yaml = default_data_write_continue_on_failure(workflow_yaml, ctx.workflow_yaml)
- convergence_reject = _recorded_outcome_convergence_reject(
- ctx,
- workflow_yaml=workflow_yaml,
- code_artifact_metadata=getattr(ctx, "code_artifact_metadata", None),
- )
- if convergence_reject is not None:
- frontier_unchanged = convergence_reject.reason == "frontier_unchanged"
- convergence_ceiling_resolution = _guardrail_ceiling_actuation_resolution(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- frontier_unchanged=frontier_unchanged,
- )
- if convergence_ceiling_resolution is not None:
- imposed_code_artifact_metadata = convergence_ceiling_resolution.code_artifact_metadata
- workflow_yaml = convergence_ceiling_resolution.workflow_yaml
- params["workflow_yaml"] = workflow_yaml
- params["code_artifact_metadata"] = imposed_code_artifact_metadata
- params["raw_code_artifact_metadata"] = imposed_code_artifact_metadata
- ctx.raw_code_artifact_metadata = imposed_code_artifact_metadata
- if isinstance(imposed_code_artifact_metadata, dict):
- ctx.code_artifact_metadata = imposed_code_artifact_metadata
- output_contract_imposed_signature = (
- _apply_output_contract_ceiling_bookkeeping(ctx, convergence_ceiling_resolution)
- or output_contract_imposed_signature
- )
- else:
- block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- _record_author_time_reject_outcome(
- ctx,
- reason_code="unchanged_after_recorded_outcome",
- summary="The authored code and output structure are unchanged after the last recorded test outcome.",
- structural_payload={
- "reason_code": "unchanged_after_recorded_outcome",
- "authored_structure_signature": convergence_reject.authored_structure_signature,
- "block_labels": block_labels,
- },
- authored_structure_signature=convergence_reject.authored_structure_signature,
- block_labels=block_labels,
- )
- _record_code_authoring_guardrail_reject(ctx, frontier_unchanged=frontier_unchanged)
- LOG.info(
- "copilot recorded outcome convergence behavior",
- convergence_reason=convergence_reject.reason,
- commit_early_terminal=convergence_reject.commit_early_terminal,
- block_labels=block_labels,
- )
- if convergence_reject.commit_early_terminal:
- _commit_recorded_outcome_early_terminal(ctx)
- return reject(
- error=(
- "Submitted workflow left the frontier the last recorded test outcome named unchanged. "
- "Revise the code block or output metadata that owns that frontier before testing again."
- ),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
-
- recorded_output_paths_required = output_path_coverage_source == "recorded_outcome"
- recorded_missing_output_paths = (
- _recorded_outcome_missing_output_paths(ctx) if recorded_output_paths_required else set()
- )
- unresolved_recorded_output_paths = recorded_missing_output_paths
-
- output_empty_labels = (
- _output_empty_code_block_labels(workflow_yaml, getattr(ctx, "code_artifact_metadata", None))
- if recorded_output_paths_required and (not recorded_missing_output_paths or unresolved_recorded_output_paths)
- else []
- )
- if output_empty_labels:
- authored_structure_signature = authored_structure_signature_from_workflow(
- workflow_yaml,
- getattr(ctx, "code_artifact_metadata", None),
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary="Submitted workflow does not return any keyed output after the last recorded test outcome.",
- structural_payload={
- "reason_code": "recorded_outcome_requires_output_candidate",
- "authored_structure_signature": authored_structure_signature,
- "empty_output_block_labels": output_empty_labels,
- "recorded_reason_code": "outcome_not_demonstrated",
- },
- authored_structure_signature=authored_structure_signature,
- block_labels=output_empty_labels,
- )
- _record_code_authoring_guardrail_reject(ctx)
- reject_data = _code_repair_progress_data()
- metadata_convergence_directive = _synthesized_metadata_reject_directive(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=ctx.code_artifact_metadata,
- label_candidates=output_empty_labels,
- required_paths=unresolved_recorded_output_paths,
- )
- if (
- metadata_convergence_directive is not None
- and metadata_convergence_directive.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- ):
- assert ctx.blocker_signal is not None
- return reject(
- error=ctx.blocker_signal.user_facing_reason,
- user_facing_summary=ctx.blocker_signal.user_facing_reason,
- data=metadata_convergence_directive,
- record_repair_context_outcome=False,
- )
- if metadata_convergence_directive is not None:
- reject_data["metadata_convergence_directive"] = metadata_convergence_directive
- return reject(
- error=(
- "Submitted workflow does not return any keyed output after the last recorded test outcome. "
- "Add structured output for the unsatisfied completion criteria before testing again."
- ),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=reject_data,
- )
-
- missing_output_paths = (
- _candidate_missing_required_output_paths(
- workflow_yaml,
- getattr(ctx, "code_artifact_metadata", None),
- required_paths=unresolved_recorded_output_paths,
- )
- if recorded_output_paths_required
- else []
- )
- if missing_output_paths:
- authored_structure_signature = authored_structure_signature_from_workflow(
- workflow_yaml,
- getattr(ctx, "code_artifact_metadata", None),
- )
- block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- missing_output_roots = sorted(
- {root for path in missing_output_paths if (root := _top_level_path_segment(path))}
- )
- _record_author_time_reject_outcome(
- ctx,
- reason_code="metadata_reject",
- summary=(
- "Submitted workflow does not cover the missing requested output paths "
- "from the last recorded test outcome."
- ),
- structural_payload={
- "reason_code": "recorded_outcome_missing_output_coverage",
- "authored_structure_signature": authored_structure_signature,
- "missing_output_paths": missing_output_paths,
- "missing_output_roots": missing_output_roots,
- "block_labels": block_labels,
- "recorded_reason_code": "outcome_not_demonstrated",
- },
- authored_structure_signature=authored_structure_signature,
- block_labels=block_labels,
- missing_requested_output_facts=[
- {
- "output_path": path,
- "output_root": _top_level_path_segment(path),
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- for path in missing_output_paths
- ],
- )
- _record_code_authoring_guardrail_reject(ctx)
- missing_path_text = ", ".join(missing_output_paths[:8])
- reject_data = _code_repair_progress_data()
- metadata_convergence_directive = _synthesized_metadata_reject_directive(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=ctx.code_artifact_metadata,
- label_candidates=block_labels,
- required_paths=set(missing_output_paths),
- )
- if (
- metadata_convergence_directive is not None
- and metadata_convergence_directive.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- ):
- assert ctx.blocker_signal is not None
- return reject(
- error=ctx.blocker_signal.user_facing_reason,
- user_facing_summary=ctx.blocker_signal.user_facing_reason,
- data=metadata_convergence_directive,
- record_repair_context_outcome=False,
- )
- if metadata_convergence_directive is not None:
- reject_data["metadata_convergence_directive"] = metadata_convergence_directive
- return reject(
- error=(
- "Submitted workflow does not cover the missing requested output paths from the last recorded test "
- f"outcome: {missing_path_text}. Declare those exact output_path values in goal_value_paths and "
- "produce matching structured output before testing again; output_root is diagnostic only."
- ),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=reject_data,
- )
-
- select_option_mismatch_context = _select_option_text_click_repair_context(workflow_yaml, ctx)
- if select_option_mismatch_context is not None:
- _set_code_authoring_repair_context(ctx, select_option_mismatch_context)
- _record_code_authoring_guardrail_reject(ctx)
- return {
- "ok": False,
- "error": (
- "Submitted workflow replaces a captured select-option interaction with a text click. "
- "Use the select element API for the captured selector before testing again."
- ),
- "user_facing_summary": _compiled_authoring_user_summary(),
- "data": _code_repair_progress_data(select_option_mismatch_context),
- }
-
- pre_persist_spine = _pre_persist_scouted_spine_result(workflow_yaml, ctx)
- if pre_persist_spine is not None:
- _record_scouted_spine_omission_reject(ctx, pre_persist_spine.omission_digest)
- _set_code_authoring_repair_context(ctx, pre_persist_spine.repair_context)
- _record_code_authoring_guardrail_reject(ctx)
- return reject(
- error="\n".join(pre_persist_spine.violations),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(pre_persist_spine.repair_context),
- repair_context=pre_persist_spine.repair_context,
- record_repair_context_outcome=False,
- )
-
try:
# A code block renders code-first (goal + plain step timeline) only when it
# carries a `prompt`; the model authors the goal as artifact `declared_goal`
# and code regeneration drops it, so carry the goal onto the block here.
- artifact_metadata = getattr(ctx, "code_artifact_metadata", None)
+ artifact_metadata = ctx.code_artifact_metadata
fallback_goals = {
label: str(meta["declared_goal"]).strip()
for label, meta in (artifact_metadata or {}).items()
@@ -14691,14 +10546,11 @@ async def _update_workflow(
ctx.workflow_yaml = workflow_yaml
ctx.persisted_draft_browser_calls = _workflow_yaml_browser_call_pairs(workflow_yaml)
ctx.code_authoring_guardrail_reject_count = 0
- ctx.scouted_spine_previous_omission_digest = None
- ctx.scouted_spine_repeated_identical_missing_steps = False
ctx.last_code_authoring_reject_was_credential_priority = False
ctx.last_output_policy_reject_reason_codes = None
- ctx.metadata_reject_ladder_state = None
_clear_code_authoring_repair_context(ctx)
_clear_held_churn_signals(ctx)
- accepted_metadata = getattr(ctx, "code_artifact_metadata", None)
+ accepted_metadata = ctx.code_artifact_metadata
if isinstance(accepted_metadata, dict) and accepted_metadata:
accepted_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
pruned_metadata = {block: row for block, row in accepted_metadata.items() if block in accepted_labels}
@@ -14723,96 +10575,27 @@ async def _update_workflow(
data["imposed_substitutions"] = imposition.substitutions
if stripped_sandbox_imports:
data["stripped_redundant_imports"] = stripped_sandbox_imports
+ findings = _author_time_findings(
+ schema_incompatibility=schema_incompatibility_finding,
+ metadata_violations=normalization.violations,
+ imposition_violations=imposition.violations,
+ )
+ if findings:
+ data["findings"] = findings
return {
"ok": True,
"data": data,
"_workflow": workflow,
}
except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:
- repair_data = None
- user_facing_summary = None
- if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- repair_data = _code_repair_progress_data()
- user_facing_summary = _code_seam_rejection_user_summary(
- metadata_rejected=False,
- code_rejected=True,
- )
- return reject(
- error=f"{INTERNAL_VALIDATION_FAILURE_PREFIX}{e}",
- user_facing_summary=user_facing_summary,
- data=repair_data,
- )
-
-
-async def _replay_captured_never_captured_obligation(ctx: AgentContext) -> dict[str, Any] | None:
- """Retry the exact rejected authoring call after its canonical browser action is captured."""
- obligation = getattr(ctx, "never_captured_obligation", None)
- if (
- obligation is None
- or obligation.state != "captured"
- or obligation.turn_id != str(getattr(ctx, "turn_id", ""))
- or obligation.replay_payload is None
- ):
- return None
- payload = obligation.replay_payload
- workflow_yaml = payload.params.get("workflow_yaml")
- if (
- not isinstance(workflow_yaml, str)
- or hashlib.sha256(workflow_yaml.encode()).hexdigest() != obligation.draft_fingerprint
- ):
- ctx.never_captured_obligation = replace(obligation, state="consumed")
- ctx.synthesized_block_reopened_for_capture_obligation = False
- LOG.warning(
- "copilot_never_captured_obligation_replay_payload_invalid",
- identity_digest=obligation.identity_digest,
- turn_id=obligation.turn_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- draft_fingerprint=obligation.draft_fingerprint,
- block_label=obligation.block_label,
- site=obligation.site,
- )
- return None
- prior_definition = getattr(getattr(ctx, "last_workflow", None), "workflow_definition", None)
- try:
- result = await _update_workflow(
- copy.deepcopy(payload.params),
- ctx,
- allow_missing_credentials=payload.allow_missing_credentials,
- allow_static_output_uncertainty=payload.allow_static_output_uncertainty,
- formation_prepared=payload.formation_prepared,
+ return _tool_error(
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}{e}",
+ user_facing_summary=(
+ _code_seam_rejection_user_summary(metadata_rejected=False, code_rejected=True)
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ else None
+ ),
)
- except Exception:
- current_obligation = getattr(ctx, "never_captured_obligation", None)
- if current_obligation is not None and current_obligation.identity_digest == obligation.identity_digest:
- ctx.never_captured_obligation = obligation
- ctx.synthesized_block_reopened_for_capture_obligation = True
- raise
- current_obligation = ctx.never_captured_obligation
- if current_obligation is not None and current_obligation.state == "captured":
- ctx.never_captured_obligation = replace(current_obligation, state="consumed")
- ctx.synthesized_block_reopened_for_capture_obligation = False
- _record_workflow_update_result(ctx, result, prior_definition)
- data = result.get("data")
- post_replay_obligation = ctx.never_captured_obligation
- obligation_state = (
- post_replay_obligation.state
- if post_replay_obligation is not None and post_replay_obligation.identity_digest == obligation.identity_digest
- else "superseded"
- )
- LOG.info(
- "copilot_never_captured_obligation_replayed",
- identity_digest=obligation.identity_digest,
- turn_id=obligation.turn_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- draft_fingerprint=obligation.draft_fingerprint,
- block_label=obligation.block_label,
- site=obligation.site,
- captured_trajectory_index=obligation.captured_trajectory_index,
- obligation_state=obligation_state,
- ok=result.get("ok") is True,
- reason_code=data.get("reason_code") if isinstance(data, dict) else None,
- )
- return result
def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow) -> None:
@@ -14878,8 +10661,6 @@ def _record_workflow_update_result(
copilot_ctx.synthesized_goal_complete_landed = True
copilot_ctx.pending_goal_complete_landing = False
copilot_ctx.synthesized_block_reopened_after_failed_run = False
- copilot_ctx.synthesized_block_reopened_for_output_coverage = False
- copilot_ctx.uncovered_output_rescout_steer_key = None
copilot_ctx.test_after_update_done = False
copilot_ctx.post_update_nudge_count = 0
copilot_ctx.last_test_ok = None
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index ef0037c88..ee70b0cb5 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -9,15 +9,12 @@ from typing import TYPE_CHECKING, Any
import structlog
from skyvern.forge.sdk.copilot.blocker_signal import (
- DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,
DISCOVERY_EXHAUSTED_NO_ENTRY_URL_REASON_CODE,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
GENUINELY_TERMINAL_BLOCKER_REASON_CODES as GENUINELY_TERMINAL_BLOCKER_REASON_CODES,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
blocker_signal_is_genuinely_terminal,
build_output_source_unobservable_blocker_signal,
@@ -32,7 +29,6 @@ from skyvern.forge.sdk.copilot.output_contracts import (
)
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
from skyvern.forge.sdk.copilot.runtime import output_contract_ladder_unresolved
-from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_REASON_CODE
from skyvern.forge.sdk.copilot.turn_ownership import (
TURN_HALT_RETIRED_LOG_EVENT,
TurnClaimant,
@@ -60,11 +56,8 @@ class TurnHaltKind(StrEnum):
ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"
PROBABLE_SITE_BLOCK = "probable_site_block"
REPAIR_CEILING_REACHED = "repair_ceiling_reached"
- SCHEMA_INCOMPATIBILITY = "schema_incompatibility"
OUTPUT_SOURCE_UNOBSERVABLE = "output_source_unobservable"
DELIVERED_UNVERIFIED = "delivered_unverified"
- DEFINITION_CONTRACT_UNSATISFIED = "definition_contract_unsatisfied"
- METADATA_REJECT_SAME_KEY = "metadata_reject_same_key"
class TurnHaltVerdict(StrEnum):
@@ -95,17 +88,13 @@ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(
}
)
_PROBABLE_SITE_BLOCK_REASON_CODES = frozenset({"probable_site_block_stop"})
-_SCHEMA_INCOMPATIBILITY_REASON_CODES = frozenset({SCHEMA_INCOMPATIBILITY_REASON_CODE})
_OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset(
{
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
ADVISORY_DISPATCH_STALLED_REASON_CODE,
}
)
-_DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES = frozenset({DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE})
-_METADATA_REJECT_SAME_KEY_REASON_CODES = frozenset({METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE})
# Halts the agent did not choose: a verified outcome may suppress these.
# ACTIVE_TERMINAL_CHALLENGE is voluntary and is deliberately excluded so a
@@ -115,19 +104,13 @@ _INVOLUNTARY_TURN_HALT_KINDS = frozenset(
TurnHaltKind.LOOP_DETECTED,
TurnHaltKind.PROBABLE_SITE_BLOCK,
TurnHaltKind.REPAIR_CEILING_REACHED,
- TurnHaltKind.SCHEMA_INCOMPATIBILITY,
TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE,
- TurnHaltKind.DEFINITION_CONTRACT_UNSATISFIED,
- TurnHaltKind.METADATA_REJECT_SAME_KEY,
}
)
_INVOLUNTARY_BLOCKER_REASON_CODES = (
_LOOP_TERMINAL_REASON_CODES
| _PROBABLE_SITE_BLOCK_REASON_CODES
- | _SCHEMA_INCOMPATIBILITY_REASON_CODES
| _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES
- | _DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES
- | _METADATA_REJECT_SAME_KEY_REASON_CODES
| frozenset({REPAIR_CEILING_REASON_CODE})
)
_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES = frozenset({ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE})
@@ -158,14 +141,8 @@ def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind |
return TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
if reason in _PROBABLE_SITE_BLOCK_REASON_CODES:
return TurnHaltKind.PROBABLE_SITE_BLOCK
- if reason in _SCHEMA_INCOMPATIBILITY_REASON_CODES:
- return TurnHaltKind.SCHEMA_INCOMPATIBILITY
if reason in _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES:
return TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE
- if reason in _DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES:
- return TurnHaltKind.DEFINITION_CONTRACT_UNSATISFIED
- if reason in _METADATA_REJECT_SAME_KEY_REASON_CODES:
- return TurnHaltKind.METADATA_REJECT_SAME_KEY
return None
diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.py
index 3452bbf01..ee8257675 100644
--- a/skyvern/forge/sdk/copilot/turn_outcome.py
+++ b/skyvern/forge/sdk/copilot/turn_outcome.py
@@ -167,15 +167,6 @@ def _string_or_none(value: object) -> str | None:
return value if isinstance(value, str) and value else None
-def _schema_incompatibility_summary(ctx: Any) -> dict[str, Any] | None:
- incompatibility = getattr(ctx, "latest_schema_incompatibility", None)
- to_summary = getattr(incompatibility, "to_summary_dict", None)
- if not callable(to_summary):
- return None
- summary = to_summary()
- return summary if isinstance(summary, dict) else None
-
-
def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, Any]:
turn_halt_kind = getattr(getattr(ctx, "turn_halt", None), "kind", None)
turn_halt_kind_value = getattr(turn_halt_kind, "value", turn_halt_kind)
@@ -186,7 +177,6 @@ def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, Any]:
),
"copilot_repair_ceiling_hit": turn_halt_kind_value == "repair_ceiling_reached",
"copilot_pending_capability": pending_capability,
- "copilot_schema_incompatibility": _schema_incompatibility_summary(ctx),
}
diff --git a/skyvern/forge/sdk/copilot/turn_ownership.py b/skyvern/forge/sdk/copilot/turn_ownership.py
index 2ee5f26c2..5ec4a85f1 100644
--- a/skyvern/forge/sdk/copilot/turn_ownership.py
+++ b/skyvern/forge/sdk/copilot/turn_ownership.py
@@ -12,9 +12,7 @@ import structlog
from skyvern.forge.sdk.copilot.blocker_signal import (
GENUINELY_TERMINAL_BLOCKER_REASON_CODES,
- RECORDED_OUTCOME_GROUNDING_REASON_CODE,
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
- UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
CopilotToolBlockerSignal,
blocker_signal_is_genuinely_terminal,
build_llm_tool_error_payload,
@@ -38,14 +36,10 @@ class TurnClaimant(StrEnum):
GENUINELY_TERMINAL = "genuinely_terminal"
OUTPUT_CONTRACT_ACTUATION = "output_contract_actuation"
CREDENTIAL_PRIORITY_CHURN = "credential_priority_authoring_churn"
- METADATA_RUN_PREFLIGHT_REJECT = "metadata_run_preflight_reject"
ACTUATION_OBLIGATION_FILL = "actuation_obligation_fill"
ACTUATION_OBLIGATION_LOGIN_COMPLETION = "actuation_obligation_login_completion"
POST_RUN_PAGE_PATH_INTERACTION = "post_run_page_path_interaction"
- CAPTURE_OBLIGATION_REOPEN = "capture_obligation_reopen"
CREDENTIAL_SCOUT_REOPEN = "credential_scout_reopen"
- UNCOVERED_OUTPUT_RESCOUT_STEER = "uncovered_output_rescout_steer"
- RECORDED_OUTCOME_GROUNDING = "recorded_outcome_grounding"
SYNTHESIZED_BLOCK_PERSISTENCE_FORCE = "synthesized_block_persistence_force"
CODE_AUTHORING_CHURN = "code_authoring_guardrail_churn"
LOOP_DETECTED = "loop_detected"
@@ -68,8 +62,6 @@ _PRECEDENCE_ORDER: tuple[TurnClaimant, ...] = (
# Bound-8 credential-priority churn defers only to terminal evidence and the actuation ladder,
# preserving the credential-scout reply.
TurnClaimant.CREDENTIAL_PRIORITY_CHURN,
- # Author-time preflight reject steers before any churn or loop floor accumulates.
- TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT,
# Actuation-obligation fill carve-out admits the required fill tool through the persistence
# gate; predicate-gated at its call site, never an unconditional rank flip.
TurnClaimant.ACTUATION_OBLIGATION_FILL,
@@ -78,22 +70,12 @@ _PRECEDENCE_ORDER: tuple[TurnClaimant, ...] = (
TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION,
# Post-run page-path repair admits only its outcome-bound click/Enter window.
TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION,
- # A never-captured obligation admits only the exact canonical interaction that can satisfy it.
- TurnClaimant.CAPTURE_OBLIGATION_REOPEN,
# One-shot credential-scout reopen admits evaluate through the persistence gate;
# predicate-gated at its call site.
TurnClaimant.CREDENTIAL_SCOUT_REOPEN,
- # First-match cascade order: the uncovered-output rescout steer is evaluated before the
- # persistence force and the grounding gate.
- TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER,
- # Grounding-over-persistence nested exception: a live grounding requirement is emitted instead
- # of the persistence force when both hold.
- TurnClaimant.RECORDED_OUTCOME_GROUNDING,
# A budget-exhausted churn halt is the loop exit and outranks the persistence force, whose
# accepted-save channel has already failed by then; the pair only co-occurs on synthesized turns.
TurnClaimant.CODE_AUTHORING_CHURN,
- # Cascade position after the rescout steer and below the grounding exception; defers to the
- # exhausted churn halt above.
TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE,
# Loop detectors defer to every steering gate above them.
TurnClaimant.LOOP_DETECTED,
@@ -106,14 +88,10 @@ CLAIMANT_REASON_CODE_FAMILIES: dict[TurnClaimant, frozenset[str]] = {
TurnClaimant.GENUINELY_TERMINAL: GENUINELY_TERMINAL_BLOCKER_REASON_CODES,
TurnClaimant.OUTPUT_CONTRACT_ACTUATION: frozenset(),
TurnClaimant.CREDENTIAL_PRIORITY_CHURN: frozenset({"credential_priority_authoring_churn"}),
- TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT: frozenset(),
TurnClaimant.ACTUATION_OBLIGATION_FILL: frozenset(),
TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION: frozenset(),
TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION: frozenset(),
- TurnClaimant.CAPTURE_OBLIGATION_REOPEN: frozenset(),
TurnClaimant.CREDENTIAL_SCOUT_REOPEN: frozenset(),
- TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER: frozenset({UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE}),
- TurnClaimant.RECORDED_OUTCOME_GROUNDING: frozenset({RECORDED_OUTCOME_GROUNDING_REASON_CODE}),
TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE: frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE}),
TurnClaimant.CODE_AUTHORING_CHURN: frozenset({"code_authoring_guardrail_churn"}),
TurnClaimant.LOOP_DETECTED: frozenset(
@@ -137,11 +115,9 @@ _CLAIMANT_BY_REASON_CODE: dict[str, TurnClaimant] = {
# reachability and conflicts, but never suppresses a later gate or render on the same turn.
_TRANSIENT_CLAIMANTS = frozenset(
{
- TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT,
TurnClaimant.ACTUATION_OBLIGATION_FILL,
TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION,
TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION,
- TurnClaimant.CAPTURE_OBLIGATION_REOPEN,
TurnClaimant.CREDENTIAL_SCOUT_REOPEN,
}
)
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index 8e4d50a48..58d7162c4 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -40,8 +40,3 @@ class TurnOutcome(BaseModel):
copilot_repair_ceiling_hit: bool = False
copilot_pending_capability: str | None = None
copilot_turn_id: str | None = None
- # Structured summary of an edited-schema-incompatibility terminal outcome
- # (incompatible_paths, known_output_paths, next_actions, ...), persisted so a
- # later "what was the problem?" turn can report it. None unless the turn ended
- # on a schema-incompatibility halt.
- copilot_schema_incompatibility: dict[str, Any] | None = None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 6316637bb..3e633c20f 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -6334,7 +6334,6 @@ class UploadToS3Block(Block):
client = self.get_async_aws_client()
# is the file path a file or a directory?
if os.path.isdir(resolved_path):
- # get all files in the directory, if there are more than 25 files, we will not upload them
files = os.listdir(resolved_path)
if len(files) > MAX_UPLOAD_FILE_COUNT:
raise ValueError("Too many files in the directory, not uploading")
diff --git a/tests/unit/forge/sdk/artifact/storage/test_azure_storage.py b/tests/unit/forge/sdk/artifact/storage/test_azure_storage.py
index 6ecbfc93d..da9ff3317 100644
--- a/tests/unit/forge/sdk/artifact/storage/test_azure_storage.py
+++ b/tests/unit/forge/sdk/artifact/storage/test_azure_storage.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -7,7 +8,9 @@ import pytest
from skyvern.config import settings
from skyvern.forge.sdk.api.azure import StandardBlobTier
from skyvern.forge.sdk.api.real_azure import RealAsyncAzureStorageClient
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.artifact.storage.azure import AzureStorage
+from skyvern.forge.sdk.artifact.storage.base import SENSITIVE_SHARE_URL_EXPIRY_HOURS
# Test constants
TEST_CONTAINER = "test-azure-container"
@@ -313,3 +316,35 @@ class TestAzureStorageContentType:
call_kwargs = mock_container_client.upload_blob.call_args.kwargs
assert call_kwargs["content_settings"] is not None
assert call_kwargs["content_settings"].content_type == expected_content_type
+
+
+def make_artifact(uri: str, artifact_type: ArtifactType = ArtifactType.SCREENSHOT) -> Artifact:
+ return Artifact(
+ artifact_id="a_1",
+ artifact_type=artifact_type,
+ uri=uri,
+ organization_id=TEST_ORGANIZATION_ID,
+ created_at=datetime.utcnow(),
+ modified_at=datetime.utcnow(),
+ )
+
+
+class TestAzureShareLinkSensitiveCap:
+ """Sensitive artifact types get hour-capped SAS URLs (SKY-12527)."""
+
+ async def test_share_links_route_sensitive_types_to_capped_expiry(
+ self, azure_storage: AzureStorageForTests
+ ) -> None:
+ recording = make_artifact(f"azure://{TEST_CONTAINER}/rec.webm", artifact_type=ArtifactType.RECORDING)
+ download = make_artifact(f"azure://{TEST_CONTAINER}/file.pdf", artifact_type=ArtifactType.DOWNLOAD)
+
+ async def fake_sas(uris: list[str], expiry_hours: int = 24) -> list[str]:
+ return [f"{uri}?h={expiry_hours}" for uri in uris]
+
+ azure_storage.async_client.create_sas_urls = AsyncMock(side_effect=fake_sas)
+ urls = await azure_storage.get_share_links([recording, download])
+ assert urls == [
+ f"{recording.uri}?h={SENSITIVE_SHARE_URL_EXPIRY_HOURS}",
+ f"{download.uri}?h=24",
+ ]
diff --git a/tests/unit/forge/sdk/artifact/storage/test_gcs_storage.py b/tests/unit/forge/sdk/artifact/storage/test_gcs_storage.py
index b586d8fd5..c113a63bd 100644
--- a/tests/unit/forge/sdk/artifact/storage/test_gcs_storage.py
+++ b/tests/unit/forge/sdk/artifact/storage/test_gcs_storage.py
@@ -10,6 +10,7 @@ from skyvern.config import settings
from skyvern.forge.sdk.api.gcp import STORAGE_CLASS_STANDARD
from skyvern.forge.sdk.api.real_gcp import RealAsyncGcsStorageClient
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.storage.base import SENSITIVE_SHARE_URL_EXPIRY_HOURS
from skyvern.forge.sdk.artifact.storage.gcs import GcsStorage
TEST_BUCKET = "test-gcs-bucket"
@@ -17,10 +18,12 @@ TEST_ORGANIZATION_ID = "test-org-123"
TEST_BROWSER_SESSION_ID = "bs_test_123"
-def make_artifact(uri: str, artifact_id: str = "a_1") -> Artifact:
+def make_artifact(
+ uri: str, artifact_id: str = "a_1", artifact_type: ArtifactType = ArtifactType.SCREENSHOT
+) -> Artifact:
return Artifact(
artifact_id=artifact_id,
- artifact_type=ArtifactType.SCREENSHOT,
+ artifact_type=artifact_type,
uri=uri,
organization_id=TEST_ORGANIZATION_ID,
created_at=datetime.utcnow(),
@@ -358,3 +361,24 @@ class TestGcsStorageClientSigning:
assert urls is not None and len(urls) == 3
fake_creds.refresh.assert_called_once()
assert mock_blob.generate_signed_url.call_count == 3
+
+
+class TestGcsShareLinkSensitiveCap:
+ """Sensitive artifact types get hour-capped signed URLs (SKY-12527)."""
+
+ async def test_share_links_route_sensitive_types_to_capped_expiry(self, gcs_storage: GcsStorageForTests) -> None:
+ screenshot = make_artifact(f"gs://{TEST_BUCKET}/shot.png", artifact_id="a_shot")
+ download = make_artifact(
+ f"gs://{TEST_BUCKET}/file.pdf", artifact_id="a_file", artifact_type=ArtifactType.DOWNLOAD
+ )
+
+ async def fake_sign(uris: list[str], expiry_hours: int = 24) -> list[str]:
+ return [f"{uri}?h={expiry_hours}" for uri in uris]
+
+ gcs_storage.async_client.create_signed_urls = AsyncMock(side_effect=fake_sign)
+ urls = await gcs_storage.get_share_links([screenshot, download])
+ assert urls == [
+ f"{screenshot.uri}?h={SENSITIVE_SHARE_URL_EXPIRY_HOURS}",
+ f"{download.uri}?h=24",
+ ]
diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
index fa5b215f8..65ae370c0 100644
--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
@@ -19,6 +19,7 @@ from skyvern.config import settings
from skyvern.forge.sdk.api.aws import S3StorageClass, S3Uri
from skyvern.forge.sdk.artifact.manager import ArtifactManager
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
+from skyvern.forge.sdk.artifact.signing import SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
from skyvern.forge.sdk.artifact.storage.s3 import S3Storage
from skyvern.forge.sdk.db.id import generate_artifact_id
from skyvern.forge.sdk.models import Step
@@ -1262,3 +1263,60 @@ async def test_delete_browser_profile_hard_raises_soft_swallows() -> None:
storage.async_client.delete_file = AsyncMock()
await storage.delete_browser_profile("o", "bp", hard_delete=False)
assert storage.async_client.delete_file.await_args.kwargs["raise_on_error"] is False
+
+
+def _share_artifact(artifact_type: ArtifactType, uri: str) -> Artifact:
+ return Artifact(
+ artifact_id=generate_artifact_id(),
+ artifact_type=artifact_type,
+ uri=uri,
+ organization_id=TEST_ORGANIZATION_ID,
+ created_at=datetime.utcnow(),
+ modified_at=datetime.utcnow(),
+ )
+
+
+class TestS3ShareLinkSensitiveCap:
+ """Sensitive artifact types get capped presigned-URL TTLs (SKY-12527)."""
+
+ async def test_screenshot_share_link_uses_capped_expiry(self, s3_storage: S3Storage) -> None:
+ s3_storage.async_client.create_presigned_urls = AsyncMock(return_value=["https://s3/shot"])
+ artifact = _share_artifact(ArtifactType.SCREENSHOT_ACTION, f"s3://{TEST_BUCKET}/shot.png")
+ assert await s3_storage.get_share_link(artifact) == "https://s3/shot"
+ s3_storage.async_client.create_presigned_urls.assert_awaited_once_with(
+ [artifact.uri], expires_in=SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
+ )
+
+ async def test_download_share_link_keeps_default_expiry(self, s3_storage: S3Storage) -> None:
+ s3_storage.async_client.create_presigned_urls = AsyncMock(return_value=["https://s3/file"])
+ artifact = _share_artifact(ArtifactType.DOWNLOAD, f"s3://{TEST_BUCKET}/file.pdf")
+ assert await s3_storage.get_share_link(artifact) == "https://s3/file"
+ s3_storage.async_client.create_presigned_urls.assert_awaited_once_with([artifact.uri])
+
+ async def test_mixed_batch_preserves_order_and_routes_by_type(self, s3_storage: S3Storage) -> None:
+ download = _share_artifact(ArtifactType.DOWNLOAD, f"s3://{TEST_BUCKET}/file.pdf")
+ screenshot = _share_artifact(ArtifactType.SCREENSHOT_FINAL, f"s3://{TEST_BUCKET}/shot.png")
+ recording = _share_artifact(ArtifactType.RECORDING, f"s3://{TEST_BUCKET}/rec.webm")
+
+ async def fake_presign(uris: list[str], expires_in: int | None = None) -> list[str]:
+ suffix = "capped" if expires_in is not None else "default"
+ return [f"{uri}?{suffix}" for uri in uris]
+
+ s3_storage.async_client.create_presigned_urls = AsyncMock(side_effect=fake_presign)
+ urls = await s3_storage.get_share_links([download, screenshot, recording])
+ assert urls == [
+ f"{download.uri}?default",
+ f"{screenshot.uri}?capped",
+ f"{recording.uri}?capped",
+ ]
+
+ async def test_failed_sensitive_presign_fails_the_batch(self, s3_storage: S3Storage) -> None:
+ screenshot = _share_artifact(ArtifactType.SCREENSHOT_LLM, f"s3://{TEST_BUCKET}/shot.png")
+ download = _share_artifact(ArtifactType.DOWNLOAD, f"s3://{TEST_BUCKET}/file.pdf")
+
+ async def fake_presign(uris: list[str], expires_in: int | None = None) -> list[str] | None:
+ return None if expires_in is not None else [f"{uri}?ok" for uri in uris]
+
+ s3_storage.async_client.create_presigned_urls = AsyncMock(side_effect=fake_presign)
+ assert await s3_storage.get_share_links([screenshot, download]) is None
diff --git a/tests/unit/test_artifact_signed_url_endpoint.py b/tests/unit/test_artifact_signed_url_endpoint.py
index e3ef2d5a4..cd2720000 100644
--- a/tests/unit/test_artifact_signed_url_endpoint.py
+++ b/tests/unit/test_artifact_signed_url_endpoint.py
@@ -5,6 +5,7 @@ URL at the point of use, so consumers never depend on the long-lived URLs
embedded in earlier API responses.
"""
+import asyncio
import json
import time
from datetime import datetime, timezone
@@ -18,6 +19,7 @@ from skyvern.config import settings
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.artifact.signing import (
ARTIFACT_URL_ON_DEMAND_EXPIRY_SECONDS,
+ SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS,
)
from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.organizations import Organization
@@ -209,6 +211,57 @@ class TestMintedUrlAgainstContentEndpoint:
assert resp.status_code == 403
+class TestSensitiveArtifactEmbeddedUrl:
+ """Screenshots/recordings embedded in API and webhook payloads (SKY-12527).
+
+ Those URLs are not minted on demand, so they carry the capped TTL rather
+ than the org's — and stop authorizing anything once it lapses.
+ """
+
+ def _embedded_url_and_client(self, artifact: Artifact) -> tuple[str, TestClient, "AsyncMock"]:
+ with (
+ patch.object(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", _KEYRING_JSON),
+ patch.object(settings, "SKYVERN_BASE_URL", "http://testserver"),
+ patch("skyvern.forge.sdk.artifact.manager.app") as manager_app,
+ ):
+ from skyvern.forge.sdk.artifact.manager import ArtifactManager
+
+ manager = ArtifactManager()
+ manager_app.DATABASE.organizations.get_organization = AsyncMock(return_value=_make_org())
+ url = asyncio.run(manager.get_share_link(artifact))
+ return url, _make_client(), AsyncMock(return_value=artifact)
+
+ def _fetch(self, artifact: Artifact, url: str, client: TestClient, age_seconds: int) -> "TestClient.Response":
+ real_time = time.time
+ with (
+ patch.object(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", _KEYRING_JSON),
+ patch("skyvern.forge.sdk.routes.agent_protocol.app") as app_module,
+ patch("skyvern.forge.sdk.artifact.signing.time.time", new=lambda: real_time() + age_seconds),
+ ):
+ app_module.DATABASE.artifacts.get_artifact_by_id_no_org = AsyncMock(return_value=artifact)
+ app_module.ARTIFACT_MANAGER.retrieve_artifact = AsyncMock(return_value=b"secret-pixels")
+ return client.get(url.replace("http://testserver", ""))
+
+ def test_embedded_screenshot_url_expires_within_the_sensitive_window(self) -> None:
+ artifact = _make_artifact()
+ url, client, _ = self._embedded_url_and_client(artifact)
+ expiry = int(parse_qs(urlparse(url).query)["expiry"][0])
+ assert expiry - int(time.time()) <= SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
+
+ assert self._fetch(artifact, url, client, age_seconds=0).status_code == 200
+ replayed = self._fetch(artifact, url, client, age_seconds=SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS + 5)
+ assert replayed.status_code == 403
+
+ def test_expired_screenshot_url_is_not_revived_by_dropping_the_signature(self) -> None:
+ """Stripping sig/kid/expiry falls back to the org-auth path, which needs credentials."""
+ artifact = _make_artifact()
+ url, client, _ = self._embedded_url_and_client(artifact)
+ unsigned = url.replace("http://testserver", "").split("?")[0]
+ with patch.object(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", _KEYRING_JSON):
+ resp = client.get(unsigned)
+ assert resp.status_code in (401, 403)
+
+
class TestKeyringUnsetFallback:
def test_falls_back_to_storage_presigned_url(self) -> None:
"""Self-host deployments without a keyring still get a working URL."""
diff --git a/tests/unit/test_artifact_signing.py b/tests/unit/test_artifact_signing.py
index eadc7b049..f7c67808b 100644
--- a/tests/unit/test_artifact_signing.py
+++ b/tests/unit/test_artifact_signing.py
@@ -13,14 +13,19 @@ import time
import pytest
+from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.artifact.signing import (
ARTIFACT_URL_EXPIRY_SECONDS,
ARTIFACT_URL_EXPIRY_SECONDS_MAX,
ARTIFACT_URL_EXPIRY_SECONDS_MIN,
+ ARTIFACT_URL_ON_DEMAND_EXPIRY_SECONDS,
+ SENSITIVE_ARTIFACT_TYPES,
+ SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS,
ArtifactHmacKeyring,
HmacKeyEntry,
_canonical_string,
_hmac_b64,
+ artifact_url_expiry_seconds_for_type,
effective_artifact_url_expiry_seconds,
parse_keyring,
sign_artifact_url,
@@ -265,6 +270,56 @@ class TestEffectiveArtifactUrlExpirySeconds:
assert effective_artifact_url_expiry_seconds(ARTIFACT_URL_EXPIRY_SECONDS_MAX) == ARTIFACT_URL_EXPIRY_SECONDS_MAX
+# ---------------------------------------------------------------------------
+# artifact_url_expiry_seconds_for_type
+# ---------------------------------------------------------------------------
+
+
+class TestArtifactUrlExpirySecondsForType:
+ def test_sensitive_ttl_is_substantially_shorter_than_default(self) -> None:
+ assert SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS <= ARTIFACT_URL_EXPIRY_SECONDS / 4
+
+ def test_sensitive_ttl_is_not_below_the_per_org_floor(self) -> None:
+ """Webhook consumers cannot re-mint, so the cap must not dip under the sanctioned floor."""
+ assert SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS >= ARTIFACT_URL_EXPIRY_SECONDS_MIN
+
+ def test_screenshots_and_recordings_are_sensitive(self) -> None:
+ assert ArtifactType.RECORDING in SENSITIVE_ARTIFACT_TYPES
+ assert ArtifactType.SCREENSHOT_FINAL in SENSITIVE_ARTIFACT_TYPES
+ assert ArtifactType.SESSION_REPLAY in SENSITIVE_ARTIFACT_TYPES
+
+ @pytest.mark.parametrize("artifact_type", sorted(SENSITIVE_ARTIFACT_TYPES))
+ def test_sensitive_type_is_capped(self, artifact_type: ArtifactType) -> None:
+ assert (
+ artifact_url_expiry_seconds_for_type(artifact_type, ARTIFACT_URL_EXPIRY_SECONDS)
+ == SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
+ )
+
+ def test_sensitive_type_with_no_expiry_gets_the_cap(self) -> None:
+ assert (
+ artifact_url_expiry_seconds_for_type(ArtifactType.RECORDING, None) == SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS
+ )
+
+ def test_never_lengthens_an_already_shorter_ttl(self) -> None:
+ """The on-demand mint endpoint's 5-minute window must survive the cap."""
+ assert (
+ artifact_url_expiry_seconds_for_type(ArtifactType.RECORDING, ARTIFACT_URL_ON_DEMAND_EXPIRY_SECONDS)
+ == ARTIFACT_URL_ON_DEMAND_EXPIRY_SECONDS
+ )
+
+ def test_download_keeps_the_callers_ttl(self) -> None:
+ assert (
+ artifact_url_expiry_seconds_for_type(ArtifactType.DOWNLOAD, ARTIFACT_URL_EXPIRY_SECONDS)
+ == ARTIFACT_URL_EXPIRY_SECONDS
+ )
+
+ def test_non_sensitive_type_with_no_expiry_stays_none(self) -> None:
+ assert artifact_url_expiry_seconds_for_type(ArtifactType.DOWNLOAD, None) is None
+
+ def test_none_type_passes_through(self) -> None:
+ assert artifact_url_expiry_seconds_for_type(None, ARTIFACT_URL_EXPIRY_SECONDS) == ARTIFACT_URL_EXPIRY_SECONDS
+
+
# ---------------------------------------------------------------------------
# verify_artifact_signature
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_artifact_url_expiry_org_config.py b/tests/unit/test_artifact_url_expiry_org_config.py
index 3cd5715be..5a3d74789 100644
--- a/tests/unit/test_artifact_url_expiry_org_config.py
+++ b/tests/unit/test_artifact_url_expiry_org_config.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.artifact.signing import (
ARTIFACT_URL_EXPIRY_SECONDS,
ARTIFACT_URL_EXPIRY_SECONDS_MAX,
ARTIFACT_URL_EXPIRY_SECONDS_MIN,
+ SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS,
)
from skyvern.forge.sdk.routes.agent_protocol import _artifact_content_response_headers
from skyvern.forge.sdk.schemas.organizations import Organization
@@ -391,9 +392,23 @@ class TestGetShareLinksWithBundleSupportAlwaysUsesSignedContentUrl:
async def test_per_org_expiry_propagates_to_non_bundled(self) -> None:
"""The per-org TTL override must reach non-bundled artifacts too — they
used to bypass the resolver entirely on the presigned-URL path."""
- manager = ArtifactManager()
- artifacts = [_make_artifact("a_1")]
+ from skyvern.forge.sdk.artifact.models import ArtifactType
+
+ assert await self._minted_expiry(_make_artifact("a_1", artifact_type=ArtifactType.DOWNLOAD)) == 3 * 3600
+
+ @pytest.mark.asyncio
+ async def test_sensitive_artifacts_are_capped_below_the_per_org_expiry(self) -> None:
+ """Screenshots and recordings ignore a longer org TTL (SKY-12527)."""
+ from skyvern.forge.sdk.artifact.models import ArtifactType
+
+ for artifact_type in (ArtifactType.SCREENSHOT_FINAL, ArtifactType.RECORDING):
+ minted = await self._minted_expiry(_make_artifact("a_1", artifact_type=artifact_type))
+ assert minted == SENSITIVE_ARTIFACT_URL_EXPIRY_SECONDS, artifact_type
+ @staticmethod
+ async def _minted_expiry(artifact: "Artifact") -> int | None: # type: ignore[name-defined] # noqa: F821
+ """TTL handed to the URL signer for ``artifact``, with a 3-hour org override."""
+ manager = ArtifactManager()
resolve = AsyncMock(return_value=3 * 3600)
with (
patch.object(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", _DUMMY_KEYRING_JSON),
@@ -402,9 +417,9 @@ class TestGetShareLinksWithBundleSupportAlwaysUsesSignedContentUrl:
patch("skyvern.forge.sdk.artifact.manager.app") as app,
):
app.STORAGE.get_share_links = AsyncMock()
- await manager.get_share_links_with_bundle_support(artifacts)
+ await manager.get_share_links_with_bundle_support([artifact])
- assert bundle.call_args.kwargs["expiry_seconds"] == 3 * 3600
+ return bundle.call_args.kwargs["expiry_seconds"]
__all__ = [
diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.py
index 62b03ec93..d774b35ac 100644
--- a/tests/unit/test_copilot_agent_blocker_renderer.py
+++ b/tests/unit/test_copilot_agent_blocker_renderer.py
@@ -26,7 +26,6 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
BlockerKind,
CopilotToolBlockerSignal,
RecoveryHint,
- build_definition_contract_unsatisfied_blocker_signal,
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
@@ -554,21 +553,6 @@ def test_turn_halt_exit_keeps_halt_signal_when_context_signal_is_cleared() -> No
assert result.user_response == signal.user_facing_reason
-def test_definition_contract_halt_renders_sorted_parameter_names_without_internal_machinery() -> None:
- ctx = _ctx()
- signal = build_definition_contract_unsatisfied_blocker_signal(
- unresolved_parameter_keys=["service_address", "business_name", "contact_email"]
- )
- halt = TurnHalt(kind=TurnHaltKind.DEFINITION_CONTRACT_UNSATISFIED, blocker_signal=signal)
-
- result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
-
- assert result.user_response == signal.user_facing_reason
- assert result.user_response.index("business_name") < result.user_response.index("contact_email")
- assert result.user_response.index("contact_email") < result.user_response.index("service_address")
- assert "update_and_run_blocks" not in result.user_response
-
-
def test_turn_halt_exit_renders_code_authoring_churn_reason() -> None:
ctx = _ctx()
signal = _signal(
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 8fae45352..3a5a5d177 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -97,7 +97,6 @@ from skyvern.forge.sdk.copilot.request_policy import (
)
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOTS_PROMPT_NAME
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
-from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.completion import (
_authored_output_contract_criteria,
_carry_degraded_ids,
@@ -2271,71 +2270,6 @@ class TestBlockGoalMainGoal:
class TestRuntimeBlockGoalPersistenceBoundary:
- @pytest.mark.asyncio
- async def test_update_and_run_blocks_metadata_repair_context_preflight_blocks_before_update(
- self, monkeypatch
- ) -> None:
- workflow_yaml = """
-title: Test workflow
-workflow_definition:
- parameters: []
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
-"""
- repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id", "output.flags"],
- required_extraction_schema_paths=["output.record_id", "output.flags"],
- required_code_return_paths=["output.record_id", "output.flags"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- ctx = _ctx(
- block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
- last_code_authoring_repair_context=repair_context,
- )
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(
- tools_module,
- "_tool_loop_error",
- lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("loop check should not run")),
- )
- monkeypatch.setattr(
- tools_module,
- "_update_workflow",
- AsyncMock(side_effect=AssertionError("_update_workflow should not run")),
- )
- record_calls: list[tuple[str, dict[str, object]]] = []
- monkeypatch.setattr(
- tools_module,
- "record_tool_step_result_for_ctx",
- lambda _ctx, tool, _args, result: record_calls.append((tool, result)),
- )
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["extract_entry_output"]}),
- )
-
- parsed = json.loads(result)
- assert parsed["ok"] is False
- assert "cannot attempt a run" in parsed["error"]
- assert parsed["data"]["reason_code"] == "metadata_contract_required_before_run"
- assert parsed["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
- "output.flags",
- "output.record_id",
- ]
- assert parsed["data"]["metadata_repair_contract"]["block_label"] == "extract_entry_output"
- assert record_calls
- assert record_calls[0][0] == "update_and_run_blocks"
- assert record_calls[0][1]["data"]["reason_code"] == "metadata_contract_required_before_run"
-
@pytest.mark.asyncio
async def test_update_and_run_blocks_metadata_contract_scaffold_reaches_update_workflow(self, monkeypatch) -> None:
workflow_yaml = """
@@ -2392,92 +2326,6 @@ workflow_definition:
]
assert captured_metadata[-1]["formation_prepared"] is True
- @pytest.mark.asyncio
- async def test_update_and_run_blocks_typed_advisory_static_return_gap_reaches_run(self, monkeypatch) -> None:
- workflow_yaml = """
-title: Test workflow
-workflow_definition:
- parameters: []
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return "not a structured output"
-"""
- required_paths = {"output.record_id"}
- schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
- repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- ctx = _ctx(
- block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
- last_code_authoring_repair_context=repair_context,
- )
- ctx.turn_id = "metadata-contract-run-preflight"
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
- workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
- captured: dict[str, object] = {}
-
- async def fake_update_workflow(payload, update_ctx, **_kwargs):
- captured["update_called"] = True
- captured["metadata"] = payload["code_artifact_metadata"]
- update_ctx.workflow_yaml = payload["workflow_yaml"]
- update_ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
- return {"ok": True, "data": {"block_count": 1}}
-
- async def fake_run_blocks(params, _ctx, **_kwargs):
- captured["run_called"] = True
- captured["run_params"] = params
- return {
- "ok": True,
- "data": {
- "workflow_run_id": "wr-1",
- "overall_status": "completed",
- "blocks": [{"label": "extract_entry_output", "status": "completed"}],
- },
- }
-
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
- monkeypatch.setattr(
- tools_module, "_plan_frontier", lambda *args: (["extract_entry_output"], {}, "extract_entry_output")
- )
- monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
- monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": workflow_yaml,
- "block_labels": ["extract_entry_output"],
- }
- ),
- )
-
- parsed = json.loads(result)
- assert parsed["ok"] is True
- assert captured["update_called"] is True
- assert captured["run_called"] is True
- update_metadata = captured["metadata"]
- assert isinstance(update_metadata, list)
- assert update_metadata[0]["block_label"] == "extract_entry_output"
- assert update_metadata[0]["claimed_outcomes"][0]["goal_value_paths"] == ["output.record_id"]
- assert update_metadata[0]["claimed_outcomes"][0]["extraction_schema"] == schema
-
@pytest.mark.asyncio
async def test_update_and_run_blocks_persists_clean_yaml(self, monkeypatch) -> None:
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
@@ -2886,10 +2734,10 @@ class TestTranslateToAgentResultGating:
assert agent_result.updated_workflow is None
assert "runtime self-heal" in agent_result.user_response.lower()
- def test_inline_replace_workflow_rejects_stale_block_metadata(self, monkeypatch) -> None:
- # Inline REPLACE_WORKFLOW bypasses _update_workflow, so it must also
- # reject a corrected workflow whose labels/titles still describe the
- # prior subject.
+ def test_inline_replace_workflow_steers_on_stale_block_metadata(self, monkeypatch) -> None:
+ # A label still describing the prior subject is authoring quality, not disclosure, so the
+ # draft is kept and reported on rather than thrown away. The test credit is cleared, so the
+ # kept draft still cannot be surfaced as a verified proposal.
process_mock = AsyncMock(return_value=SimpleNamespace(name="new"))
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
@@ -2929,12 +2777,45 @@ workflow_definition:
)
)
- process_mock.assert_not_called()
+ process_mock.assert_called_once()
assert "corrected block metadata still appears stale" in agent_result.user_response
+ assert ctx.last_test_ok is None
+ assert agent_result.workflow_yaml is None
+
+ def test_inline_replace_workflow_rejects_unsafe_code_block(self, monkeypatch) -> None:
+ # This surface persists a draft without _update_workflow, so it carries the same
+ # code_safety block. Unsafe in-page code on a page holding a filled credential is
+ # the one thing a later test-run cannot undo.
+ process_mock = AsyncMock(return_value=SimpleNamespace(name="new"))
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
+
+ submitted_yaml = """
+title: Registry lookup
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ import requests
+ requests.get("https://example.com")
+"""
+ ctx = _ctx(workflow_yaml="", last_workflow_yaml="")
+ result = _fake_run_result(
+ {"type": "REPLACE_WORKFLOW", "user_response": "Here you go.", "workflow_yaml": submitted_yaml}
+ )
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+ )
+
+ process_mock.assert_not_called()
assert agent_result.updated_workflow is None
assert agent_result.workflow_yaml is None
- def test_inline_replace_workflow_rejects_page_dependent_blocks_without_inspection(self, monkeypatch) -> None:
+ def test_inline_replace_workflow_steers_on_page_dependent_blocks_without_inspection(self, monkeypatch) -> None:
+ # Missing page evidence is what the test-run settles, so the draft is kept and reported on.
+ # The turn needs update authority or the inline REPLACE is downgraded before this gate runs.
process_mock = AsyncMock(return_value=SimpleNamespace(name="new"))
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
@@ -2953,7 +2834,10 @@ workflow_definition:
ctx = _ctx(
workflow_yaml="",
build_phase=BuildPhase.COMPOSING,
- turn_intent=TurnIntent(mode=TurnIntentMode.BUILD),
+ turn_intent=TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True),
+ ),
composition_page_evidence=None,
)
result = _fake_run_result(
@@ -2966,13 +2850,11 @@ workflow_definition:
)
)
- process_mock.assert_not_called()
- # The reject note must be product language, never the gate's
- # agent-directed tool instruction.
+ process_mock.assert_called_once()
+ # The note must be product language, never the gate's agent-directed tool instruction.
assert "(Note:" in agent_result.user_response
assert "inspect_page_for_composition" not in agent_result.user_response
- assert agent_result.updated_workflow is None
- assert agent_result.workflow_yaml is None
+ assert ctx.last_test_ok is None
def test_code_only_inline_replace_workflow_rejects_native_browser_block(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot.output_policy import OutputPolicyVerdict
@@ -3006,6 +2888,50 @@ workflow_definition:
assert agent_result.updated_workflow is None
assert agent_result.workflow_yaml is None
+ def test_inline_replace_verdict_steers_a_reason_the_tool_seam_demotes(self, monkeypatch) -> None:
+ # This seam persists a draft, so it is graded like the update_workflow tool body. Grading it
+ # like a final reply walled drafts on reasons the tool seam only steers on, which is how the
+ # test-run signal was lost on this path.
+ from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, OutputPolicyVerdict
+
+ monkeypatch.setattr(
+ agent_module,
+ "evaluate_output_policy",
+ lambda **kwargs: OutputPolicyVerdict(reason_codes=[OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS]),
+ )
+
+ _, raw_verdict, author_time_verdict = agent_module._inline_replace_workflow_credential_verdict(
+ _ctx(), {"workflow_yaml": "title: Example\n"}, "REPLACE_WORKFLOW", "Here you go."
+ )
+
+ assert author_time_verdict.allowed is True
+ assert list(author_time_verdict.reason_codes) == []
+ # The raw verdict is what diagnostics report, so demotion must not consume it.
+ assert list(raw_verdict.reason_codes) == [OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS]
+
+ def test_inline_replace_verdict_still_blocks_a_credential_reason_co_firing_with_a_demoted_one(
+ self, monkeypatch
+ ) -> None:
+ from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, OutputPolicyVerdict
+
+ monkeypatch.setattr(
+ agent_module,
+ "evaluate_output_policy",
+ lambda **kwargs: OutputPolicyVerdict(
+ reason_codes=[
+ OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
+ OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
+ ]
+ ),
+ )
+
+ _, _, author_time_verdict = agent_module._inline_replace_workflow_credential_verdict(
+ _ctx(), {"workflow_yaml": "title: Example\n"}, "REPLACE_WORKFLOW", "Here you go."
+ )
+
+ assert author_time_verdict.allowed is False
+ assert list(author_time_verdict.reason_codes) == [OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED]
+
def test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:
tested_wf = SimpleNamespace(name="tested")
@@ -5897,80 +5823,6 @@ class TestRunBlocksCredentialApproval:
prepare_workflow.assert_not_called()
execute_workflow.assert_not_called()
- @pytest.mark.asyncio
- async def test_update_and_run_blocks_rejects_unapproved_credential_at_shared_run_seam(self, monkeypatch) -> None:
- from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
-
- workflow = self._workflow("cred_unapproved")
- database = self._db(
- workflow=workflow,
- credentials=[SimpleNamespace(credential_id="cred_unapproved")],
- )
- prepare_workflow = AsyncMock(side_effect=AssertionError("prepare_workflow called"))
- execute_workflow = AsyncMock(side_effect=AssertionError("execute_workflow called"))
- monkeypatch.setattr(run_execution_module.app, "DATABASE", database)
- monkeypatch.setattr(
- run_execution_module.app,
- "WORKFLOW_SERVICE",
- SimpleNamespace(prepare_workflow=prepare_workflow, execute_workflow=execute_workflow),
- )
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["login"], {}, "login"))
- monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
- monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))
- monkeypatch.setattr(
- tools_module,
- "_tool_visible_result_after_completion_verification",
- lambda _ctx, result, _completion: result,
- )
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
-
- async def fake_update_workflow(*args: object, **kwargs: object) -> dict[str, object]:
- ctx = args[1]
- ctx.staged_workflow = workflow
- ctx.last_workflow = workflow
- ctx.last_update_block_count = 1
- return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}
-
- monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(
- context=_ctx(request_policy=RequestPolicy(resolved_credentials=[])),
- tool_name="update_and_run_blocks",
- ),
- json.dumps(
- {
- "workflow_yaml": """
-workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: credential_id
- key: login_credentials
- default_value: cred_unapproved
- blocks:
- - block_type: login
- label: login
-""",
- "block_labels": ["login"],
- "parameters": {},
- }
- ),
- )
-
- payload = json.loads(result)
- assert payload["ok"] is False
- assert "unapproved_credential_reference" in payload["error"]
- database.credentials.get_credentials_by_ids.assert_not_called()
- database.organizations.get_organization.assert_not_called()
- prepare_workflow.assert_not_called()
- execute_workflow.assert_not_called()
-
@pytest.mark.asyncio
async def test_resolved_credential_reaches_existing_run_validation_path(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug
diff --git a/tests/unit/test_copilot_author_time_block.py b/tests/unit/test_copilot_author_time_block.py
new file mode 100644
index 000000000..e6395455f
--- /dev/null
+++ b/tests/unit/test_copilot_author_time_block.py
@@ -0,0 +1,515 @@
+"""The author-time refusal capability: three hard blocks, one decision point.
+
+OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import inspect
+import json
+import re
+import textwrap
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge.sdk.copilot import blocker_signal as blocker_signal_module
+from skyvern.forge.sdk.copilot import build_phase as build_phase_module
+from skyvern.forge.sdk.copilot import failure_tracking as failure_tracking_module
+from skyvern.forge.sdk.copilot import run_outcome as run_outcome_module
+from skyvern.forge.sdk.copilot import tools as tools_module
+from skyvern.forge.sdk.copilot.author_time_block import (
+ AUTHOR_TIME_HARD_BLOCKS,
+ BANNED_BLOCKS_BLOCK_ID,
+ CODE_SAFETY_BLOCK_ID,
+ CREDENTIAL_SCOUT_BLOCK_ID,
+ AuthorTimeBlock,
+)
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, OutputPolicyVerdict
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.schema_incompatibility import (
+ SCHEMA_INCOMPATIBILITY_REASON_CODE,
+ SchemaIncompatibility,
+)
+from skyvern.forge.sdk.copilot.tools import _update_workflow
+from skyvern.forge.sdk.copilot.tools import blockers as blockers_module
+from skyvern.forge.sdk.copilot.tools import guardrails as guardrails_module
+from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
+from skyvern.forge.sdk.copilot.tools.guardrails import _workflow_yaml_output_policy_guardrail
+
+_BUILD_PHASE_SOURCE = inspect.getsource(build_phase_module)
+_BLOCKERS_SOURCE = inspect.getsource(blockers_module)
+_BLOCKER_SIGNAL_SOURCE = inspect.getsource(blocker_signal_module)
+
+
+def _yaml(body: str) -> str:
+ return textwrap.dedent(body).strip() + "\n"
+
+
+_SAFE_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.goto("https://example.com/search")
+ await page.locator("#search").fill("widget")
+ """
+)
+
+_UNSAFE_CODE_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ import requests
+ requests.get("https://example.com")
+ """
+)
+
+_BANNED_BLOCK_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: task
+ label: do_the_thing
+ prompt: Find the widget
+ """
+)
+
+_CONFLICT_MARKER_YAML = (
+ "<<<<<<< HEAD\n"
+ "title: Registry lookup\n"
+ "=======\n"
+ "title: Registry lookup v2\n"
+ ">>>>>>> theirs\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: search_registry\n"
+ " code: |\n"
+ ' await page.goto("https://example.com/search")\n'
+)
+
+_CREDENTIAL_LOGIN_YAML = _yaml(
+ """
+ title: Portal login
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: portal_login
+ code: |
+ await page.goto("https://example.com/login")
+ await page.locator("#submit").click()
+ """
+)
+
+
+def _ctx() -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#search-submit",
+ "source_url": "https://example.com/search",
+ "trajectory_index": 0,
+ }
+ ]
+ return ctx
+
+
+def _stub_persist(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label="search_registry")]),
+ proxy_location=None,
+ )
+
+ async def _fake_get_prior_workflow(_ctx: CopilotContext) -> None:
+ return None
+
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", _fake_process_workflow_yaml)
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", _fake_get_prior_workflow)
+
+
+def test_hard_block_constant_has_exactly_three_members() -> None:
+ assert AUTHOR_TIME_HARD_BLOCKS == frozenset(
+ {CODE_SAFETY_BLOCK_ID, CREDENTIAL_SCOUT_BLOCK_ID, BANNED_BLOCKS_BLOCK_ID}
+ )
+ assert len(AUTHOR_TIME_HARD_BLOCKS) == 3
+ assert sorted(AUTHOR_TIME_HARD_BLOCKS) == ["banned_blocks", "code_safety", "credential_scout"]
+
+
+ "block_id",
+ ["definition_contract_unsatisfied", "metadata_reject", "schema_incompatibility", "", "CODE_SAFETY"],
+)
+def test_constructing_a_block_outside_the_constant_raises(block_id: str) -> None:
+ with pytest.raises(ValueError):
+ AuthorTimeBlock(block_id=block_id, error="a new validator wants to refuse")
+
+
+def test_each_retained_identity_constructs() -> None:
+ for block_id in AUTHOR_TIME_HARD_BLOCKS:
+ assert AuthorTimeBlock(block_id=block_id, error="x").block_id == block_id
+
+
+def test_the_identity_cannot_be_swapped_after_construction() -> None:
+ """The constructor check only bounds refusal if the identity is immutable afterwards —
+ otherwise a caller constructs a declared block and reassigns its way out of the set."""
+ block = AuthorTimeBlock(block_id=CODE_SAFETY_BLOCK_ID, error="x")
+
+ with pytest.raises(dataclasses.FrozenInstanceError):
+ block.block_id = "newly_invented_gate" # type: ignore[misc]
+
+
+async def test_synthetic_validator_finding_cannot_block_the_persist(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A validator added next month reports a finding in the most refusal-shaped way it can —
+ a violation list, a truthy error string, a rejection dict — and the draft still persists."""
+ _stub_persist(monkeypatch)
+ ctx = _ctx()
+
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_maybe_impose_synthesized_code_block",
+ lambda yaml_text, *_a, **_k: workflow_update_module._SynthesizedCodeImpositionResult(
+ workflow_yaml=yaml_text,
+ violations=["REFUSED: imposition violated. ok: False. allowed=False."],
+ ),
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_apply_scouted_typed_default_promotions",
+ lambda yaml_text, *_a, **_k: (yaml_text, ["REFUSED: typed default violation."], []),
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "merge_schema_incompatibilities",
+ lambda *_a, **_k: None,
+ )
+
+ result = await _update_workflow({"workflow_yaml": _SAFE_YAML}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.has_staged_proposal is True
+ assert ctx.staged_workflow is not None
+ assert ctx.blocker_signal is None
+ assert ctx.turn_halt is None
+
+
+async def test_code_safety_still_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_persist(monkeypatch)
+ ctx = _ctx()
+
+ result = await _update_workflow({"workflow_yaml": _UNSAFE_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert result["block_id"] == CODE_SAFETY_BLOCK_ID
+ assert ctx.has_staged_proposal is False
+
+
+async def test_banned_block_type_still_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_persist(monkeypatch)
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+
+ result = await _update_workflow({"workflow_yaml": _BANNED_BLOCK_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert result["block_id"] == BANNED_BLOCKS_BLOCK_ID
+ assert ctx.has_staged_proposal is False
+
+
+async def test_credential_scout_gate_still_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_persist(monkeypatch)
+ ctx = _ctx()
+ ctx.scout_trajectory = []
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_credentialed_code_block_scout_gate_errors",
+ lambda *_a, **_k: ["Scout the login form before authoring a credential fill."],
+ )
+
+ result = await _update_workflow({"workflow_yaml": _CREDENTIAL_LOGIN_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert result["block_id"] == CREDENTIAL_SCOUT_BLOCK_ID
+ assert ctx.has_staged_proposal is False
+
+
+def test_residual_raw_credential_fill_is_admitted_on_an_allowed_verdict(monkeypatch: pytest.MonkeyPatch) -> None:
+ workflow_yaml = _yaml(
+ """
+ title: Portal login
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: portal_login
+ code: |
+ await page.locator("#username").fill(parameters["operator"])
+ """
+ )
+ tool_context = SimpleNamespace(
+ tool_name="update_workflow",
+ tool_call_id="call_1",
+ tool_arguments=json.dumps({"workflow_yaml": workflow_yaml}),
+ context=SimpleNamespace(request_policy=RequestPolicy()),
+ )
+ rebind = SimpleNamespace(
+ changed=False,
+ workflow_yaml=workflow_yaml,
+ residual_selectors=("#username",),
+ authored=(),
+ skips=(),
+ )
+ monkeypatch.setattr(
+ guardrails_module,
+ "_rebind_scouted_credential_literals_in_place",
+ lambda *_a, **_k: rebind,
+ )
+
+ output = _workflow_yaml_output_policy_guardrail(SimpleNamespace(context=tool_context)) # type: ignore[arg-type]
+
+ assert output.output_info["allowed"] is True
+ assert output.output_info["residual_raw_credential_fill_selectors"] == ["#username"]
+ assert "block_id" not in output.output_info
+
+
+def test_tool_input_guardrail_allows_a_non_credential_verdict(monkeypatch: pytest.MonkeyPatch) -> None:
+ tool_context = SimpleNamespace(
+ tool_name="update_workflow",
+ tool_call_id="call_1",
+ tool_arguments=json.dumps({"workflow_yaml": _SAFE_YAML}),
+ context=SimpleNamespace(request_policy=RequestPolicy()),
+ )
+ monkeypatch.setattr(
+ guardrails_module,
+ "evaluate_output_policy",
+ lambda **_k: OutputPolicyVerdict(reason_codes=[OutputPolicyReason.WORKFLOW_YAML_IN_REPLY]),
+ )
+
+ output = _workflow_yaml_output_policy_guardrail(SimpleNamespace(context=tool_context)) # type: ignore[arg-type]
+
+ assert output.output_info["allowed"] is True
+ assert "block_id" not in output.output_info
+
+
+async def test_unparseable_yaml_is_an_honest_error_not_a_block(monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_persist(monkeypatch)
+ ctx = _ctx()
+
+ result = await _update_workflow({"workflow_yaml": _CONFLICT_MARKER_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "block_id" not in result
+ assert ctx.blocker_signal is None
+ assert ctx.turn_halt is None
+ assert ctx.code_authoring_guardrail_reject_count == 0
+
+
+def test_author_time_findings_label_a_persisted_draft_without_refusing_it() -> None:
+ """The finding bucket is the whole replacement for the deleted refusals: three reason codes,
+ each carrying a summary, and an empty list when the draft is clean."""
+ assert (
+ workflow_update_module._author_time_findings(
+ schema_incompatibility=None, metadata_violations=[], imposition_violations=[]
+ )
+ == []
+ )
+
+ incompat = SchemaIncompatibility(
+ block_label="extract_registry_row",
+ incompatible_paths=["output.serial_number"],
+ known_output_paths=["output.title"],
+ )
+ findings = workflow_update_module._author_time_findings(
+ schema_incompatibility=incompat,
+ metadata_violations=["missing declared goal for extract_registry_row"],
+ imposition_violations=["could not impose the synthesized block"],
+ )
+
+ assert [finding["reason_code"] for finding in findings] == [
+ SCHEMA_INCOMPATIBILITY_REASON_CODE,
+ "code_artifact_metadata_incomplete",
+ "synthesized_code_block_not_imposed",
+ ]
+ assert all(finding["summary"] for finding in findings)
+ assert findings[0]["schema_incompatibility"]["incompatible_paths"] == ["output.serial_number"]
+ # A finding is a label, never an identity that could refuse.
+ for finding in findings:
+ assert finding["reason_code"] not in AUTHOR_TIME_HARD_BLOCKS
+
+
+def test_a_new_build_phase_gate_cannot_wall_the_turn() -> None:
+ """The third plane's ratchet. `_phase_blocker_signal` ends turns without ever constructing an
+ AuthorTimeBlock, so its vocabulary is bounded at its own reviewed constant."""
+ synthetic = CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text="Stop: this phase may not author.",
+ user_facing_reason="I can't do that yet.",
+ recovery_hint="ask_user_clarifying",
+ internal_reason_code="build_phase_newly_invented_gate",
+ blocked_tool="update_workflow",
+ )
+
+ with pytest.raises(ValueError, match="not a declared build-phase refusal"):
+ build_phase_module._declared_phase_refusal(synthetic)
+
+
+def test_the_three_planes_declare_disjoint_vocabularies() -> None:
+ assert not build_phase_module.BUILD_PHASE_REFUSAL_REASON_CODES & blockers_module.LOOP_PLANE_REFUSAL_REASON_CODES
+ assert not build_phase_module.BUILD_PHASE_REFUSAL_REASON_CODES & AUTHOR_TIME_HARD_BLOCKS
+
+
+def test_every_build_phase_refusal_it_can_emit_is_declared() -> None:
+ """The inverse ratchet: renaming a reason code in the builder would otherwise turn a working
+ refusal into an unhandled ValueError mid-turn."""
+ emitted = set(re.findall(r'internal_reason_code="(build_phase_[a-z_]+)"', _BUILD_PHASE_SOURCE))
+
+ assert emitted == set(build_phase_module.BUILD_PHASE_REFUSAL_REASON_CODES)
+
+
+def test_every_loop_plane_refusal_it_can_emit_is_declared() -> None:
+ """Same inverse ratchet for the loop plane: every reason code that can reach
+ `_emit_loop_plane_refusal` has to be declared, or it raises where a refusal used to work.
+
+ Every code the loop plane actually carries is a named constant defined outside `blockers.py`,
+ so matching literals in that one module asserts nothing about them — this compares the
+ constants themselves, and fails if a rename moves a value out of the declared set.
+ """
+ carried = {
+ blocker_signal_module.SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
+ blocker_signal_module.DISCOVERY_EXHAUSTED_NO_ENTRY_URL_REASON_CODE,
+ failure_tracking_module.ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ run_outcome_module.TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
+ blockers_module._POST_BUDGET_CHALLENGE_RESULT_EVIDENCE_REASON,
+ blockers_module._POST_BUDGET_CHALLENGE_BLOCKER_REASON,
+ }
+ # The loop-detection branch names its codes inline rather than via constants.
+ carried |= set(re.findall(r'internal = "(loop_detected_[a-z_]+)"', _BLOCKER_SIGNAL_SOURCE))
+ carried |= set(re.findall(r'internal_reason_code="([a-z_]+)"', _BLOCKERS_SOURCE))
+
+ assert carried, "found no loop-plane reason codes — the collection above has gone stale"
+ assert carried - blockers_module.LOOP_PLANE_REFUSAL_REASON_CODES == set()
+
+
+def test_phase_blocker_signal_is_not_an_author_time_block() -> None:
+ """Post-run and phase/authority signals live on the runtime plane; forcing them into the
+ three author-time identities would re-admit refusal to a surface that authors nothing."""
+ ctx = _ctx()
+ ctx.build_phase = build_phase_module.BuildPhase.DISCOVERING
+
+ signal = build_phase_module._phase_blocker_signal(ctx, "update_workflow")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert not isinstance(signal, AuthorTimeBlock)
+ assert not hasattr(signal, "block_id")
+
+
+def test_a_new_loop_plane_check_cannot_wall_the_turn() -> None:
+ """The ratchet's other half. `_tool_loop_error` runs before the authoring seam and sees the
+ submitted draft, so an undeclared refusal minted there would wall a turn while never
+ constructing an AuthorTimeBlock."""
+ synthetic = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="Stop: the draft omits an observation ref.",
+ user_facing_reason="I couldn't save that workflow.",
+ recovery_hint="report_blocker_to_user",
+ renders_final_reply=True,
+ internal_reason_code="tool_error_draft_missing_observation_ref",
+ blocked_tool="update_workflow",
+ )
+
+ with pytest.raises(ValueError, match="not a declared loop-plane refusal"):
+ blockers_module._emit_loop_plane_refusal(_ctx(), synthetic)
+
+
+async def test_findings_on_a_persisted_draft_survive_the_combined_update_and_run_tool(monkeypatch) -> None:
+ """The findings bucket is what replaced the deleted refusals, so it is only worth anything if
+ it reaches the model on the tool the agent actually authors with."""
+ ctx = _ctx()
+
+ async def fake_update_workflow(_payload, _ctx, **_kwargs):
+ return {
+ "ok": True,
+ "data": {"message": "Workflow updated successfully.", "findings": [{"reason_code": "r", "summary": "s"}]},
+ }
+
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": _SAFE_YAML, "block_labels": ["search_registry"]}),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is True
+ assert parsed["data"]["findings"] == [{"reason_code": "r", "summary": "s"}]
+
+
+async def test_a_credential_deferred_draft_persists_instead_of_being_redirected(monkeypatch) -> None:
+ """Credentials that do not exist yet are not a disclosure, so the draft saves and says so
+ rather than failing the save tool to make the model retry through a sibling tool."""
+ ctx = _ctx()
+
+ async def fake_update_workflow(_payload, _ctx, **kwargs):
+ assert kwargs["allow_missing_credentials"] is True
+ return {"ok": True, "data": {"block_count": 1}}
+
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_request_policy_allows_credential_deferred_draft", lambda *args: True)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_workflow_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_workflow"),
+ json.dumps({"workflow_yaml": _SAFE_YAML}),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is True
+ assert parsed["data"]["skipped_run"] is True
+ assert parsed["data"]["skip_reason"] == "workflow_credential_inputs_unbound"
+ assert "credentials aren't set up yet" in parsed["data"]["message"]
+ assert ctx.last_run_skipped_unbound_credentials is True
+ assert ctx.blocker_signal is None
+
+
+def test_every_declared_loop_plane_refusal_is_runtime_not_draft_content() -> None:
+ """Each entry is a reviewed decision. `tool_error_challenge_gated_submit_disabled` is the one
+ that reads the submitted draft (its proxy value); SKY-13044 owns dispositioning it."""
+ assert "tool_error_challenge_gated_submit_disabled" in blockers_module.LOOP_PLANE_REFUSAL_REASON_CODES
+ assert not blockers_module.LOOP_PLANE_REFUSAL_REASON_CODES & AUTHOR_TIME_HARD_BLOCKS
diff --git a/tests/unit/test_copilot_author_time_gate_log_only.py b/tests/unit/test_copilot_author_time_gate_log_only.py
deleted file mode 100644
index e79e1b737..000000000
--- a/tests/unit/test_copilot_author_time_gate_log_only.py
+++ /dev/null
@@ -1,646 +0,0 @@
-from __future__ import annotations
-
-import copy
-import json
-import pathlib
-import re
-from types import SimpleNamespace
-from unittest.mock import AsyncMock
-
-import pytest
-from structlog.testing import capture_logs
-
-from skyvern.config import settings
-from skyvern.forge.sdk.copilot import agent, runtime
-from skyvern.forge.sdk.copilot.build_test_outcome import (
- RecordedBuildTestOutcome,
- RecordedOutcomeBindingConstraint,
- RecordedOutcomeGroundingRequirement,
-)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
-from skyvern.forge.sdk.copilot.runtime import (
- AUTHOR_TIME_GATE_LOG_ONLY_IDS,
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
- OUTPUT_CONTRACT_ACTUATION_GATE_ID,
- RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
- SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- cache_copilot_author_time_gate_log_only_ids,
- copilot_author_time_gate_log_only_enabled,
- record_author_time_gate_ablation_event,
-)
-from skyvern.forge.sdk.copilot.tools import workflow_update as wu
-from tests.unit.copilot_test_helpers import make_copilot_ctx
-from tests.unit.test_copilot_code_artifact_metadata_violations import _valid_metadata
-
-_GOAL_PATH_PLACEHOLDER = "<fill: output JSON path(s) carrying requested goal values>"
-
-_UNREFERENCED_DEFINITION_YAML = """\
-title: Submit reusable request
-workflow_definition:
- parameters:
- - {parameter_type: workflow, key: account_name, workflow_parameter_type: string}
- blocks:
- - block_type: code
- label: submit_request
- parameter_keys: []
- code: |
- await page.locator("#submit").click()
-"""
-
-
-def _definition_ctx() -> CopilotContext:
- ctx = make_copilot_ctx()
- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="definition-1",
- outcome="The workflow uses account name as a reusable input.",
- level="definition",
- output_path="workflow.parameters",
- )
- ]
- )
- return ctx
-
-
-def _stub_successful_update(monkeypatch: pytest.MonkeyPatch) -> None:
- async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
- return SimpleNamespace(
- workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label="submit_request")]),
- proxy_location=None,
- )
-
- async def fake_get_prior_workflow(_ctx: object) -> None:
- return None
-
- monkeypatch.setattr(wu, "_process_workflow_yaml", fake_process_workflow_yaml)
- monkeypatch.setattr(wu, "_get_prior_workflow", fake_get_prior_workflow)
- monkeypatch.setattr(wu, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
-
-
[email protected](autouse=True)
-def _cloud_environment(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "ENV", "prod")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", False)
-
-
-def test_oss_policy_filters_ineligible_ids_and_preserves_legacy_blanket(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- ctx = make_copilot_ctx()
- with capture_logs() as logs:
- cache_copilot_author_time_gate_log_only_ids(
- ctx,
- frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID, "credential_reference_presence"}),
- )
-
- assert ctx.author_time_gate_log_only_ids == frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID})
- assert copilot_author_time_gate_log_only_enabled(ctx, DEFINITION_CONTRACT_UNSATISFIED_GATE_ID) is True
- assert copilot_author_time_gate_log_only_enabled(ctx, "credential_reference_presence") is False
- assert any(
- log["event"] == "copilot_gate_log_only_ineligible" and log["gate_id"] == "credential_reference_presence"
- for log in logs
- )
-
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- assert copilot_author_time_gate_log_only_enabled(ctx, "output_contract_actuation") is True
- assert copilot_author_time_gate_log_only_enabled(ctx, DEFINITION_CONTRACT_UNSATISFIED_GATE_ID) is True
- assert copilot_author_time_gate_log_only_enabled(ctx, "credential_reference_presence") is False
-
-
-async def test_turn_setup_resolves_once_and_gate_checks_use_cached_ids(monkeypatch: pytest.MonkeyPatch) -> None:
- resolver = AsyncMock(return_value=frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}))
- monkeypatch.setattr(
- agent.app, "AGENT_FUNCTION", SimpleNamespace(resolve_copilot_author_time_gate_log_only_ids=resolver)
- )
- ctx = make_copilot_ctx(turn_id="turn-1")
-
- with capture_logs() as logs:
- await agent._cache_copilot_author_time_gate_log_only_ids(ctx)
- for _ in range(4):
- assert copilot_author_time_gate_log_only_enabled(ctx, DEFINITION_CONTRACT_UNSATISFIED_GATE_ID) is True
-
- resolver.assert_awaited_once_with(turn_id="turn-1", organization_id="org-1")
- assert [log for log in logs if log["event"] == "copilot_author_time_gate_log_only_registry_resolved"] == [
- {
- "event": "copilot_author_time_gate_log_only_registry_resolved",
- "log_level": "info",
- "selected_gate_ids": [DEFINITION_CONTRACT_UNSATISFIED_GATE_ID],
- "selected_gate_count": 1,
- }
- ]
-
-
-def test_definition_gate_replay_records_one_event_per_suppression() -> None:
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}))
- rejection = wu._DefinitionPlaneReject(
- criterion_ids=("criterion-1",),
- reason_codes=("definition_parameters_unreferenced",),
- unreferenced_parameter_keys=("account_name",),
- )
- workflow_yaml = _UNREFERENCED_DEFINITION_YAML
-
- for _ in range(16):
- assert wu._record_definition_plane_ablation_event(ctx, workflow_yaml, rejection) is True
-
- assert len(ctx.author_time_gate_ablation_events) == 16
- assert {event.gate_id for event in ctx.author_time_gate_ablation_events} == {
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID
- }
- assert len({event.fingerprint for event in ctx.author_time_gate_ablation_events}) == 1
-
-
-async def test_selected_definition_reject_persists_without_reject_state(monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}))
-
- result = await wu._update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is True
- assert ctx.latest_recorded_build_test_outcome is None
- assert ctx.blocker_signal is None
- assert ctx.turn_halt is None
- assert [event.gate_id for event in ctx.author_time_gate_ablation_events] == [
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID
- ]
-
-
-def test_selected_definition_reject_strict_run_preflight_records_ablation_event() -> None:
- ctx = _definition_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}))
-
- result = wu._metadata_contract_run_preflight_reject(
- ctx,
- _UNREFERENCED_DEFINITION_YAML,
- [],
- enforce_untagged_declared_inputs=True,
- )
-
- assert result is None
- assert ctx.latest_recorded_build_test_outcome is None
- assert [event.gate_id for event in ctx.author_time_gate_ablation_events] == [
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID
- ]
-
-
-def test_binder_ceiling_suppression_does_not_stash_reject_state(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID}))
- outcome = RecordedBuildTestOutcome(
- phase="author_time_reject",
- verdict="authoring_rejected",
- reason_code="definition_contract_unsatisfied",
- structural_failure_identity="definition-fingerprint",
- )
- assert outcome.structural_key is not None
- ctx.latest_recorded_build_test_outcome = outcome
- ctx.recorded_outcome_grounding_requirement = RecordedOutcomeGroundingRequirement(
- phase="author_time_reject",
- reason_code="definition_contract_unsatisfied",
- structural_key=outcome.structural_key,
- satisfied=True,
- )
- ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=outcome.structural_key,
- phase="author_time_reject",
- reason_code="definition_contract_unsatisfied",
- frontier_facet="value_shape",
- )
- monkeypatch.setattr(wu, "latest_recorded_build_test_outcome_repeated", lambda _ctx: True)
-
- assert wu._stash_unresolved_recorded_outcome_grounding_halt(ctx, ["account_name"]) is False
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert ctx.latest_recorded_build_test_outcome is outcome
- assert ctx.author_time_gate_ablation_events[-1].gate_id == RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID
-
-
-def test_synthesized_binding_suppression_records_bounded_structural_event() -> None:
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID}))
- repair_context = CodeAuthoringRepairContext(
- block_label="submit_request",
- reason_code="synthesized_parameter_binding_ambiguous",
- unresolved_names=["account_name"],
- parameter_keys=["account_name"],
- available_parameter_keys=["customer_name"],
- binding_candidates=["account_name", "customer_name"],
- )
-
- assert (
- wu._record_synthesized_parameter_binding_ablation_event(
- ctx,
- _UNREFERENCED_DEFINITION_YAML,
- repair_context,
- )
- is True
- )
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID
- assert event.reason_code == "synthesized_parameter_binding_ambiguous"
- assert event.payload["unresolved_names"] == ["account_name"]
-
-
-def test_structural_gate_suppression_fails_closed_without_fingerprint(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(
- ctx,
- frozenset(
- {
- DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- }
- ),
- )
- rejection = wu._DefinitionPlaneReject(
- criterion_ids=("criterion-1",),
- reason_codes=("definition_parameters_unreferenced",),
- unreferenced_parameter_keys=("account_name",),
- )
- repair_context = CodeAuthoringRepairContext(
- block_label="submit_request",
- reason_code="synthesized_parameter_binding_ambiguous",
- unresolved_names=["account_name"],
- parameter_keys=["account_name"],
- available_parameter_keys=["customer_name"],
- binding_candidates=["account_name", "customer_name"],
- )
- monkeypatch.setattr(wu, "authored_structure_signature_from_workflow", lambda *_args: None)
-
- assert wu._record_definition_plane_ablation_event(ctx, "invalid", rejection) is False
- assert wu._record_synthesized_parameter_binding_ablation_event(ctx, "invalid", repair_context) is False
- assert ctx.author_time_gate_ablation_events == []
-
-
-async def test_selected_synthesized_binding_reject_persists_without_repair_state(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID}))
- repair_context = CodeAuthoringRepairContext(
- block_label="submit_request",
- reason_code="synthesized_parameter_binding_ambiguous",
- unresolved_names=["account_name"],
- parameter_keys=["account_name"],
- available_parameter_keys=["customer_name"],
- binding_candidates=["account_name", "customer_name"],
- )
- monkeypatch.setattr(
- wu,
- "_maybe_impose_synthesized_code_block",
- lambda *_args, **_kwargs: wu._SynthesizedCodeImpositionResult(
- workflow_yaml=_UNREFERENCED_DEFINITION_YAML,
- violations=["Unable to bind synthesized parameter."],
- repair_context=repair_context,
- ablation_gate_id=SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- ),
- )
-
- result = await wu._update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is True
- assert ctx.latest_recorded_build_test_outcome is None
- assert ctx.last_code_authoring_repair_context is None
- assert ctx.author_time_gate_ablation_events[-1].gate_id == SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID
-
-
-def test_unselected_new_gate_keeps_enforcing() -> None:
- ctx = make_copilot_ctx()
- assert (
- record_author_time_gate_ablation_event(
- ctx,
- gate_id=DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
- reason_code="definition_contract_unsatisfied",
- fingerprint="fingerprint",
- )
- is False
- )
- assert ctx.author_time_gate_ablation_events == []
-
-
-_MISSING_METADATA_OUTPUT_YAML = """\
-title: Collect provider records
-workflow_definition:
- parameters:
- - {parameter_type: output, key: records}
- blocks:
- - block_type: code
- label: collect_records
- parameter_keys: []
- code: |
- await page.goto("https://example.com/records")
- return {"records": [{"number": "123"}]}
-"""
-
-
-_IMPOSABLE_METADATA = {
- "block_label": "collect_records",
- "declared_goal": "Collect the provider records",
- "terminal_verifier_expectations": [{"goal_value_paths": ["records[].number"]}],
-}
-
-
-def _missing_metadata_ctx() -> CopilotContext:
- ctx = make_copilot_ctx()
- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
- return ctx
-
-
-def test_metadata_family_gate_ids_are_posthog_eligible() -> None:
- ctx = make_copilot_ctx()
- with capture_logs() as logs:
- cache_copilot_author_time_gate_log_only_ids(
- ctx,
- frozenset(
- {
- OUTPUT_CONTRACT_ACTUATION_GATE_ID,
- METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
- "raw_secret_leak",
- "credential_reference_presence",
- }
- ),
- )
-
- assert ctx.author_time_gate_log_only_ids == frozenset(
- {OUTPUT_CONTRACT_ACTUATION_GATE_ID, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID}
- )
- assert copilot_author_time_gate_log_only_enabled(ctx, OUTPUT_CONTRACT_ACTUATION_GATE_ID) is True
- assert copilot_author_time_gate_log_only_enabled(ctx, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID) is True
- ineligible = {log["gate_id"] for log in logs if log["event"] == "copilot_gate_log_only_ineligible"}
- assert ineligible == {"raw_secret_leak", "credential_reference_presence"}
- for security_gate_id in ("raw_secret_leak", "credential_reference_presence"):
- assert copilot_author_time_gate_log_only_enabled(ctx, security_gate_id) is False
-
-
-async def test_missing_metadata_reject_persists_draft_under_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _missing_metadata_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
-
- result = await wu._update_workflow(
- {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is True
- assert ctx.latest_recorded_build_test_outcome is None
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == OUTPUT_CONTRACT_ACTUATION_GATE_ID
- assert event.reason_code == "missing_code_artifact_metadata"
- assert event.log_only is True
- assert event.blocked_tool == "update_workflow"
- assert event.fingerprint
- assert event.payload["block_labels"] == ["collect_records"]
- # No request-policy output contract is in play, so there is nothing to scaffold from and the
- # draft persists carrying no metadata rows.
- assert not ctx.code_artifact_metadata
-
-
-async def test_missing_metadata_reject_stays_enforcing_when_gate_unselected(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _missing_metadata_ctx()
-
- result = await wu._update_workflow(
- {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is False
- assert "code_artifact_metadata" in str(result["error"])
- assert ctx.author_time_gate_ablation_events == []
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
-
-
-_NORMALIZATION_YAML = """\
-title: Collect provider records
-workflow_definition:
- parameters: []
- blocks:
- - block_type: code
- label: collect_records
- parameter_keys: []
- code: |
- await page.goto("https://example.com/records")
- return {"records": [{"number": "123"}]}
-"""
-
-
-_TWO_BLOCK_YAML = """\
-title: Collect provider records
-workflow_definition:
- parameters: []
- blocks:
- - block_type: code
- label: block_ok
- parameter_keys: []
- code: |
- await page.goto("https://example.com/a")
- return {"records": [{"number": "1"}]}
- - block_type: code
- label: block_bad
- parameter_keys: []
- code: |
- await page.goto("https://example.com/b")
- return {"records": [{"number": "2"}]}
-"""
-
-
-async def test_normalization_seam_keeps_rows_the_enforcing_pass_accepted(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
- conforming = copy.deepcopy(_valid_metadata("block_ok"))
- conforming["claimed_outcomes"][0]["goal_value_paths"] = [_GOAL_PATH_PLACEHOLDER]
-
- result = await wu._update_workflow(
- {
- "workflow_yaml": _TWO_BLOCK_YAML,
- "code_artifact_metadata": [conforming, {"block_label": "block_bad", "declared_goal": "g"}],
- },
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is True
- assert ctx.latest_recorded_build_test_outcome is None
- assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"
- # block_ok normalizes on the enforcing pass; the seam must not drop it while relieving block_bad.
- assert set(ctx.code_artifact_metadata or {}) == {"block_ok"}
-
-
-async def test_schema_incompatibility_still_rejects_and_records_no_ablation_event(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
- incompatible = copy.deepcopy(_valid_metadata("collect_records"))
- incompatible["terminal_verifier_expectations"] = [
- {
- "goal_value_paths": ["records[].number"],
- "extraction_schema": json.dumps(
- {"type": "object", "properties": {"unrelated": {"type": "string"}}, "required": ["unrelated"]}
- ),
- }
- ]
-
- result = await wu._update_workflow(
- {"workflow_yaml": _NORMALIZATION_YAML, "code_artifact_metadata": [incompatible]},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- if result["ok"] is False and ctx.author_time_gate_ablation_events:
- raise AssertionError("a rejected save must not also report a log-only ablation for the same call")
-
-
-async def test_undefaultable_normalization_contradiction_saves_no_row_under_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = make_copilot_ctx()
- cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
-
- result = await wu._update_workflow(
- {
- "workflow_yaml": _NORMALIZATION_YAML,
- "code_artifact_metadata": [
- {
- "block_label": "collect_records",
- "declared_goal": "collect records",
- "claimed_outcomes": [{"id": "claim:x", "scope": "outcome", "text": "x", "status": "satisfied"}],
- }
- ],
- },
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is True
- assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"
- assert not ctx.code_artifact_metadata
-
-
-async def test_normalization_reject_stays_enforcing_when_gate_unselected(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- _stub_successful_update(monkeypatch)
- ctx = make_copilot_ctx()
-
- result = await wu._update_workflow(
- {
- "workflow_yaml": _NORMALIZATION_YAML,
- "code_artifact_metadata": [_IMPOSABLE_METADATA],
- },
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is False
- assert ctx.author_time_gate_ablation_events == []
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
-
-
-def test_eligible_gate_set_matches_the_seamed_gates_exactly() -> None:
- """Gate ids reaching an ablation recorder are the ground truth for what can be disabled:
- the flag's list must cover all of them, and every listed id must emit ablation events."""
- recorders = r"(?:record_author_time_gate_ablation_event|_record_output_contract_ablation_event)"
- seamed: set[str] = set()
- for path in pathlib.Path("skyvern/forge/sdk/copilot").rglob("*.py"):
- source = path.read_text()
- for match in re.finditer(recorders + r"\((.*?)\)", source, re.S):
- gate = re.search(r"gate_id=([A-Za-z_][A-Za-z_0-9]*|\"[a-z_]+\")", match.group(1))
- if gate is None:
- continue
- token = gate.group(1)
- if token == "gate_id":
- continue
- resolved = (
- token.strip('"') if token.startswith('"') else getattr(runtime, token, None) or getattr(wu, token, None)
- )
- if isinstance(resolved, str):
- seamed.add(resolved)
-
- assert seamed, "found no gate ids; the seam-detection regex needs updating"
- unreachable = seamed - AUTHOR_TIME_GATE_LOG_ONLY_IDS
- assert not unreachable, (
- f"these gates have a suppression seam but the flag cannot disable them: {sorted(unreachable)}. "
- "Add them to AUTHOR_TIME_GATE_LOG_ONLY_IDS."
- )
- eventless = AUTHOR_TIME_GATE_LOG_ONLY_IDS - seamed
- assert not eventless, (
- f"these flag-eligible gates emit no ablation events when suppressed: {sorted(eventless)}. "
- "Suppression without events cannot be graded or monitored — add a recording seam."
- )
-
-
-_SECURITY_GATE_IDS = frozenset(
- {
- "raw_secret_leak",
- "raw_secret_handling",
- "code_safety_reject",
- "credential_reference_presence",
- "unapproved_credential_reference",
- "credential_scout_reopen",
- }
-)
-
-
-def test_eligible_gate_ids_are_exactly_the_sanctioned_seven() -> None:
- """Security-critical: OutputPolicy security gates must never become flag-suppressible.
- Exact-content assertion is intentional — any change to this set is a security decision."""
- assert AUTHOR_TIME_GATE_LOG_ONLY_IDS == frozenset(
- {
- "definition_contract_unsatisfied",
- "recorded_outcome_grounding_binder_ceiling",
- "synthesized_parameter_binding_ambiguous",
- "output_contract_actuation",
- "metadata_run_preflight_reject",
- "uncovered_output_rescout_steer",
- "recorded_outcome_grounding",
- }
- )
- assert not (AUTHOR_TIME_GATE_LOG_ONLY_IDS & _SECURITY_GATE_IDS)
diff --git a/tests/unit/test_copilot_blocker_lifecycle.py b/tests/unit/test_copilot_blocker_lifecycle.py
index 9a1857b10..6e0dd246a 100644
--- a/tests/unit/test_copilot_blocker_lifecycle.py
+++ b/tests/unit/test_copilot_blocker_lifecycle.py
@@ -1,21 +1,9 @@
from __future__ import annotations
-import pytest
-
from skyvern.forge.sdk.copilot.blocker_signal import BlockerKind, CopilotToolBlockerSignal
-from skyvern.forge.sdk.copilot.enforcement import (
- MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
- _record_code_authoring_guardrail_reject,
-)
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
from skyvern.forge.sdk.copilot.mcp_adapter import _stash_and_emit_loop_blocker
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
-from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
-from skyvern.forge.sdk.copilot.turn_ownership import (
- TurnClaimant,
- blocker_signal_render_allowed,
- current_turn_owner,
-)
from tests.unit.conftest import make_copilot_context as _ctx
@@ -284,31 +272,6 @@ def test_reconciliation_requires_input_does_not_clear_after_metadata_only_succes
assert ctx.blocker_signal.internal_reason_code == "tool_error_pending_reconciliation_requires_input"
-def test_preflight_reject_leaves_churn_floor_armed_for_rest_of_turn(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _ctx()
- monkeypatch.setattr(
- workflow_update_module,
- "_recorded_outcome_convergence_reject",
- lambda *_args, **_kwargs: workflow_update_module._ConvergenceReject(
- "sig", "identical_authored_structure", False
- ),
- )
-
- reject = workflow_update_module._metadata_contract_run_preflight_reject(ctx, "title: X", [])
- assert reject is not None
- assert ctx.turn_ownership is not None
- assert TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT in ctx.turn_ownership.claims
- assert current_turn_owner(ctx) is None
-
- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
- _record_code_authoring_guardrail_reject(ctx)
-
- signal = ctx.blocker_signal
- assert signal is not None
- assert signal.internal_reason_code == "code_authoring_guardrail_churn"
- assert blocker_signal_render_allowed(ctx, signal) is True
-
-
def test_mcp_loop_blocker_yield_still_blocks_the_tool() -> None:
ctx = _ctx()
ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
index 20473e9af..737ca0479 100644
--- a/tests/unit/test_copilot_blocker_signal.py
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -6,14 +6,12 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
_LEAK_DENY_TOKENS,
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
BlockerKind,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
build_llm_tool_error_payload,
build_loop_blocker_signal,
- build_metadata_reject_same_key_terminal_signal,
clear_blocker_signal_for_reason_codes,
maybe_clear_blocker_signal_on_tool_success,
refresh_held_loop_blocker_evidence,
@@ -87,38 +85,6 @@ def test_to_trace_data_namespaces_extra_so_it_cannot_shadow_explicit_fields() ->
assert trace["extra"] == {"blocker_kind": "evil", "custom_metric": 7}
-def test_metadata_same_key_terminal_names_gate_and_exact_missing_fields() -> None:
- missing_fields = {"extract_record": ["declared_goal", "claimed_outcomes", "evidence_refs_or_observation_refs"]}
-
- signal = build_metadata_reject_same_key_terminal_signal(
- structural_key="same-key",
- reject_family="missing_code_artifact_metadata",
- missing_fields_by_label=missing_fields,
- )
-
- assert signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert signal.preserves_workflow_draft is True
- assert signal.renders_final_reply is True
- assert "code_artifact_metadata" in signal.user_facing_reason
- assert '"extract_record"' in signal.user_facing_reason
- assert signal.extra["gate_id"] == "code_artifact_metadata"
- assert signal.extra["missing_fields_by_label"] == missing_fields
-
-
-def test_metadata_same_key_terminal_degrades_leaky_label_in_user_facing_reason() -> None:
- missing_fields = {"safe_reason_code": ["declared_goal"]}
-
- signal = build_metadata_reject_same_key_terminal_signal(
- structural_key="same-key",
- reject_family="missing_code_artifact_metadata",
- missing_fields_by_label=missing_fields,
- )
-
- assert "safe_reason_code" not in signal.user_facing_reason
- assert signal.extra["missing_fields_by_label"] == missing_fields
- assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=signal.blocked_tool)
-
-
@pytest.mark.parametrize("token", _LEAK_DENY_TOKENS)
def test_assert_clean_raises_on_each_deny_token(token: str) -> None:
with pytest.raises(ValueError):
@@ -385,84 +351,6 @@ def test_stash_churn_stop_does_not_replace_held_output_contract_terminal() -> No
assert ctx.blocker_signal is oc_terminal
-def test_stash_grounding_replaces_synthesized_persistence_tool_error() -> None:
- ctx = _Ctx()
- existing = _make(
- kind="tool_error",
- internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
- renders_final_reply=False,
- )
- grounding = _make(
- kind="missing_required_context",
- internal_reason_code="recorded_outcome_grounding_required",
- renders_final_reply=False,
- )
-
- stash_blocker_signal(ctx, existing)
- payload = stash_blocker_signal(ctx, grounding)
-
- assert payload == grounding.agent_steering_text
- assert ctx.blocker_signal is grounding
- assert ctx.latest_tool_blocker_signal is grounding
- assert ctx.tool_blocker_signals == [existing, grounding]
-
-
-def test_stash_grounding_replaces_generic_non_final_tool_error() -> None:
- ctx = _Ctx()
- existing = _make(kind="tool_error", internal_reason_code="tool_error_generic", renders_final_reply=False)
- grounding = _make(
- kind="missing_required_context",
- internal_reason_code="recorded_outcome_grounding_required",
- renders_final_reply=False,
- )
-
- stash_blocker_signal(ctx, existing)
- stash_blocker_signal(ctx, grounding)
-
- assert ctx.blocker_signal is grounding
-
-
-def test_stash_grounding_does_not_replace_final_reply_blocker() -> None:
- ctx = _Ctx()
- existing = _make(
- kind="tool_error",
- internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
- renders_final_reply=True,
- )
- grounding = _make(
- kind="missing_required_context",
- internal_reason_code="recorded_outcome_grounding_required",
- renders_final_reply=False,
- )
-
- stash_blocker_signal(ctx, existing)
- stash_blocker_signal(ctx, grounding)
-
- assert ctx.blocker_signal is existing
- assert ctx.latest_tool_blocker_signal is grounding
-
-
-def test_stash_repair_ceiling_replaces_non_final_grounding_blocker() -> None:
- ctx = _Ctx()
- grounding = _make(
- kind="missing_required_context",
- internal_reason_code="recorded_outcome_grounding_required",
- renders_final_reply=False,
- )
- repair_ceiling = _make(
- kind="loop_detected",
- internal_reason_code="repair_ceiling_reached",
- renders_final_reply=True,
- )
-
- stash_blocker_signal(ctx, grounding)
- payload = stash_blocker_signal(ctx, repair_ceiling)
-
- assert payload == repair_ceiling.agent_steering_text
- assert ctx.blocker_signal is repair_ceiling
- assert ctx.latest_tool_blocker_signal is repair_ceiling
-
-
def test_agent_context_and_copilot_context_blocker_signal_defaults_match() -> None:
"""The field is declared on both AgentContext (parent) and CopilotContext
(child) per the field-shadowing convention. Default values must stay in
@@ -505,6 +393,28 @@ _LATE_RECORDED_REASON = (
)
+def test_stash_non_final_signal_does_not_replace_a_held_final_reply_blocker() -> None:
+ """A non-final incoming signal must not displace a held final-reply blocker, while
+ latest_tool_blocker_signal still advances to it."""
+ ctx = _Ctx()
+ existing = _make(
+ kind="tool_error",
+ internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
+ renders_final_reply=True,
+ )
+ incoming = _make(
+ kind="missing_required_context",
+ internal_reason_code="tool_error_post_budget_page_inspection_required",
+ renders_final_reply=False,
+ )
+
+ stash_blocker_signal(ctx, existing)
+ stash_blocker_signal(ctx, incoming)
+
+ assert ctx.blocker_signal is existing
+ assert ctx.latest_tool_blocker_signal is incoming
+
+
def test_stash_refreshes_held_loop_signal_with_evidence_recorded_after_the_stash() -> None:
ctx = _copilot_ctx()
loop_signal = build_loop_blocker_signal(_CONSECUTIVE_LOOP_MESSAGE, tool_name="evaluate")
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 2d07d71b8..21433c602 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -8,9 +8,12 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedOutcomeBindingConstraint,
_binding_frontier_facet,
adjudicate_metadata_reject_ladder,
+ arm_recorded_outcome_grounding_requirement,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
+ clear_recorded_outcome_grounding_requirement,
latest_recorded_build_test_outcome_repeated,
+ maybe_satisfy_recorded_outcome_grounding_requirement,
observed_value_extraction_scaffold_lines,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
@@ -1399,3 +1402,113 @@ def test_persisted_run_prose_only_failure_is_not_authoritative() -> None:
assert outcome is None or outcome.is_authoritative is False
assert outcome is None or outcome.structural_key is None
(adjudicate_metadata_reject_ladder,)
+
+
+def _grounding_outcome(*, workflow_run_id: str | None = "wr_grounded") -> RecordedBuildTestOutcome:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id=workflow_run_id,
+ block_labels=["search_records"],
+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
+ page_evidence_refs=["origin_present"],
+ )
+
+
+def _grounding_ctx(outcome: RecordedBuildTestOutcome | None) -> SimpleNamespace:
+ return SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_grounding_requirement=None,
+ recorded_outcome_binding_constraint=None,
+ composition_page_evidence=None,
+ scout_observation_contract=None,
+ last_run_blocks_workflow_run_id=None,
+ observed_browser_urls=[],
+ )
+
+
+def test_grounding_arms_on_an_authoritative_non_progress_outcome() -> None:
+ outcome = _grounding_outcome()
+ ctx = _grounding_ctx(outcome)
+
+ requirement = arm_recorded_outcome_grounding_requirement(ctx)
+
+ assert requirement is not None
+ assert requirement.structural_key == outcome.structural_key
+ assert requirement.workflow_run_id == "wr_grounded"
+ assert requirement.satisfied is False
+ assert ctx.recorded_outcome_grounding_requirement is requirement
+
+
+def test_grounding_does_not_arm_on_progress_or_without_an_authoritative_outcome() -> None:
+ progressed = _grounding_outcome().model_copy(update={"verdict": "progress_observed"})
+ assert arm_recorded_outcome_grounding_requirement(_grounding_ctx(progressed)) is None
+
+ assert arm_recorded_outcome_grounding_requirement(_grounding_ctx(None)) is None
+
+ unauthoritative = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ )
+ assert unauthoritative.is_authoritative is False
+ assert arm_recorded_outcome_grounding_requirement(_grounding_ctx(unauthoritative)) is None
+
+
+def test_grounding_arm_is_idempotent_for_the_same_outcome_and_run() -> None:
+ ctx = _grounding_ctx(_grounding_outcome())
+
+ first = arm_recorded_outcome_grounding_requirement(ctx)
+ second = arm_recorded_outcome_grounding_requirement(ctx)
+
+ assert first is not None
+ assert second is first
+
+
+def test_grounding_is_satisfied_only_by_inspected_evidence_from_the_same_run() -> None:
+ ctx = _grounding_ctx(_grounding_outcome())
+ arm_recorded_outcome_grounding_requirement(ctx)
+
+ ctx.composition_page_evidence = {
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.com/results",
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_other",
+ }
+ assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
+ assert ctx.recorded_outcome_grounding_requirement.satisfied is False
+
+ ctx.composition_page_evidence = {**ctx.composition_page_evidence, "workflow_run_id": "wr_grounded"}
+ assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
+
+ satisfied = ctx.recorded_outcome_grounding_requirement
+ assert satisfied.satisfied is True
+ assert satisfied.payload is not None
+ assert satisfied.payload.source_url == "https://example.com/results"
+ assert satisfied.payload.repeated_structural_key == satisfied.structural_key
+
+
+def test_grounding_is_not_satisfied_by_evidence_from_another_tool() -> None:
+ ctx = _grounding_ctx(_grounding_outcome())
+ arm_recorded_outcome_grounding_requirement(ctx)
+ ctx.composition_page_evidence = {
+ "source_tool": "evaluate",
+ "current_url": "https://example.com/results",
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_grounded",
+ }
+
+ assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
+
+
+def test_clearing_the_grounding_requirement_also_drops_the_binding_constraint() -> None:
+ ctx = _grounding_ctx(_grounding_outcome())
+ arm_recorded_outcome_grounding_requirement(ctx)
+ ctx.recorded_outcome_binding_constraint = object()
+
+ clear_recorded_outcome_grounding_requirement(ctx)
+
+ assert ctx.recorded_outcome_grounding_requirement is None
+ assert ctx.recorded_outcome_binding_constraint is None
diff --git a/tests/unit/test_copilot_cancel_helpers.py b/tests/unit/test_copilot_cancel_helpers.py
index 0742ad64d..9b0d68b0c 100644
--- a/tests/unit/test_copilot_cancel_helpers.py
+++ b/tests/unit/test_copilot_cancel_helpers.py
@@ -248,9 +248,7 @@ def _guard_ctx(pending_run_id: str | None = None) -> _NS:
repeated_action_fingerprint_streak_count=0,
last_test_non_retriable_nav_error=None,
pending_reconciliation_run_id=pending_run_id,
- synthesized_block_reopened_for_output_coverage=False,
output_contract_actuation_by_signature={},
- output_contract_reject_count_by_signature={},
output_contract_actuation_count_by_signature={},
output_contract_armed_directive_fingerprint_by_signature={},
)
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
index db5389349..80d9572a6 100644
--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
from skyvern.forge.sdk.copilot.tools.workflow_update import (
_code_block_returns_flat_string,
_code_block_returns_uninvoked_structured_function,
+ _download_descriptor_leak_finding,
_normalize_code_artifact_metadata_detailed,
)
@@ -981,106 +982,39 @@ def _download_intent_metadata(label: str) -> dict:
return metadata
-class TestDownloadReturnShape:
- def test_expect_download_idiom_with_descriptor_passes(self) -> None:
- code = """
- async with page.expect_download() as dl_info:
- await page.click("a#statement-pdf")
- return {"saved_as": dl_info.value.suggested_filename}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert error is None
- assert list(normalized.keys()) == ["dl_block"]
+class TestDownloadDescriptorLeak:
+ """A run cannot reveal this arm: the run succeeds and the path lands in workflow output.
+ The registration-detection arms were deleted precisely because a run does reveal those."""
- def test_self_asserted_keys_without_idiom_is_rejected(self) -> None:
- code = """
- await page.click("a#statement-pdf")
- return {"downloaded_files": ["/tmp/statement.pdf"]}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "expect_download" in error
+ def test_returned_path_or_url_is_flagged(self) -> None:
+ for code in (
+ 'return {"downloaded_file_path": p}',
+ 'return {"download_url": u}',
+ 'out = {"download_url": u}\nreturn out',
+ ):
+ assert _download_descriptor_leak_finding("b", code) is not None
- def test_plain_click_fabricated_dict_is_rejected(self) -> None:
- code = """
- await page.click("a#statement-pdf")
- return {"downloaded_file_name": "statement.pdf", "evidence": "ok"}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "expect_download" in error
+ def test_clean_descriptor_and_registration_keys_are_not_flagged(self) -> None:
+ assert _download_descriptor_leak_finding("b", 'return {"saved_as": n}') is None
+ assert _download_descriptor_leak_finding("b", 'return {"downloaded_files": [f]}') is None
- def test_static_fetch_body_is_rejected(self) -> None:
- code = """
- import requests
- body = requests.get("https://example.com/statement.pdf").content
- return {"downloaded_files": [body]}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "expect_download" in error
- def test_idiom_but_returns_registration_keys_is_rejected(self) -> None:
- code = """
- async with page.expect_download() as dl_info:
- await page.click("a#statement-pdf")
- return {"downloaded_files": [await dl_info.value.path()]}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "downloaded_files" in error
- assert "self-certifies" in error
+class TestDownloadShapesThatMustNotBeFlagged:
+ """Negative space for the surviving descriptor-leak arm: a clean descriptor, an extraction
+ block, and a non-download block must all normalize without a violation."""
- def test_idiom_but_returns_raw_path_or_url_descriptor_keys_is_rejected(self) -> None:
+ def test_expect_download_idiom_with_descriptor_passes(self) -> None:
code = """
async with page.expect_download() as dl_info:
await page.click("a#statement-pdf")
- payload = dict(
- download_url="https://example.com/statement.pdf",
- downloaded_file_path=await dl_info.value.path(),
- downloaded_file_name="statement.pdf",
- )
- return payload
+ return {"saved_as": dl_info.value.suggested_filename}
"""
normalized, error = _normalize_code_artifact_metadata(
[_download_intent_metadata("dl_block")],
_extraction_code_block_yaml("dl_block", code),
)
- assert normalized == {}
- assert error is not None
- assert "raw download path/URL descriptor keys" in error
-
- def test_self_asserted_keys_detected_without_goal_path_declaration(self) -> None:
- code = """
- await page.click("a#statement-pdf")
- return {"downloaded_files": ["/tmp/statement.pdf"]}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_non_extraction_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "expect_download" in error
+ assert error is None
+ assert list(normalized.keys()) == ["dl_block"]
def test_extraction_block_is_not_treated_as_download_intent(self) -> None:
code = """
@@ -1094,20 +1028,6 @@ class TestDownloadReturnShape:
assert error is None
assert list(normalized.keys()) == ["ex_block"]
- def test_bare_expect_download_attribute_does_not_satisfy_the_idiom(self) -> None:
- code = """
- _ = page.expect_download
- await page.click("a#statement-pdf")
- return {"downloaded_files": ["/tmp/statement.pdf"]}
- """
- normalized, error = _normalize_code_artifact_metadata(
- [_download_intent_metadata("dl_block")],
- _extraction_code_block_yaml("dl_block", code),
- )
- assert normalized == {}
- assert error is not None
- assert "expect_download" in error
-
def test_non_download_non_extraction_block_passes(self) -> None:
code = """
await page.click("a#statement-pdf")
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 6eeb6fa30..edf55a962 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -9,9 +9,9 @@ import ast
import hashlib
import inspect
import json
+import re
import textwrap
from collections.abc import Iterable
-from dataclasses import replace
from pathlib import Path
from types import EllipsisType, SimpleNamespace
from typing import Any
@@ -21,7 +21,6 @@ import pytest
import yaml
from structlog.testing import capture_logs
-from skyvern.config import settings
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot import code_block_synthesis as code_block_synthesis_module
from skyvern.forge.sdk.copilot import enforcement as enforcement_module
@@ -29,26 +28,18 @@ from skyvern.forge.sdk.copilot import request_policy as request_policy_module
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
AuthoringParameterBindingCandidate,
- AuthoringParameterBindingDirective,
- authored_selector_parameter_bindings,
build_authoring_parameter_binding_directive,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
- build_definition_contract_unsatisfied_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestOutcomeReasonCode,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
- arm_recorded_outcome_grounding_requirement,
- authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
- latest_recorded_build_test_outcome_repeated,
- maybe_satisfy_recorded_outcome_grounding_requirement,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
recorded_outcome_from_authoring_repair_context,
@@ -101,7 +92,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
)
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
-from skyvern.forge.sdk.copilot.runtime import AgentContext, NeverCapturedObligation
+from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tools import (
_code_block_safety_errors,
_detect_stale_block_metadata,
@@ -111,7 +102,6 @@ from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modul
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.workflow_update import (
- _code_block_render_reject,
_code_safety_reject_payload,
_OutputContractEvaluation,
_strip_redundant_sandbox_imports_in_yaml,
@@ -479,7 +469,6 @@ def _stub_successful_update(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", _fake_process_workflow_yaml)
monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", _fake_get_prior_workflow)
- monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
def _single_code_block(parsed: dict[str, object]) -> dict[str, object]:
@@ -706,27 +695,6 @@ def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") ->
)
-class TestCredentialRealValueBindings:
- def test_real_username_reference_is_not_falsely_rejected(self) -> None:
- yaml_text = _credential_code_yaml(
- code='await page.locator("#email").fill("{{ login_credential_real_username }}")',
- )
- assert _code_block_render_reject(yaml_text, None) is None
-
- def test_real_password_reference_is_not_falsely_rejected(self) -> None:
- yaml_text = _credential_code_yaml(
- code='await page.locator("input[type=\'password\']").fill("{{ login_credential_real_password }}")',
- )
- assert _code_block_render_reject(yaml_text, None) is None
-
- def test_unbound_credential_derived_name_still_rejected(self) -> None:
- yaml_text = _credential_code_yaml(
- code='await page.locator("#email").fill("{{ nonexistent_credential_real_username }}")',
- )
- result = _code_block_render_reject(yaml_text, None)
- assert result is not None
-
-
def _directory_blocks_yaml(blocks: str) -> str:
indented_blocks = textwrap.indent(textwrap.dedent(blocks).strip(), " ")
return f"title: Directory lookup\nworkflow_definition:\n blocks:\n{indented_blocks}\n"
@@ -1207,7 +1175,8 @@ class TestCodeSafetySeam:
result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
assert result["ok"] is False
- assert "authoring_repair_context" not in result["data"]
+ # A block carrying no data omits the key entirely, which satisfies the same contract.
+ assert "authoring_repair_context" not in result.get("data", {})
assert ctx.last_code_authoring_repair_context is None
@pytest.mark.parametrize(
@@ -1227,29 +1196,6 @@ class TestCodeSafetySeam:
assert "authoring_repair_context" not in result["data"]
assert ctx.last_code_authoring_repair_context is None
- @pytest.mark.asyncio
- async def test_imposition_reject_carries_ambiguous_selector_repair_context(self) -> None:
- ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
- private_url = "https://example.com/orders?session=secret-token#account"
- for interaction in ctx.scout_trajectory:
- interaction["source_url"] = private_url
-
- result = await _update_workflow({"workflow_yaml": _resale_submitted_yaml("button:nth-of-type(2)")}, ctx)
-
- assert result["ok"] is False
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "ambiguous_bare_selector"
- assert repair_context["block_label"] == "order_status"
- assert repair_context["selector"] == "button"
- assert repair_context["source_url"] == "https://example.com"
- assert "secret-token" not in str(repair_context)
- assert repair_context["refiner_selector"] is None
- assert repair_context["selector_alternatives"] == [
- {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"}
- ]
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
-
def test_ambiguous_selector_repair_context_only_carries_valid_refiner(self) -> None:
code_block = {"label": "order_status"}
dropped = {
@@ -1361,20 +1307,6 @@ class TestCodeSafetySeam:
assert "button:nth-of-type" not in str(dumped)
assert "other-page" not in str(dumped)
- @pytest.mark.asyncio
- async def test_imposition_reject_replaces_stale_repair_context(self) -> None:
- ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
-
- result = await _update_workflow({"workflow_yaml": _resale_submitted_yaml("button:nth-of-type(2)")}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["authoring_repair_context"]["reason_code"] == "ambiguous_bare_selector"
- assert result["data"]["authoring_repair_context"]["selector_alternatives"] == [
- {"tool_name": "type_text", "role": "textbox", "selector": "#order-id"}
- ]
- assert ctx.last_code_authoring_repair_context != _stale_unresolved_repair_context()
-
def _live_root_value_bearing_ctx() -> CopilotContext:
ctx = _code_only_ctx()
@@ -1427,110 +1359,6 @@ class TestCodeRepairProgressClassification:
"32d1a9443d6dae7e8f6367cbe52902d7aaeb270e78c0275a66648d6013f24f9d"
)
- @pytest.mark.asyncio
- async def test_update_workflow_rejects_unreferenced_definition_before_persistence(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- prior_yaml = ctx.workflow_yaml
-
- for _ in range(2):
- result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "definition_contract_unsatisfied"
-
- assert ctx.workflow_yaml == prior_yaml
- assert ctx.has_staged_proposal is False
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert latest_recorded_build_test_outcome_repeated(ctx) is True
- requirement = arm_recorded_outcome_grounding_requirement(ctx)
- assert requirement is not None
- assert requirement.required_target_url == "current_page"
-
- @pytest.mark.asyncio
- @pytest.mark.parametrize("reason_code", ["definition_contract_unsatisfied", "metadata_reject"])
- async def test_recut_binding_changes_candidate(
- self,
- monkeypatch: pytest.MonkeyPatch,
- reason_code: BuildTestOutcomeReasonCode,
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- for _ in range(2):
- rejected = await _update_workflow({"workflow_yaml": _UNREFERENCED_DEFINITION_YAML}, ctx)
- assert rejected["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- source_url = "https://example.com/utility"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#open", "source_url": source_url, "trajectory_index": 0},
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 1},
- ]
- runtime_parameters = {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- }
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "page_title": "Utility request",
- "forms": [
- {
- "fields": [
- {"selector": "#company", "value": runtime_parameters["business_name"]},
- {"selector": "#email", "value": runtime_parameters["contact_email"]},
- {"selector": "#address", "value": runtime_parameters["service_address"]},
- {"selector": "#date", "value": runtime_parameters["desired_start_date"]},
- ],
- "submit_controls": [{"selector": "#submit"}],
- }
- ],
- "navigation_targets": [],
- "result_containers": [],
- "challenge_controls": [],
- "anti_bot_indicators": [],
- "observed_after_workflow_run": False,
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
- requirement = ctx.recorded_outcome_grounding_requirement
- constraint = ctx.recorded_outcome_binding_constraint
- assert requirement is not None
- assert constraint is not None
- ctx.recorded_outcome_grounding_requirement = requirement.model_copy(update={"reason_code": reason_code})
- ctx.recorded_outcome_binding_constraint = constraint.model_copy(update={"reason_code": reason_code})
-
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML, "parameters": runtime_parameters},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
- bound = next(log for log in logs if log["event"] == "copilot recorded outcome submit rung bound")
- consumed = next(
- log for log in logs if log["event"] == "copilot recorded outcome binding consumed by synthesizer"
- )
- assert consumed["binding_fingerprints"] == [bound["binding_fingerprint"]]
- assert all("submit_rung_binding" not in interaction for interaction in ctx.scout_trajectory)
- code = str(_single_code_block(parse_workflow_yaml(ctx.workflow_yaml))["code"])
- for selector, key in (
- ("#company", "business_name"),
- ("#email", "contact_email"),
- ("#address", "service_address"),
- ("#date", "desired_start_date"),
- ):
- assert f'page.locator("{selector}").fill(str({key}))' in code
- assert code.index(f'page.locator("{selector}").fill') < code.index('page.locator("#submit").click()')
-
def test_recut_binding_is_inert_for_runtime_self_heal(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _definition_contract_ctx()
_enable_imposition(ctx)
@@ -1553,184 +1381,6 @@ class TestCodeRepairProgressClassification:
assert result.substitutions is None
assert ctx.authoring_parameter_binding_snapshot is None
- @pytest.mark.asyncio
- @pytest.mark.parametrize("terminal_tool", ["click", "press_key"])
- async def test_recut_binding_scopes_to_unreferenced_keys_and_rebinds_captured_fills(
- self, monkeypatch: pytest.MonkeyPatch, terminal_tool: str
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- submitted = _yaml(
- """
- title: Submit reusable service request
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
- - {parameter_type: workflow, key: account_reference, workflow_parameter_type: string}
- blocks:
- - block_type: code
- label: submit_service_request
- parameter_keys: []
- code: |
- await page.locator("#open").click()
- await page.locator("#submit").click()
- - block_type: code
- label: retain_account_reference
- parameter_keys: [account_reference]
- code: |
- return {"account_reference": account_reference}
- """
- )
- source_url = "https://example.com/utility"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#open", "source_url": source_url, "trajectory_index": 0},
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 1},
- ]
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": submitted}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- values = {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- "account_reference": "account-1",
- }
- selectors = {
- "business_name": "#company",
- "contact_email": "#email",
- "service_address": "#address",
- "desired_start_date": "#date",
- }
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": selector,
- "source_url": source_url,
- "typed_length": len(values[key]),
- "trajectory_index": index,
- }
- for index, (key, selector) in enumerate(selectors.items())
- ]
- terminal_selector = "#submit" if terminal_tool == "click" else "#date"
- terminal_interaction = {
- "tool_name": terminal_tool,
- "selector": terminal_selector,
- "source_url": source_url,
- "trajectory_index": len(ctx.scout_trajectory),
- }
- if terminal_tool == "press_key":
- terminal_interaction["key"] = "Enter"
- ctx.scout_trajectory.append(terminal_interaction)
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [{"selector": selector, "value": values[key]} for key, selector in selectors.items()],
- "submit_controls": [{"selector": "#submit"}] if terminal_tool == "click" else [],
- }
- ],
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": submitted, "parameters": values},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = next(block for block in workflow_blocks(parsed) if block.get("label") == "submit_service_request")
- code = str(block["code"])
- assert block["parameter_keys"] == [
- "business_name",
- "contact_email",
- "service_address",
- "desired_start_date",
- ]
- assert "account_reference" not in block["parameter_keys"]
- terminal_call = (
- 'page.locator("#submit").click()' if terminal_tool == "click" else 'page.locator("#date").press("Enter")'
- )
- for key, selector in selectors.items():
- fill = f'page.locator("{selector}").fill(str({key}))'
- assert code.count(fill) == 1
- assert code.index(fill) < code.index(terminal_call)
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
-
- @pytest.mark.asyncio
- async def test_selection_shaped_binding_references_declared_keys_without_fillable_surface(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- submitted = _yaml(
- """
- title: Retrieve statement rows
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: account_number, workflow_parameter_type: string, default_value: "AC12345"}
- - {parameter_type: workflow, key: billing_start_date, workflow_parameter_type: string, default_value: "2026-08-01"}
- blocks:
- - block_type: code
- label: open_statement_rows
- parameter_keys: []
- code: |
- await page.locator("[data-account=\\"AC12345\\"]").click()
- await page.locator("[data-range=\\"2026-08-01\\"]").click()
- """
- )
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": submitted}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- source_url = "https://example.com/statements"
- values = {"account_number": "AC12345", "billing_start_date": "2026-08-01"}
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": '[data-account="AC12345"]',
- "source_url": source_url,
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": '[data-range="2026-08-01"]',
- "source_url": source_url,
- "trajectory_index": 1,
- },
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [],
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": submitted, "parameters": values},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = next(block for block in workflow_blocks(parsed) if block.get("label") == "open_statement_rows")
- code = str(block["code"])
- assert 'f"[data-account=\\"{account_number}\\"]"' in code
- assert 'f"[data-range=\\"{billing_start_date}\\"]"' in code
- assert set(block["parameter_keys"]) == {"account_number", "billing_start_date"}
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
- assert ctx.authoring_parameter_binding_snapshot is not None
-
def test_same_month_file_match_runtime_values_override_defaults_and_satisfy_definition(self) -> None:
ctx = _definition_contract_ctx()
_enable_imposition(ctx)
@@ -1796,65 +1446,6 @@ class TestCodeRepairProgressClassification:
assert workflow_update_module._definition_plane_preflight_reject(ctx, result.workflow_yaml) is None
assert ctx.authoring_parameter_binding_snapshot is None
- def test_same_month_file_match_rebinds_account_already_referenced_by_navigation(self) -> None:
- ctx, _, values = _same_month_file_match_case()
- source_url = "https://example.com/statements"
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": '[data-account="100245"]',
- "source_url": source_url,
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Download reusable invoice
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: account_number, workflow_parameter_type: string, default_value: "stale-account"}
- - {parameter_type: workflow, key: download_start_date, workflow_parameter_type: string, default_value: "2026-06-01"}
- - {parameter_type: workflow, key: download_end_date, workflow_parameter_type: string, default_value: "2026-06-30"}
- blocks:
- - block_type: code
- label: download_invoice
- parameter_keys: [account_number]
- code: |
- await page.locator(f'[data-account="{account_number}"]').click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(
- submitted,
- ctx,
- runtime_parameters=values,
- )
-
- assert result.violations == []
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- code = str(block["code"])
- assert set(block["parameter_keys"]) == set(values)
- assert 'f"[data-account=\\"{account_number}\\"]"' in code
- assert 'f"a[href=\\"/files/invoice_{account_number}_{_scout_same_month_iso(' in code
- assert "100245" not in code
- assert "2026-05" not in code
- synthesized = ctx.imposition_synthesized_block
- assert synthesized is not None
- assert {parameter["key"] for parameter in synthesized.parameters} == set(values)
- assert all("default_value" not in parameter for parameter in synthesized.parameters)
- assert workflow_update_module._definition_plane_preflight_reject(ctx, result.workflow_yaml) is None
- assert (
- workflow_update_module._metadata_contract_run_preflight_reject(
- ctx,
- result.workflow_yaml,
- ctx.raw_code_artifact_metadata,
- values,
- )
- is None
- )
-
def test_same_month_file_match_rebinds_literal_when_other_block_references_all_keys(self) -> None:
ctx, _, values = _same_month_file_match_case()
submitted = _yaml(
@@ -2114,229 +1705,92 @@ class TestCodeRepairProgressClassification:
assert resolution.directive is None
@pytest.mark.asyncio
- async def test_corrective_pass_satisfies_definition_and_scouted_spine_gates_jointly(
+ async def test_synthesized_business_input_floor_allows_fully_bound_draft(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": _UNREFERENCED_DEFINITION_YAML}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- source_url = "https://example.com/utility"
- values = {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- }
- selectors = {
- "business_name": "#company",
- "contact_email": "#email",
- "service_address": "#address",
- "desired_start_date": "#date",
- }
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": selector,
- "source_url": source_url,
- "typed_length": len(values[key]),
- "trajectory_index": index,
- }
- for index, (key, selector) in enumerate(selectors.items())
- ]
- ctx.scout_trajectory.append(
- {
- "tool_name": "click",
- "selector": "#submit",
- "source_url": source_url,
- "trajectory_index": len(ctx.scout_trajectory),
- }
- )
- assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is True
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [{"selector": selector, "value": values[key]} for key, selector in selectors.items()],
- "submit_controls": [{"selector": "#submit"}],
- }
- ],
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML, "parameters": values},
- ctx,
- allow_missing_credentials=True,
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="submit-request",
+ outcome="the service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ level="run",
+ )
+ ]
)
-
- assert result["ok"] is True
- assert ctx.spine_imposition_owned_attempt is True
- assert enforcement_module._scouted_spine_open_obligation(ctx) == []
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
- code = str(_single_code_block(parse_workflow_yaml(ctx.workflow_yaml))["code"])
- for key, selector in selectors.items():
- assert code.count(f'page.locator("{selector}").fill(str({key}))') == 1
- assert 'page.locator("#submit").click()' in code
-
- @pytest.mark.asyncio
- async def test_recut_fill_captured_after_terminal_is_not_imposed(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- source_url = "https://example.com/form"
- runtime_parameters = {"business_name": "Example Co", "contact_email": "[email protected]"}
- rejected_yaml = _yaml(
+ ctx.synthesized_block_offered = True
+ ctx.spine_imposition_owned_attempt = True
+ bound_yaml = _yaml(
"""
title: Submit reusable service request
workflow_definition:
parameters:
- {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
+ - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
blocks:
- block_type: code
label: submit_service_request
- parameter_keys: []
+ parameter_keys: [business_name, service_address]
code: |
+ await page.locator("#business").fill(str(business_name))
+ await page.locator("#address").fill(str(service_address))
await page.locator("#submit").click()
"""
)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#company",
- "source_url": source_url,
- "typed_length": 10,
- "trajectory_index": 0,
- },
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 1},
- {
- "tool_name": "type_text",
- "selector": "#email",
- "source_url": source_url,
- "typed_length": 15,
- "trajectory_index": 2,
- },
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [
- {"selector": "#company", "value": runtime_parameters["business_name"]},
- {"selector": "#email", "value": runtime_parameters["contact_email"]},
- ],
- "submit_controls": [{"selector": "#submit"}],
- }
- ],
- }
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_maybe_impose_synthesized_code_block",
+ lambda *_args, **_kwargs: workflow_update_module._SynthesizedCodeImpositionResult(
+ workflow_yaml=bound_yaml,
+ substitutions={"submit_service_request": "synthesized"},
+ ),
+ )
result = await _update_workflow(
- {"workflow_yaml": rejected_yaml, "parameters": runtime_parameters},
+ {"workflow_yaml": bound_yaml},
ctx,
allow_missing_credentials=True,
)
- assert result["ok"] is False
- directive = ctx.last_code_authoring_repair_context
- assert directive is not None
- assert directive.parameter_binding_directive is not None
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == bound_yaml
@pytest.mark.asyncio
- async def test_keyless_draft_persists_but_run_blocks_is_rejected_when_criteria_dark(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_bound_draft_run_blocks_dispatches_when_criteria_dark(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(allow_run_blocks=True)
- keyless_yaml = _yaml(
+ bound_yaml = _yaml(
"""
title: Download monthly statement
workflow_definition:
parameters:
- {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
- - {parameter_type: workflow, key: billing_start_date, workflow_parameter_type: string}
- - {parameter_type: workflow, key: billing_end_date, workflow_parameter_type: string}
blocks:
- - block_type: text_prompt
- label: normalize_invoice_inputs
- prompt: "Return these inputs unchanged: account number 100245, dates 2026-05-01 to 2026-05-31."
- block_type: code
label: fetch_invoice_pdf
- parameter_keys: []
+ parameter_keys: [account_number]
code: |
+ await page.locator("#account").fill(str(account_number))
await page.locator("#current-statement-row").click()
- await page.locator('a[href="/statements/100245_2026-05.pdf"]').click()
"""
)
persist_result = await _update_workflow(
- {"workflow_yaml": keyless_yaml},
- ctx,
- allow_missing_credentials=True,
- )
- assert persist_result["ok"] is True
-
- dispatched: list[object] = []
-
- async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
- dispatched.append(_params)
- return {"ok": True}
-
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- run_result = json.loads(
- await tools_module.run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug"),
- json.dumps({"block_labels": ["fetch_invoice_pdf"]}),
- )
- )
-
- assert run_result["ok"] is False
- assert run_result["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert run_result["data"]["unreferenced_parameter_keys"] == [
- "account_number",
- "billing_end_date",
- "billing_start_date",
- ]
- assert dispatched == []
-
- @pytest.mark.asyncio
- async def test_keyless_run_blocks_dispatches_in_log_only_shadow_mode(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(allow_run_blocks=True)
- keyless_yaml = _yaml(
- """
- title: Download monthly statement
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
- blocks:
- - block_type: code
- label: fetch_invoice_pdf
- parameter_keys: []
- code: |
- await page.locator("#current-statement-row").click()
- """
- )
- persist_result = await _update_workflow(
- {"workflow_yaml": keyless_yaml},
+ {"workflow_yaml": bound_yaml},
ctx,
allow_missing_credentials=True,
)
assert persist_result["ok"] is True
- ctx.author_time_gate_log_only_ids = {workflow_update_module.DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}
dispatched: list[object] = []
async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
dispatched.append(_params)
- return {"ok": True}
+ return {"ok": True, "run_status": "completed"}
monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
@@ -2346,2915 +1800,3015 @@ class TestCodeRepairProgressClassification:
)
assert dispatched != []
- assert [event.gate_id for event in ctx.author_time_gate_ablation_events] == [
- workflow_update_module.DEFINITION_CONTRACT_UNSATISFIED_GATE_ID
- ]
@pytest.mark.asyncio
- async def test_synthesized_business_tour_stays_rejected_when_definition_gate_is_log_only(
+ async def test_public_update_and_run_consumes_existing_page_binding_before_definition_reject(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="request-id",
- outcome="the request id is output",
- level="run",
- requested_output_floor_rekeyed=True,
- floor_rekeyed_from_path="output.request_id",
- ),
- CompletionCriterion(
- id="submit-request",
- outcome="the service request is submitted",
- kind="terminal_action",
- terminal_action_family="request",
- level="run",
+ ctx = _definition_contract_ctx()
+ _enable_imposition(ctx)
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
+ source_url = "https://example.com/utility"
+ runtime_parameters = {
+ "business_name": "Example Co",
+ "contact_email": "[email protected]",
+ "service_address": "1 Example Way",
+ "desired_start_date": "2026-08-01",
+ }
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#company",
+ "source_url": source_url,
+ "typed_length": 10,
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "type_text",
+ "selector": "#email",
+ "source_url": source_url,
+ "typed_length": 15,
+ "trajectory_index": 1,
+ },
+ {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 2},
+ ]
+ ctx.composition_page_evidence = {
+ "source_tool": "inspect_page_for_composition",
+ "current_url": source_url,
+ "forms": [
+ {
+ "fields": [
+ {"selector": "#company", "value": runtime_parameters["business_name"]},
+ {"selector": "#email", "value": runtime_parameters["contact_email"]},
+ {"selector": "#address", "value": runtime_parameters["service_address"]},
+ {"selector": "#date", "value": runtime_parameters["desired_start_date"]},
+ ],
+ "submit_controls": [{"selector": "#submit"}],
+ }
+ ],
+ }
+
+ async def _no_prior_definition(_ctx: CopilotContext) -> None:
+ return None
+
+ dispatched_candidates: list[str] = []
+
+ async def _run(_params: object, run_ctx: CopilotContext, **_kwargs: object) -> dict[str, object]:
+ dispatched_candidates.append(run_ctx.workflow_yaml or "")
+ return {"ok": False, "error": "stop after exact candidate"}
+
+ async def _verification(*_args: object, **_kwargs: object) -> None:
+ return None
+
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", _no_prior_definition)
+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *_args: (["submit_service_request"], {}, None))
+ monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *_args: None)
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
+ monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", _verification)
+ monkeypatch.setattr(tools_module, "_tool_visible_result_after_completion_verification", lambda _c, r, _v: r)
+
+ with capture_logs() as logs:
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps(
+ {
+ "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
+ "block_labels": ["submit_service_request"],
+ "parameters": runtime_parameters,
+ }
),
- ]
- )
- ctx.synthesized_block_offered = True
- ctx.spine_imposition_owned_attempt = True
- ctx.author_time_gate_log_only_ids = {workflow_update_module.DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}
- monkeypatch.setattr(
- workflow_update_module,
- "_maybe_impose_synthesized_code_block",
- lambda *_args, **_kwargs: workflow_update_module._SynthesizedCodeImpositionResult(
- workflow_yaml=_UNREFERENCED_DEFINITION_YAML,
- substitutions={"submit_service_request": "synthesized"},
- ),
- )
+ )
- result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML},
- ctx,
- allow_missing_credentials=True,
+ bound = next(log for log in logs if log["event"] == "copilot recorded outcome submit rung bound")
+ consumed = next(
+ log for log in logs if log["event"] == "copilot recorded outcome binding consumed by synthesizer"
)
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "synthesized_business_input_floor_unsatisfied"
- assert result["data"]["unreferenced_parameter_keys"] == [
+ assert consumed["binding_fingerprints"] == [bound["binding_fingerprint"]]
+ assert dispatched_candidates == [ctx.workflow_yaml]
+ assert json.loads(result)["error"] == "stop after exact candidate"
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ block = _single_code_block(parsed)
+ code = str(block["code"])
+ assert block["parameter_keys"] == [
"business_name",
"contact_email",
"desired_start_date",
"service_address",
]
- assert ctx.workflow_yaml == ""
- assert ctx.synthesized_block_offered is False
- assert ctx.synthesized_block_offered_trajectory_len == 0
- assert ctx.synthesized_block_offered_goal_complete is False
- assert ctx.spine_imposition_owned_attempt is False
- assert ctx.author_time_gate_ablation_events == []
+ assert code.count('page.locator("#company").fill(str(business_name))') == 1
+ assert code.count('page.locator("#email").fill(str(contact_email))') == 1
+ assert code.count('page.locator("#address").fill(str(service_address))') == 1
+ assert code.count('page.locator("#date").fill(str(desired_start_date))') == 1
+ assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
@pytest.mark.asyncio
- async def test_synthesized_business_input_floor_keeps_required_inputs_when_retry_shrinks_draft(
+ async def test_public_initial_preflight_preserves_code_safety_precedence_for_untagged_inputs(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="submit-request",
- outcome="the service request is submitted",
- kind="terminal_action",
- terminal_action_family="request",
- level="run",
- )
- ]
+ completion_criteria=[CompletionCriterion(id="run", outcome="The service request is submitted.")]
)
- ctx.synthesized_block_offered = True
- ctx.spine_imposition_owned_attempt = True
- monkeypatch.setattr(
- workflow_update_module,
- "_maybe_impose_synthesized_code_block",
- lambda *_args, **_kwargs: workflow_update_module._SynthesizedCodeImpositionResult(
- workflow_yaml=_UNREFERENCED_DEFINITION_YAML,
- substitutions={"submit_service_request": "synthesized"},
- ),
+ unsafe_yaml = _UNREFERENCED_DEFINITION_YAML.replace(
+ 'await page.locator("#open").click()',
+ 'import requests\n await page.locator("#open").click()',
)
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- first_result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML},
- ctx,
- allow_missing_credentials=True,
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": unsafe_yaml, "block_labels": ["submit_service_request"]}),
)
- assert first_result["ok"] is False
- assert ctx.synthesized_business_required_parameter_keys == {
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- }
+ parsed = json.loads(result)
+ assert parsed["ok"] is False
+ assert parsed.get("data", {}).get("reason_code") != "definition_contract_unsatisfied"
+ assert "sandbox safety check" in parsed["error"]
- narrowed_yaml = _yaml(
+ def test_definition_preflight_drops_bindings_from_non_fallthrough_branch(self) -> None:
+ ctx = _definition_contract_ctx()
+ terminal_branch_yaml = _yaml(
"""
- title: Start reusable service request
+ title: Submit reusable service request
workflow_definition:
parameters:
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
+ - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
label: submit_service_request
- parameter_keys: [service_address]
+ parameter_keys: [business_name]
code: |
- await page.locator("#address").fill(str(service_address))
+ if page.url.endswith("/done"):
+ company = business_name
+ if True:
+ return {"already_done": True}
+ else:
+ company = "fixed"
+ await page.locator("#company").fill(company)
"""
)
- ctx.synthesized_block_offered = True
- ctx.spine_imposition_owned_attempt = True
- monkeypatch.setattr(
- workflow_update_module,
- "_maybe_impose_synthesized_code_block",
- lambda *_args, **_kwargs: workflow_update_module._SynthesizedCodeImpositionResult(
- workflow_yaml=narrowed_yaml,
- substitutions={"submit_service_request": "synthesized"},
- ),
- )
- second_result = await _update_workflow(
- {"workflow_yaml": narrowed_yaml},
- ctx,
- allow_missing_credentials=True,
- )
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, terminal_branch_yaml)
- assert second_result["ok"] is False
- assert second_result["data"]["reason_code"] == "synthesized_business_input_floor_unsatisfied"
- assert second_result["data"]["unreferenced_parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- ]
- assert ctx.workflow_yaml == ""
+ assert result is not None
+ assert result.unreferenced_parameter_keys == ("business_name",)
- @pytest.mark.asyncio
- async def test_synthesized_business_input_floor_allows_fully_bound_draft(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="submit-request",
- outcome="the service request is submitted",
- kind="terminal_action",
- terminal_action_family="request",
- level="run",
- )
- ]
+ @pytest.mark.parametrize(
+ "terminal_test", ["if 1:\n return {'already_done': True}", "while True:\n return {'already_done': True}"]
+ )
+ def test_definition_preflight_drops_bindings_from_constant_terminal_control(self, terminal_test: str) -> None:
+ ctx = _definition_contract_ctx()
+ indented_terminal = textwrap.indent(terminal_test, " ")
+ workflow_yaml = _yaml(
+ "title: Submit reusable service request\n"
+ "workflow_definition:\n"
+ " parameters:\n"
+ " - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: submit_service_request\n"
+ " parameter_keys: [business_name]\n"
+ " code: |\n"
+ " if page.url.endswith('/done'):\n"
+ " company = business_name\n"
+ f"{indented_terminal}\n"
+ " else:\n"
+ " company = 'fixed'\n"
+ " await page.locator('#company').fill(company)\n"
)
- ctx.synthesized_block_offered = True
- ctx.spine_imposition_owned_attempt = True
- bound_yaml = _yaml(
+
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
+
+ assert result is not None
+ assert result.unreferenced_parameter_keys == ("business_name",)
+
+ def test_definition_preflight_ignores_break_in_unreachable_constant_branch(self) -> None:
+ ctx = _definition_contract_ctx()
+ workflow_yaml = _yaml(
"""
title: Submit reusable service request
workflow_definition:
parameters:
- {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
blocks:
- block_type: code
label: submit_service_request
- parameter_keys: [business_name, service_address]
+ parameter_keys: [business_name]
code: |
- await page.locator("#business").fill(str(business_name))
- await page.locator("#address").fill(str(service_address))
- await page.locator("#submit").click()
+ if page.url.endswith("/done"):
+ company = business_name
+ while True:
+ if False:
+ break
+ return {"already_done": True}
+ else:
+ company = "fixed"
+ await page.locator("#company").fill(company)
"""
)
- monkeypatch.setattr(
- workflow_update_module,
- "_maybe_impose_synthesized_code_block",
- lambda *_args, **_kwargs: workflow_update_module._SynthesizedCodeImpositionResult(
- workflow_yaml=bound_yaml,
- substitutions={"submit_service_request": "synthesized"},
- ),
- )
- result = await _update_workflow(
- {"workflow_yaml": bound_yaml},
- ctx,
- allow_missing_credentials=True,
- )
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
- assert result["ok"] is True
- assert ctx.workflow_yaml == bound_yaml
+ assert result is not None
+ assert result.unreferenced_parameter_keys == ("business_name",)
- @pytest.mark.asyncio
- async def test_bound_draft_run_blocks_dispatches_when_criteria_dark(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(allow_run_blocks=True)
- bound_yaml = _yaml(
+ def test_definition_preflight_ignores_break_overridden_by_terminal_finally(self) -> None:
+ ctx = _definition_contract_ctx()
+ workflow_yaml = _yaml(
"""
- title: Download monthly statement
+ title: Submit reusable service request
workflow_definition:
parameters:
- - {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
+ - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: fetch_invoice_pdf
- parameter_keys: [account_number]
+ label: submit_service_request
+ parameter_keys: [business_name]
code: |
- await page.locator("#account").fill(str(account_number))
- await page.locator("#current-statement-row").click()
+ if page.url.endswith("/done"):
+ company = business_name
+ while True:
+ try:
+ break
+ finally:
+ return {"already_done": True}
+ else:
+ company = "fixed"
+ await page.locator("#company").fill(company)
"""
)
- persist_result = await _update_workflow(
- {"workflow_yaml": bound_yaml},
- ctx,
- allow_missing_credentials=True,
- )
- assert persist_result["ok"] is True
-
- dispatched: list[object] = []
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
- async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
- dispatched.append(_params)
- return {"ok": True, "run_status": "completed"}
+ assert result is not None
+ assert result.unreferenced_parameter_keys == ("business_name",)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- await tools_module.run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug"),
- json.dumps({"block_labels": ["fetch_invoice_pdf"]}),
+ @pytest.mark.parametrize(
+ ("code", "expected_unreferenced"),
+ [
+ (
+ "values = [business_name for business_name in ['fixed']]\n"
+ "await page.locator('#company').fill(values[0])",
+ ["business_name"],
+ ),
+ (
+ "async def fill_company():\n"
+ " await page.locator('#company').fill(str(business_name))\n"
+ "await fill_company()",
+ [],
+ ),
+ ],
+ )
+ def test_definition_preflight_honors_python_scopes(self, code: str, expected_unreferenced: list[str]) -> None:
+ ctx = _definition_contract_ctx()
+ indented_code = textwrap.indent(code, " ")
+ scoped_yaml = _yaml(
+ "title: Submit reusable service request\n"
+ "workflow_definition:\n"
+ " parameters:\n"
+ " - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: submit_service_request\n"
+ " parameter_keys: [business_name]\n"
+ " code: |\n"
+ f"{indented_code}\n"
)
- assert dispatched != []
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, scoped_yaml)
- @pytest.mark.asyncio
- async def test_recut_ambiguous_field_binding_returns_value_free_typed_directive(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ if expected_unreferenced:
+ assert result is not None
+ assert list(result.unreferenced_parameter_keys) == expected_unreferenced
+ else:
+ assert result is None
+
+ @pytest.mark.parametrize(
+ ("template", "is_consumed"),
+ [
+ ('{{ "business_name" }}', False),
+ ("{% if business_name %}/customer{% endif %}", True),
+ ],
+ )
+ def test_definition_preflight_uses_jinja_semantics(self, template: str, is_consumed: bool) -> None:
ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": _UNREFERENCED_DEFINITION_YAML}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- source_url = "https://example.com/utility"
- runtime_parameters = {
- "business_name": "duplicate-value",
- "contact_email": "duplicate-value",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- }
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 0}
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [
- {"selector": "#company", "value": "duplicate-value"},
- {"selector": "#email", "value": "duplicate-value"},
- {"selector": "#address", "value": "1 Example Way"},
- {"selector": "#date", "value": "2026-08-01"},
- ],
- "submit_controls": [{"selector": "#submit"}],
- }
- ],
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": _UNREFERENCED_DEFINITION_YAML, "parameters": runtime_parameters},
- ctx,
+ templated_yaml = _yaml(
+ f"""
+ title: Submit reusable service request
+ workflow_definition:
+ parameters:
+ - {{parameter_type: workflow, key: business_name, workflow_parameter_type: string}}
+ blocks:
+ - block_type: navigation
+ label: open_customer
+ url: {json.dumps(template)}
+ """
)
- assert result["ok"] is False
- repair_context = result["data"]["authoring_repair_context"]
- directive = repair_context["parameter_binding_directive"]
- assert directive["structural_key"] == ctx.recorded_outcome_binding_constraint.repeated_structural_key
- assert {tuple(candidate.values()) for candidate in directive["candidates"]} >= {
- ("business_name", "#company"),
- ("business_name", "#email"),
- ("contact_email", "#company"),
- ("contact_email", "#email"),
- }
- assert "duplicate-value" not in str(directive)
- assert source_url not in str(directive)
- assert ctx.turn_halt is None
+ result = workflow_update_module._definition_plane_preflight_reject(ctx, templated_yaml)
+
+ assert (result is None) is is_consumed
@pytest.mark.asyncio
- async def test_recut_typed_directive_is_consumed_by_next_exact_authored_binding(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ async def test_definition_preflight_defers_to_repairable_syntax_error(self) -> None:
ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- source_url = "https://example.com/directory"
- runtime_parameters = {
- "address_city_county_or_zip_code": "duplicate-value",
- "contact_name": "duplicate-value",
- }
- rejected_yaml = _yaml(
+ malformed_yaml = _yaml(
"""
- title: Search directory
+ title: Submit reusable service request
workflow_definition:
parameters:
- - parameter_type: workflow
- key: address_city_county_or_zip_code
- workflow_parameter_type: string
- - parameter_type: workflow
- key: contact_name
- workflow_parameter_type: string
+ - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: search_directory
- parameter_keys: []
+ label: submit_service_request
+ parameter_keys: [business_name]
code: |
- await page.locator("#submit").click()
+ await page.locator("#company").fill(str(business_name)
"""
)
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 0}
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [
- {"selector": "#search_location", "value": "duplicate-value"},
- {"selector": "#contact_name", "value": "duplicate-value"},
- ],
- "submit_controls": [{"selector": "#submit"}],
- }
- ],
- }
- first_result = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": rejected_yaml,
- "block_labels": ["search_directory"],
- "parameters": runtime_parameters,
- }
- ),
- )
- )
+ result = await _update_workflow({"workflow_yaml": malformed_yaml}, ctx)
- assert first_result["ok"] is False
- directive = ctx.last_code_authoring_repair_context
- assert directive is not None
- assert directive.parameter_binding_directive is not None
- authored_yaml = rejected_yaml.replace(
- 'parameter_keys: []\n code: |\n await page.locator("#submit").click()',
- "parameter_keys: [address_city_county_or_zip_code, contact_name]\n"
- " code: |\n"
- ' await page.locator("#search_location").fill(str(address_city_county_or_zip_code))\n'
- ' await page.locator("#contact_name").fill(str(contact_name))\n'
- ' await page.locator("#submit").click()',
- )
- stale_source_url = "https://other.example/directory"
- ctx.scout_trajectory[0]["source_url"] = stale_source_url
- ctx.composition_page_evidence["current_url"] = stale_source_url
+ assert result["ok"] is False
+ assert "not valid Python" in result["error"]
+ assert result.get("data", {}).get("reason_code") != "definition_contract_unsatisfied"
- stale_result = await _update_workflow(
- {"workflow_yaml": authored_yaml, "parameters": runtime_parameters},
- ctx,
- allow_missing_credentials=True,
+ @pytest.mark.asyncio
+ async def test_code_safety_seam_reject_carries_progress_surface_kind(self) -> None:
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": _REQUESTS_IMPORT_CODE_YAML}, ctx)
+ assert result["ok"] is False
+ assert result["data"]["surface_kind"] == "code_repair_progress"
+ assert result["data"]["progress_text"]
+ # The substantive copy is unchanged; the progress text is a separate carrier.
+ assert result["user_facing_summary"] == (
+ "I need to adjust the workflow's code so it can run safely before testing."
)
-
- assert stale_result["ok"] is False
- assert "stored parameter binding directive is stale" in stale_result["error"]
- ctx.scout_trajectory[0]["source_url"] = source_url
- ctx.composition_page_evidence["current_url"] = source_url
-
- with capture_logs() as logs:
- second_result = await _update_workflow(
- {"workflow_yaml": authored_yaml, "parameters": runtime_parameters},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert second_result["ok"] is True
- assert any(log["event"] == "copilot authoring parameter binding directive consumed" for log in logs)
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["address_city_county_or_zip_code", "contact_name"]
- code = str(block["code"])
- assert code.count('page.locator("#search_location").fill(str(address_city_county_or_zip_code))') == 1
- assert code.count('page.locator("#contact_name").fill(str(contact_name))') == 1
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
+ assert result["data"]["progress_text"] != result["user_facing_summary"]
@pytest.mark.asyncio
- async def test_public_update_and_run_consumes_regrounding_in_synthesis(
+ async def test_generated_code_preflight_string_reject_is_not_authoritative(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- initial_arguments = json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
- }
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_code_block_safety_errors",
+ lambda workflow_yaml, prior_yaml: [
+ "Code block `search_registry` failed the generated-code preflight check: "
+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."
+ ],
)
- for _ in range(2):
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- initial_arguments,
- )
- assert json.loads(result)["data"]["reason_code"] == "definition_contract_unsatisfied"
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.phase == "author_time_reject"
- source_url = "https://example.com/utility"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#open", "source_url": source_url, "trajectory_index": 0},
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 1},
- ]
- runtime_parameters = {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- }
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "page_title": "Utility request",
- "forms": [
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "code_safety_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is False
+ assert ctx.latest_recorded_build_test_outcome.structural_key is None
+
+ def test_code_safety_payload_keeps_typed_security_error_authoritative(self) -> None:
+ payload = _code_safety_reject_payload(
+ [
+ CodeBlockSecurityError(
+ "Code block `search_registry` failed the Copilot code security check: page.request is not allowed.",
+ block_label="search_registry",
+ reason_code="AUTHOR_PAGE_REQUEST",
+ surface="page.request",
+ )
+ ]
+ )
+
+ assert payload == {
+ "code_safety_errors": [
{
- "fields": [
- {"selector": "#company", "value": runtime_parameters["business_name"]},
- {"selector": "#email", "value": runtime_parameters["contact_email"]},
- {"selector": "#address", "value": runtime_parameters["service_address"]},
- {"selector": "#date", "value": runtime_parameters["desired_start_date"]},
- ],
- "submit_controls": [{"selector": "#submit"}],
+ "block_label": "search_registry",
+ "reason_code": "AUTHOR_PAGE_REQUEST",
+ "surface": "page.request",
}
- ],
- "navigation_targets": [],
- "result_containers": [],
- "challenge_controls": [],
- "anti_bot_indicators": [],
- "observed_after_workflow_run": False,
+ ]
}
- async def _no_prior_definition(_ctx: CopilotContext) -> None:
- return None
-
- dispatched_candidates: list[str] = []
+ def test_code_safety_payload_rejects_string_only_preflight_authority(self) -> None:
+ payload = _code_safety_reject_payload(
+ [
+ "Code block `search_registry` failed the generated-code preflight check: "
+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."
+ ]
+ )
- async def _run(_params: object, run_ctx: CopilotContext, **_kwargs: object) -> dict[str, object]:
- dispatched_candidates.append(run_ctx.workflow_yaml or "")
- return {"ok": False, "error": "stop after exact candidate"}
+ assert payload is None
- async def _verification(*_args: object, **_kwargs: object) -> None:
- return None
+ @pytest.mark.asyncio
+ async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
+ changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ authored_structure_signature=signature,
+ )
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", _no_prior_definition)
- monkeypatch.setattr(tools_module, "_plan_frontier", lambda *_args: (["submit_service_request"], {}, None))
- monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *_args: None)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
- monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", _verification)
- monkeypatch.setattr(tools_module, "_tool_visible_result_after_completion_verification", lambda _c, r, _v: r)
- with capture_logs() as logs:
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
- "parameters": runtime_parameters,
- }
- ),
- )
+ result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)
- bound = next(log for log in logs if log["event"] == "copilot recorded outcome submit rung bound")
- consumed = next(
- log for log in logs if log["event"] == "copilot recorded outcome binding consumed by synthesizer"
- )
- assert consumed["binding_fingerprints"] == [bound["binding_fingerprint"]]
- code = str(_single_code_block(parse_workflow_yaml(ctx.workflow_yaml))["code"])
- assert 'page.locator("#company").fill(str(business_name))' in code
- assert 'page.locator("#date").fill(str(desired_start_date))' in code
- assert dispatched_candidates == [ctx.workflow_yaml]
- assert workflow_update_module._definition_plane_preflight_reject(ctx, dispatched_candidates[0]) is None
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == changed_yaml
+ assert ctx.has_staged_proposal is True
@pytest.mark.asyncio
- async def test_public_update_and_run_consumes_existing_page_binding_before_definition_reject(
+ async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- source_url = "https://example.com/utility"
- runtime_parameters = {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- }
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#company",
- "source_url": source_url,
- "typed_length": 10,
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#email",
- "source_url": source_url,
- "typed_length": 15,
- "trajectory_index": 1,
- },
- {"tool_name": "click", "selector": "#submit", "source_url": source_url, "trajectory_index": 2},
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": source_url,
- "forms": [
- {
- "fields": [
- {"selector": "#company", "value": runtime_parameters["business_name"]},
- {"selector": "#email", "value": runtime_parameters["contact_email"]},
- {"selector": "#address", "value": runtime_parameters["service_address"]},
- {"selector": "#date", "value": runtime_parameters["desired_start_date"]},
- ],
- "submit_controls": [{"selector": "#submit"}],
- }
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ }
}
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ npi = "1234567890"
+ locations = [{{"address": "123 Main St"}}]
+ """
+ )
- async def _no_prior_definition(_ctx: CopilotContext) -> None:
- return None
-
- dispatched_candidates: list[str] = []
-
- async def _run(_params: object, run_ctx: CopilotContext, **_kwargs: object) -> dict[str, object]:
- dispatched_candidates.append(run_ctx.workflow_yaml or "")
- return {"ok": False, "error": "stop after exact candidate"}
-
- async def _verification(*_args: object, **_kwargs: object) -> None:
- return None
-
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", _no_prior_definition)
- monkeypatch.setattr(tools_module, "_plan_frontier", lambda *_args: (["submit_service_request"], {}, None))
- monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *_args: None)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
- monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", _verification)
- monkeypatch.setattr(tools_module, "_tool_visible_result_after_completion_verification", lambda _c, r, _v: r)
-
- with capture_logs() as logs:
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
- "parameters": runtime_parameters,
- }
- ),
- )
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- bound = next(log for log in logs if log["event"] == "copilot recorded outcome submit rung bound")
- consumed = next(
- log for log in logs if log["event"] == "copilot recorded outcome binding consumed by synthesizer"
- )
- assert consumed["binding_fingerprints"] == [bound["binding_fingerprint"]]
- assert dispatched_candidates == [ctx.workflow_yaml]
- assert json.loads(result)["error"] == "stop after exact candidate"
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- block = _single_code_block(parsed)
- code = str(block["code"])
- assert block["parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
- assert code.count('page.locator("#company").fill(str(business_name))') == 1
- assert code.count('page.locator("#email").fill(str(contact_email))') == 1
- assert code.count('page.locator("#address").fill(str(service_address))') == 1
- assert code.count('page.locator("#date").fill(str(desired_start_date))') == 1
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
@pytest.mark.asyncio
- async def test_recut_unbindable_regrounding_halts_before_reauthor(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_authoritative_outcome_not_demonstrated_accepts_canonical_output_path_candidate_with_sibling(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": _UNREFERENCED_DEFINITION_YAML}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- ctx.blocker_signal = ctx.turn_halt = ctx.turn_ownership = None
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#submit", "source_url": "https://example.com/utility"}
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": "https://example.com/utility",
- "forms": [],
+ ctx = _code_only_ctx()
+ label = "submit_form_and_extract_confirmation"
+ schema = workflow_update_module._schema_template_text_for_required_paths({"output.confirmation_number"})
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}
+ ],
+ }
}
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
- synthesize = Mock(side_effect=AssertionError("must halt before an identical re-author"))
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block_with_extraction", synthesize)
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(
- _UNREFERENCED_DEFINITION_YAML,
- ctx,
- {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- },
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {
+ "output_path": "output.confirmation_number",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Form submission
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"output": {{"confirmation_number": "ABC-123", "account_number": "100245"}}}}
+ """
)
- assert result.workflow_yaml == _UNREFERENCED_DEFINITION_YAML
- assert result.violations
- synthesize.assert_not_called()
- assert ctx.blocker_signal.extra["unresolved_parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
- assert "safely connect the current page fields" in ctx.blocker_signal.user_facing_reason
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml is not None
+ assert "confirmation_number" in ctx.workflow_yaml
+ assert ctx.code_artifact_metadata[label]["claimed_outcomes"][0]["extraction_schema"] == schema
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "outcome_not_demonstrated"
@pytest.mark.asyncio
- async def test_public_update_and_run_halts_active_author_reject_before_third_identical_author(
+ async def test_authoritative_outcome_not_demonstrated_allows_unprovable_dynamic_output_shape(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- ctx = _definition_contract_ctx()
- _enable_imposition(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- arguments = json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
- "parameters": {
- "business_name": "Example Co",
- "contact_email": "[email protected]",
- "service_address": "1 Example Way",
- "desired_start_date": "2026-08-01",
- },
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
}
- )
- for _ in range(2):
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- arguments,
- )
- assert json.loads(result)["data"]["reason_code"] == "definition_contract_unsatisfied"
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.phase == "author_time_reject"
- active_structural_key = requirement.structural_key
- ctx.blocker_signal = ctx.turn_halt = ctx.turn_ownership = None
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#submit", "source_url": "https://example.com/utility"}
- ]
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": "https://example.com/utility",
- "forms": [],
- "observed_after_workflow_run": False,
}
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- arguments,
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ result = {{}}
+ key = "npi"
+ value = await page.locator("#npi").inner_text(timeout=5000)
+ result[key] = value
+ return result
+ """
)
- assert json.loads(result)["ok"] is False
- assert ctx.latest_recorded_build_test_outcome.structural_key == active_structural_key
- assert ctx.blocker_signal.internal_reason_code == "definition_contract_unsatisfied"
- assert ctx.blocker_signal.extra["unresolved_parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind.value == "definition_contract_unsatisfied"
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
@pytest.mark.asyncio
- async def test_definition_reject_stays_repairable_at_both_exact_gates(
+ async def test_authoritative_outcome_not_demonstrated_allows_literal_list_of_output_dicts(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- update_ctx = _definition_contract_ctx()
-
- update_result = await _update_workflow({"workflow_yaml": _UNREFERENCED_DEFINITION_YAML}, update_ctx)
-
- assert update_result["ok"] is False
- assert update_result["data"]["unreferenced_parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
- assert update_ctx.blocker_signal is None
- assert update_ctx.turn_halt is None
-
- dispatch_ctx = _definition_contract_ctx()
- dispatch_result = workflow_update_module._metadata_contract_run_preflight_reject(
- dispatch_ctx,
- _UNREFERENCED_DEFINITION_YAML,
- [],
- )
-
- assert dispatch_result is not None
- assert dispatch_result["ok"] is False
- assert dispatch_result["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert dispatch_ctx.blocker_signal is None
- assert dispatch_ctx.turn_halt is None
-
- @pytest.mark.asyncio
- async def test_public_update_and_run_definition_reject_returns_to_repair_loop(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _definition_contract_ctx()
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- arguments = json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
}
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
)
-
- for _ in range(2):
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- arguments,
- )
-
- parsed = json.loads(result)
- assert parsed["ok"] is False
- assert parsed["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert ctx.blocker_signal is None
- assert ctx.turn_halt is None
-
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.latest_recorded_build_test_outcome.reason_code == "definition_contract_unsatisfied"
- assert ctx.recorded_outcome_grounding_requirement is not None
-
- third_result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- arguments,
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return [{{"npi": "123"}}]
+ """
)
- assert json.loads(third_result)["ok"] is False
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
-
- @pytest.mark.asyncio
- @pytest.mark.parametrize("grounded_binding_available", [False, True])
- async def test_public_update_and_run_rejects_unbound_declared_inputs_without_definition_tag(
- self,
- monkeypatch: pytest.MonkeyPatch,
- grounded_binding_available: bool,
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="run", outcome="The service request is submitted.")]
- )
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- dispatched = False
- updated = False
- real_update = tools_module._update_workflow
-
- async def _observed_update(*args: object, **kwargs: object) -> dict[str, object]:
- nonlocal updated
- result = await real_update(*args, **kwargs)
- updated = bool(result.get("ok"))
- return result
-
- async def _unexpected_dispatch(*_args: object, **_kwargs: object) -> dict[str, object]:
- nonlocal dispatched
- dispatched = True
- return {"ok": True}
-
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _unexpected_dispatch)
- monkeypatch.setattr(tools_module, "_update_workflow", _observed_update)
- if grounded_binding_available:
- monkeypatch.setattr(
- workflow_update_module,
- "_authoring_parameter_binding_resolution_for_definition",
- lambda *_args, **_kwargs: SimpleNamespace(snapshot=Mock(), directive=None),
- )
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": _UNREFERENCED_DEFINITION_YAML,
- "block_labels": ["submit_service_request"],
- }
- ),
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
)
- parsed = json.loads(result)
- assert parsed["ok"] is False
- assert parsed["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert parsed["data"]["unreferenced_parameter_keys"] == [
- "business_name",
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
- assert updated is True
- assert dispatched is False
-
@pytest.mark.asyncio
- async def test_public_initial_preflight_preserves_code_safety_precedence_for_untagged_inputs(
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_list_output_to_abstain(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="run", outcome="The service request is submitted.")]
- )
- unsafe_yaml = _UNREFERENCED_DEFINITION_YAML.replace(
- 'await page.locator("#open").click()',
- 'import requests\n await page.locator("#open").click()',
- )
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps({"workflow_yaml": unsafe_yaml, "block_labels": ["submit_service_request"]}),
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
)
-
- parsed = json.loads(result)
- assert parsed["ok"] is False
- assert parsed.get("data", {}).get("reason_code") != "definition_contract_unsatisfied"
- assert "sandbox safety check" in parsed["error"]
-
- def test_recut_dispatch_controls_preserve_passing_and_zero_input_candidates(self) -> None:
- ctx = _definition_contract_ctx()
- ctx.output_contract_actuation_by_signature["p5-advisory"] = OutputContractAdvisoryState.GRANTED
-
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, _UNREFERENCED_DEFINITION_YAML, [])
-
- assert result is not None
- assert result["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.blocker_signal is None
- assert ctx.turn_halt is None
-
- passing_yaml = _yaml(
- """
- title: Submit reusable service request
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name, contact_email, service_address, desired_start_date]
+ label: {label}
code: |
- await page.locator("#company").fill(str(business_name))
- await page.locator("#email").fill(str(contact_email))
- await page.locator("#address").fill(str(service_address))
- await page.locator("#date").fill(str(desired_start_date))
- await page.locator("#submit").click()
+ record = {{"npi": "123"}}
+ return [record]
"""
)
- passing_ctx = _definition_contract_ctx()
- assert workflow_update_module._metadata_contract_run_preflight_reject(passing_ctx, passing_yaml, []) is None
- zero_input_ctx = _code_only_ctx()
- zero_input_ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="run", outcome="The request is submitted.")]
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
)
- zero_input_yaml = _yaml(
- """
- title: Open status page
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_list_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
- parameters: []
blocks:
- block_type: code
- label: open_status
- parameter_keys: []
+ label: {label}
code: |
- await page.goto("https://example.com/status")
+ key = "npi"
+ return [{{key: "123"}}]
"""
)
- assert (
- workflow_update_module._metadata_contract_run_preflight_reject(zero_input_ctx, zero_input_yaml, []) is None
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
)
- def test_definition_preflight_rejects_partial_parameter_references(self) -> None:
- ctx = _definition_contract_ctx()
- partially_referenced_yaml = _yaml(
- """
- title: Submit reusable service request
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_local_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: {label}
code: |
- await page.locator("#company").fill(str(business_name))
- await page.locator("#submit").click()
+ key = "npi"
+ result = {{key: "123"}}
+ return result
"""
)
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, partially_referenced_yaml, [])
-
- assert result is not None
- assert result["data"]["unreferenced_parameter_keys"] == [
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- def test_definition_preflight_ignores_parameter_names_in_comments_and_string_literals(self) -> None:
- ctx = _definition_contract_ctx()
- lexical_only_yaml = _yaml(
- """
- title: Submit reusable service request
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
- blocks:
- - block_type: code
- label: submit_service_request
- parameter_keys: [business_name, contact_email, service_address, desired_start_date]
- code: |
- # contact_email and service_address are not executable references.
- note = "desired_start_date"
- await page.locator("#company").fill(str(business_name))
- await page.locator("#submit").click()
- """
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
)
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, lexical_only_yaml, [])
-
- assert result is not None
- assert result["data"]["unreferenced_parameter_keys"] == [
- "contact_email",
- "desired_start_date",
- "service_address",
- ]
-
- def test_definition_preflight_requires_live_parameter_dataflow(self) -> None:
- ctx = _definition_contract_ctx()
- dead_or_shadowed_yaml = _yaml(
- """
- title: Submit reusable service request
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_dynamic_key_local_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name, contact_email]
+ label: {label}
code: |
- if False:
- await page.locator("#company").fill(str(business_name))
- contact_email = "[email protected]"
- await page.locator("#email").fill(str(contact_email))
- await page.locator("#submit").click()
+ def build():
+ key = "npi"
+ result = {{key: "123"}}
+ return result
+
+ return build()
"""
)
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, dead_or_shadowed_yaml, [])
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result is not None
- assert result["data"]["unreferenced_parameter_keys"] == ["business_name", "contact_email"]
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
- def test_definition_preflight_accepts_parameter_alias_dataflow(self) -> None:
- ctx = _definition_contract_ctx()
- alias_yaml = _yaml(
- """
- title: Submit reusable service request
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_exact_missing_array_child_paths(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ child_paths = ["output.npi", "output.locations[].address", "output.locations[].status", "output.statuses"]
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"locations":{"type":"array","items":{"type":"object","properties":{'
+ '"address":{"type":"string"},"status":{"type":"string"}}}},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
+ {
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.locations[].status",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: {label}
code: |
- company = str(business_name)
- await page.locator("#company").fill(company)
- await page.locator("#submit").click()
+ return {{
+ "output": {{
+ "npi": "1234567890",
+ "locations": [{{"address": "Example location", "status": "active"}}],
+ "statuses": ["active"],
+ }}
+ }}
"""
)
- assert workflow_update_module._metadata_contract_run_preflight_reject(ctx, alias_yaml, []) is None
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- def test_definition_preflight_tracks_live_control_flow_dataflow(self) -> None:
- ctx = _definition_contract_ctx()
- control_flow_yaml = _yaml(
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
+
+ def test_single_output_contract_evaluator_reports_all_deficiency_classes(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
"""
- title: Submit reusable service request
+ title: Entry lookup
workflow_definition:
parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: value
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name, contact_email]
+ label: extract_record
+ parameter_keys:
+ - value
code: |
- company = business_name
- if page.url:
- company = company.strip()
- else:
- company = company.upper()
- for selector in ["#company"]:
- await page.locator(selector).fill(company)
+ _scout_entry_target = page.locator("#filter")
try:
- await page.locator("#email").fill(contact_email)
+ await _scout_entry_target.wait_for(state="visible", timeout=1000)
except Exception:
- raise
+ await page.goto("https://example.com/records", wait_until="domcontentloaded")
+ await _scout_entry_target.wait_for(state="visible")
+ await page.locator("#filter").fill(str(value))
+ await page.locator("#choose").click()
+ await page.wait_for_load_state("domcontentloaded")
+ return {"output": {"summary": "found"}}
"""
)
- assert workflow_update_module._metadata_contract_run_preflight_reject(ctx, control_flow_yaml, []) is None
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- def test_definition_preflight_drops_bindings_from_non_fallthrough_branch(self) -> None:
- ctx = _definition_contract_ctx()
- terminal_branch_yaml = _yaml(
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is True
+ assert evaluation.block_label == "extract_record"
+ assert evaluation.missing_metadata_paths == ["output.record_id"]
+ assert evaluation.missing_schema_paths == ["output.record_id"]
+ assert evaluation.missing_return_paths == ["output.record_id"]
+ assert evaluation.shape_violations == ["separated_spine_shape_required"]
+ assert evaluation.payload["satisfying_templates"]["code_artifact_metadata"]["block_label"] == "extract_record"
+ assert evaluation.payload["satisfying_templates"]["return_skeleton"] == (
+ 'return {"output": {"record_id": record_id}}'
+ )
+
+ def test_schema_template_derives_nested_array_paths_without_semantic_values(self) -> None:
+ schema = workflow_update_module._schema_template_for_required_paths(
+ {"output.npi", "output.locations[].address", "output.locations[].status"}
+ )
+
+ output = schema["properties"]["output"]
+ assert output["type"] == "object"
+ assert output["required"] == ["locations", "npi"]
+ locations = output["properties"]["locations"]
+ assert locations["type"] == "array"
+ assert locations["items"]["required"] == ["address", "status"]
+ assert locations["items"]["properties"]["address"] == {}
+ assert locations["items"]["properties"]["status"] == {}
+ assert "description" not in json.dumps(schema)
+ assert "example" not in json.dumps(schema)
+
+ def test_static_return_uncertainty_requires_typed_advisory_grant(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ schema = workflow_update_module._schema_template_text_for_required_paths({"output.record_id"})
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+ workflow_yaml = _yaml(
"""
- title: Submit reusable service request
+ title: Entry lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: extract_entry_output
code: |
- if page.url.endswith("/done"):
- company = business_name
- if True:
- return {"already_done": True}
- else:
- company = "fixed"
- await page.locator("#company").fill(company)
+ return {"output": build_output()}
"""
)
- result = workflow_update_module._definition_plane_preflight_reject(ctx, terminal_branch_yaml)
+ update_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=False
+ )
+ run_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=True
+ )
- assert result is not None
- assert result.unreferenced_parameter_keys == ("business_name",)
+ assert update_eval is not None
+ assert update_eval.missing_return_paths == ["output.record_id"]
+ assert update_eval.can_attempt_run is False
+ assert run_eval is not None
+ assert run_eval.missing_return_paths == ["output.record_id"]
+ assert run_eval.can_attempt_run is False
- @pytest.mark.parametrize(
- "terminal_test", ["if 1:\n return {'already_done': True}", "while True:\n return {'already_done': True}"]
- )
- def test_definition_preflight_drops_bindings_from_constant_terminal_control(self, terminal_test: str) -> None:
- ctx = _definition_contract_ctx()
- indented_terminal = textwrap.indent(terminal_test, " ")
- workflow_yaml = _yaml(
- "title: Submit reusable service request\n"
- "workflow_definition:\n"
- " parameters:\n"
- " - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: submit_service_request\n"
- " parameter_keys: [business_name]\n"
- " code: |\n"
- " if page.url.endswith('/done'):\n"
- " company = business_name\n"
- f"{indented_terminal}\n"
- " else:\n"
- " company = 'fixed'\n"
- " await page.locator('#company').fill(company)\n"
+ workflow_update_module._grant_output_contract_advisory_run(ctx, run_eval.canonical_signature)
+ granted_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=True
)
- result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
-
- assert result is not None
- assert result.unreferenced_parameter_keys == ("business_name",)
+ assert granted_eval is not None
+ assert granted_eval.missing_return_paths == []
+ assert granted_eval.payload["static_return_advisory_paths"] == ["output.record_id"]
+ assert granted_eval.payload["actuated_static_return_advisory"] is True
+ assert granted_eval.can_attempt_run is True
- def test_definition_preflight_ignores_break_in_unreachable_constant_branch(self) -> None:
- ctx = _definition_contract_ctx()
+ @pytest.mark.asyncio
+ async def test_run_path_allows_typed_advisory_declared_output_return_shape_gap(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
workflow_yaml = _yaml(
"""
- title: Submit reusable service request
+ title: Entry lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: extract_entry_output
code: |
- if page.url.endswith("/done"):
- company = business_name
- while True:
- if False:
- break
- return {"already_done": True}
- else:
- company = "fixed"
- await page.locator("#company").fill(company)
+ return "not a structured output"
"""
)
+ ctx.turn_id = "run-static-return-gap"
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths={"output.record_id"})
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
- result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
- assert result is not None
- assert result.unreferenced_parameter_keys == ("business_name",)
+ assert result["ok"] is True
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
+ "output.record_id"
+ ]
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["extraction_schema"]
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ block = _single_code_block(parsed)
+ schema = json.loads(block["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["record_id"] == {}
- def test_definition_preflight_ignores_break_overridden_by_terminal_finally(self) -> None:
- ctx = _definition_contract_ctx()
+ @pytest.mark.asyncio
+ async def test_run_path_persists_effective_output_contract_for_readback(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ required_paths = {"output.record_id"}
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
workflow_yaml = _yaml(
"""
- title: Submit reusable service request
+ title: Entry lookup
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: extract_entry_output
code: |
- if page.url.endswith("/done"):
- company = business_name
- while True:
- try:
- break
- finally:
- return {"already_done": True}
- else:
- company = "fixed"
- await page.locator("#company").fill(company)
+ return {"output": {"record_id": "ABC123"}}
"""
)
- result = workflow_update_module._definition_plane_preflight_reject(ctx, workflow_yaml)
+ def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
+ return SimpleNamespace(
+ title="Entry lookup",
+ description=None,
+ workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
+ proxy_location=None,
+ webhook_callback_url=None,
+ totp_verification_url=None,
+ totp_identifier=None,
+ persist_browser_session=False,
+ pin_saved_session_ip=False,
+ browser_profile_id=None,
+ browser_profile_key=None,
+ model=None,
+ max_screenshot_scrolls=None,
+ extra_http_headers=None,
+ cdp_connect_headers=None,
+ run_with=None,
+ ai_fallback=None,
+ cache_key=None,
+ adaptive_caching=None,
+ enable_self_healing=None,
+ code_version=None,
+ run_sequentially=False,
+ sequential_key=None,
+ )
- assert result is not None
- assert result.unreferenced_parameter_keys == ("business_name",)
+ persisted: dict[str, object] = {}
- @pytest.mark.parametrize(
- ("code", "expected_unreferenced"),
- [
- (
- "values = [business_name for business_name in ['fixed']]\n"
- "await page.locator('#company').fill(values[0])",
- ["business_name"],
- ),
- (
- "async def fill_company():\n"
- " await page.locator('#company').fill(str(business_name))\n"
- "await fill_company()",
- [],
- ),
- ],
- )
- def test_definition_preflight_honors_python_scopes(self, code: str, expected_unreferenced: list[str]) -> None:
- ctx = _definition_contract_ctx()
- indented_code = textwrap.indent(code, " ")
- scoped_yaml = _yaml(
- "title: Submit reusable service request\n"
- "workflow_definition:\n"
- " parameters:\n"
- " - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: submit_service_request\n"
- " parameter_keys: [business_name]\n"
- " code: |\n"
- f"{indented_code}\n"
- )
+ async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
+ return fake_workflow(blocks=[])
- result = workflow_update_module._definition_plane_preflight_reject(ctx, scoped_yaml)
+ async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return fake_workflow(blocks=[SimpleNamespace(label="extract_entry_output")])
- if expected_unreferenced:
- assert result is not None
- assert list(result.unreferenced_parameter_keys) == expected_unreferenced
- else:
- assert result is None
+ async def fake_update_workflow_definition(**kwargs: object) -> None:
+ persisted.update(kwargs)
- def test_definition_preflight_counts_allowed_non_code_parameter_consumption(self) -> None:
- ctx = _definition_contract_ctx()
- mixed_yaml = _yaml(
- """
- title: Submit reusable service request
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- blocks:
- - block_type: navigation
- label: open_customer
- url: "https://example.com/customer/{{ business_name }}"
- - block_type: code
- label: submit_service_request
- parameter_keys: [contact_email]
- code: |
- await page.locator("#email").fill(contact_email)
- await page.locator("#submit").click()
- """
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
+ monkeypatch.setattr(
+ workflow_update_module.app.WORKFLOW_SERVICE,
+ "update_workflow_definition",
+ fake_update_workflow_definition,
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "resolve_copilot_created_by_stamp",
+ AsyncMock(return_value="copilot"),
)
- assert workflow_update_module._metadata_contract_run_preflight_reject(ctx, mixed_yaml, []) is None
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
- @pytest.mark.parametrize(
- ("template", "is_consumed"),
- [
- ('{{ "business_name" }}', False),
- ("{% if business_name %}/customer{% endif %}", True),
- ],
- )
- def test_definition_preflight_uses_jinja_semantics(self, template: str, is_consumed: bool) -> None:
- ctx = _definition_contract_ctx()
- templated_yaml = _yaml(
- f"""
- title: Submit reusable service request
- workflow_definition:
- parameters:
- - {{parameter_type: workflow, key: business_name, workflow_parameter_type: string}}
- blocks:
- - block_type: navigation
- label: open_customer
- url: {json.dumps(template)}
- """
- )
-
- result = workflow_update_module._definition_plane_preflight_reject(ctx, templated_yaml)
-
- assert (result is None) is is_consumed
+ assert result["ok"] is True
+ assert persisted["workflow_id"] == "w"
+ definition = persisted["workflow_definition"]
+ assert definition.blocks == [SimpleNamespace(label="extract_entry_output")]
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ block = _single_code_block(parsed)
+ assert json.loads(block["extraction_schema"])["properties"]["output"]["properties"]["record_id"] == {}
@pytest.mark.asyncio
- async def test_definition_preflight_defers_to_repairable_syntax_error(self) -> None:
- ctx = _definition_contract_ctx()
- malformed_yaml = _yaml(
+ async def test_run_path_persists_blocks_for_api_readback_when_metadata_missing(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ workflow_yaml = _yaml(
"""
- title: Submit reusable service request
+ title: Public path validation
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
blocks:
- block_type: code
- label: submit_service_request
- parameter_keys: [business_name]
+ label: validate_public_path
code: |
- await page.locator("#company").fill(str(business_name)
+ return {"public_form_exists": False}
"""
)
- result = await _update_workflow({"workflow_yaml": malformed_yaml}, ctx)
+ def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
+ return SimpleNamespace(
+ title="Public path validation",
+ description=None,
+ workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
+ proxy_location=None,
+ webhook_callback_url=None,
+ totp_verification_url=None,
+ totp_identifier=None,
+ persist_browser_session=False,
+ pin_saved_session_ip=False,
+ browser_profile_id=None,
+ browser_profile_key=None,
+ model=None,
+ max_screenshot_scrolls=None,
+ extra_http_headers=None,
+ cdp_connect_headers=None,
+ run_with=None,
+ ai_fallback=None,
+ cache_key=None,
+ adaptive_caching=None,
+ enable_self_healing=None,
+ code_version=None,
+ run_sequentially=False,
+ sequential_key=None,
+ )
- assert result["ok"] is False
- assert "not valid Python" in result["error"]
- assert result.get("data", {}).get("reason_code") != "definition_contract_unsatisfied"
+ persisted: dict[str, object] = {}
- def test_definition_preflight_is_code_first_only(self) -> None:
- ctx = _definition_contract_ctx()
- ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
- standard_yaml = _yaml(
- """
- title: Submit reusable service request
- workflow_definition:
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
- blocks:
- - block_type: task
- label: submit_service_request
- url: "{{business_name}}/{{contact_email}}/{{service_address}}/{{desired_start_date}}"
- """
- )
+ async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
+ return fake_workflow(blocks=[])
- assert (
- workflow_update_module._metadata_contract_run_preflight_reject(
- ctx,
- standard_yaml,
- [],
- )
- is None
- )
+ async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return fake_workflow(blocks=[SimpleNamespace(label="validate_public_path")])
- @pytest.mark.asyncio
- async def test_code_safety_seam_reject_carries_progress_surface_kind(self) -> None:
- ctx = _code_only_ctx()
- result = await _update_workflow({"workflow_yaml": _REQUESTS_IMPORT_CODE_YAML}, ctx)
- assert result["ok"] is False
- assert result["data"]["surface_kind"] == "code_repair_progress"
- assert result["data"]["progress_text"]
- # The substantive copy is unchanged; the progress text is a separate carrier.
- assert result["user_facing_summary"] == (
- "I need to adjust the workflow's code so it can run safely before testing."
- )
- assert result["data"]["progress_text"] != result["user_facing_summary"]
+ async def fake_update_workflow_definition(**kwargs: object) -> None:
+ persisted.update(kwargs)
- @pytest.mark.asyncio
- async def test_generated_code_preflight_string_reject_is_not_authoritative(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
+ monkeypatch.setattr(
+ workflow_update_module.app.WORKFLOW_SERVICE,
+ "update_workflow_definition",
+ fake_update_workflow_definition,
+ )
monkeypatch.setattr(
workflow_update_module,
- "_code_block_safety_errors",
- lambda workflow_yaml, prior_yaml: [
- "Code block `search_registry` failed the generated-code preflight check: "
- "AUTHOR_PAGE_REQUEST: page.request is not allowed."
- ],
+ "resolve_copilot_created_by_stamp",
+ AsyncMock(return_value="copilot"),
)
- ctx = _code_only_ctx()
- result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "code_safety_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is False
- assert ctx.latest_recorded_build_test_outcome.structural_key is None
- def test_code_safety_payload_keeps_typed_security_error_authoritative(self) -> None:
- payload = _code_safety_reject_payload(
- [
- CodeBlockSecurityError(
- "Code block `search_registry` failed the Copilot code security check: page.request is not allowed.",
- block_label="search_registry",
- reason_code="AUTHOR_PAGE_REQUEST",
- surface="page.request",
- )
- ]
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
)
- assert payload == {
- "code_safety_errors": [
- {
- "block_label": "search_registry",
- "reason_code": "AUTHOR_PAGE_REQUEST",
- "surface": "page.request",
- }
- ]
- }
+ assert result["ok"] is True
+ assert persisted["workflow_id"] == "w"
+ assert persisted["workflow_definition"].blocks == [SimpleNamespace(label="validate_public_path")]
- def test_code_safety_payload_rejects_string_only_preflight_authority(self) -> None:
- payload = _code_safety_reject_payload(
- [
- "Code block `search_registry` failed the generated-code preflight check: "
- "AUTHOR_PAGE_REQUEST: page.request is not allowed."
- ]
+ def test_output_contract_signature_uses_stable_scope_and_required_path_identity(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "contract-scope-a"
+ first = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ required_paths={"output.locations[].address", "output.statuses"},
+ )
+ second = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ required_paths={"output.statuses", "output.locations[].address"},
+ )
+ other_ctx = _code_only_ctx()
+ other_ctx.turn_id = "contract-scope-b"
+ other_scope = workflow_update_module._output_contract_signature(
+ ctx=other_ctx,
+ required_paths={"output.statuses", "output.locations[].address"},
)
- assert payload is None
+ assert first == second
+ assert other_scope != first
+ assert (
+ workflow_update_module._output_contract_pin_key(
+ ctx,
+ "title: First\nworkflow_definition:\n blocks: []\n",
+ {"output.locations[].address", "output.statuses"},
+ )
+ == first
+ )
- @pytest.mark.asyncio
- async def test_authoritative_recorded_outcome_rejects_unchanged_authored_candidate(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_output_contract_owner_pin_is_scoped(self) -> None:
ctx = _code_only_ctx()
- ctx.workflow_yaml = _code_yaml('await page.goto("https://example.com/old")')
- signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:typed-outcome",
- authored_structure_signature=signature,
+ workflow_yaml = _yaml(
+ """
+ title: Shared output shape
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: first_goal_output
+ code: |
+ return {"output": {"status": "one"}}
+ - block_type: code
+ label: second_goal_output
+ code: |
+ return {"output": {"status": "two"}}
+ """
)
+ required_paths = {"output.status"}
- result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+ ctx.turn_id = "first-build-goal"
+ workflow_update_module._pin_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ required_paths,
+ "first_goal_output",
+ )
+ ctx.turn_id = "second-build-goal"
+ workflow_update_module._pin_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ required_paths,
+ "second_goal_output",
+ )
- assert result["ok"] is False
- assert "left the frontier the last recorded test outcome named unchanged" in result["error"]
- assert ctx.workflow_yaml == _code_yaml('await page.goto("https://example.com/old")')
- assert ctx.has_staged_proposal is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "unchanged_after_recorded_outcome"
- assert ctx.latest_recorded_build_test_outcome.structural_key is not None
+ assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
+ "second_goal_output"
+ )
+ ctx.turn_id = "first-build-goal"
+ assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
+ "first_goal_output"
+ )
@pytest.mark.asyncio
- async def test_repeated_unchanged_authored_candidate_keeps_same_reject_key_and_backstop(
+ async def test_output_contract_first_contact_imposes_keyed_return_envelope(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:typed-outcome",
- authored_structure_signature=signature,
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ _typed_completion_criterion(
+ id="requested_flags",
+ output_path="output.flags",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ record_id = "ABC123"
+ flags = ["enabled"]
+ """
)
- first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
- first_key = ctx.latest_recorded_build_test_outcome.structural_key
- first_count = ctx.code_authoring_guardrail_reject_count
- second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- assert first["ok"] is False
- assert second["ok"] is False
- assert first_key is not None
- assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
- assert first_count == 1
- assert ctx.code_authoring_guardrail_reject_count == 2
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert 'return {"output": {"flags": flags, "record_id": record_id}}' in code
+ metadata = ctx.code_artifact_metadata["extract_entry_output"]
+ assert metadata["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
+ assert (
+ json.loads(metadata["claimed_outcomes"][0]["extraction_schema"])["properties"]["output"]["properties"][
+ "record_id"
+ ]
+ == {}
+ )
- @pytest.mark.asyncio
- async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def _run_preflight_metadata_reject_ctx(self) -> CopilotContext:
ctx = _code_only_ctx()
- signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
- changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:typed-outcome",
- authored_structure_signature=signature,
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="validate_start",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
)
+ return ctx
- result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == changed_yaml
- assert ctx.has_staged_proposal is True
+ _RUN_PREFLIGHT_CLICK_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: validate_start
+ code: |
+ await page.locator("#start").click()
+ """
+ )
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_metadata_contract_scaffold_uses_recorded_paths_before_request_policy(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi", "locations"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
- ],
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.requested_value",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
)
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
+ structural_failure_identity="completion:typed-output",
+ missing_requested_output_facts=[
+ {"output_path": "output.recorded_value", "output_root": "output", "value_status": "no_typed_value"},
+ ],
)
- partial_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: address_city_county_or_zip_code
blocks:
- block_type: code
- label: {label}
- parameter_keys:
- - address_city_county_or_zip_code
+ label: extract_entry_output
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- await page.locator("#locInput").fill(str(address_city_county_or_zip_code))
- return {{}}
+ return {"output": {"recorded_value": "ABC123"}}
"""
)
+ required_paths, source, reason_code = workflow_update_module._required_child_output_paths_for_authoring(ctx)
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ scaffolded = workflow_update_module._apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ [],
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ )
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
- assert ctx.has_staged_proposal is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert scaffolded[0]["artifact_id"] == "code_artifact:extract_entry_output"
+ assert scaffolded[0]["block_label"] == "extract_entry_output"
+ assert scaffolded[0]["claimed_outcomes"][0]["goal_value_paths"] == ["output.recorded_value"]
+ schema = json.loads(scaffolded[0]["claimed_outcomes"][0]["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["recorded_value"] == {}
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_explicit_empty_return_despite_assigned_goal_root(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- ],
- )
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- npi = await page.locator("#npi").inner_text(timeout=5000)
- return {{}}
- """
+ def test_independent_judgment_output_is_not_required_as_code_return_path(self) -> None:
+ judgment_criterion = CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ judgment_truth_condition=JudgmentTruthCondition(
+ predicate="login_gate_blocks_target",
+ polarity_when_holds=True,
+ ),
)
+ judgment_only_ctx = _code_only_ctx()
+ judgment_only_ctx.request_policy = RequestPolicy(completion_criteria=[judgment_criterion])
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ judgment_only_paths = workflow_update_module._output_contract_required_paths_source(judgment_only_ctx).union
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
- assert ctx.has_staged_proposal is False
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.is_authoritative is True
- assert outcome.block_labels == [label]
- assert outcome.structural_failure_identity.startswith("author_time:")
+ assert judgment_only_paths == set()
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_known_empty_helper_without_missing_facts(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- ],
- )
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- def build():
- return {{}}
-
- return build()
- """
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ judgment_criterion,
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ ),
+ ]
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.block_labels == [label]
- assert outcome.missing_requested_output_facts == []
- assert outcome.structural_failure_identity.startswith("author_time:")
+ assert required_paths == {"output.record_id"}
+ assert source == "requested_output_contract"
+ assert reason_code == "requested_output_contract_missing_output_coverage"
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_independent_judgment_shape_output_is_not_required_as_code_return_path(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi", "locations"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
- ],
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
)
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
+
+ required_paths = workflow_update_module._output_contract_required_paths_source(ctx).union
+
+ assert required_paths == set()
+
+ def test_mixed_repair_context_keeps_non_judgment_code_return_path(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ judgment_truth_condition=JudgmentTruthCondition(
+ predicate="login_gate_blocks_target",
+ polarity_when_holds=True,
+ ),
+ )
+ ]
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- npi = "1234567890"
- locations = [{{"address": "123 Main St"}}]
- """
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.login_gate_blocks_target", "output.record_id"],
+ required_extraction_schema_paths=["output.login_gate_blocks_target", "output.record_id"],
+ required_code_return_paths=["output.login_gate_blocks_target", "output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
- assert result["ok"] is True
- labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
- assert labels == [label]
+ assert required_paths == {"output.record_id"}
+ assert source == "requested_output_contract"
+ assert reason_code == "requested_output_contract_missing_output_coverage"
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_judgment_is_excluded_from_scalar_bind_set(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
- ],
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ ),
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ ),
+ ]
)
- partial_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- address = "North Carolina, USA"
- """
+
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ assert contract.observation_paths == {"output.record_id"}
+ assert contract.union == {"output.record_id"}
+ assert "output.login_gate_blocks_target" not in contract.declaration_paths
+
+ def test_runtime_output_judgment_alone_yields_empty_bind_set(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
)
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ assert workflow_update_module._output_contract_required_paths_source(ctx).union == set()
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- assert ctx.has_staged_proposal is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ def test_non_judgment_scalar_bind_set_is_unchanged(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ )
+ ]
+ )
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_accepts_canonical_output_path_candidate_with_sibling(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ assert workflow_update_module._output_contract_required_paths_source(ctx).observation_paths == {
+ "output.record_id"
+ }
+
+ def test_judgment_output_paths_widens_independent_predicate(self) -> None:
ctx = _code_only_ctx()
- label = "submit_form_and_extract_confirmation"
- schema = workflow_update_module._schema_template_text_for_required_paths({"output.confirmation_number"})
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": schema}
- ],
- }
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="independent_gate",
+ outcome="the independent path is blocked by a login gate",
+ output_path="output.independent_gate_blocks_target",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ CompletionCriterion(
+ id="runtime_gate",
+ outcome="the runtime path is blocked by a login gate",
+ output_path="output.runtime_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ ),
+ ]
+ )
+
+ all_judgment = workflow_update_module._judgment_output_paths(ctx)
+
+ assert all_judgment == {
+ "output.independent_gate_blocks_target",
+ "output.runtime_gate_blocks_target",
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+
+ def test_independent_judgment_runtime_repair_fact_is_not_rehydrated_as_code_return_path(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+ )
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
{
- "output_path": "output.confirmation_number",
+ "workflow_run_id": "wr_current",
+ "block_label": "judge_login_gate_blocks_target",
+ "output_path": "output.login_gate_blocks_target",
"output_root": "output",
- "value_status": "no_typed_value",
- },
+ "criterion_id": "__copilot_requested_output__output_login_gate_blocks_target",
+ "reason_code": "structurally_abstained",
+ "grounding_mode": "judgment_boolean",
+ "value_status": "structural_abstained",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
- title: Form submission
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"output": {{"confirmation_number": "ABC-123", "account_number": "100245"}}}}
- """
- )
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ required_paths = workflow_update_module._output_contract_required_paths_source(ctx).union
- assert result["ok"] is True
- assert ctx.workflow_yaml is not None
- assert "confirmation_number" in ctx.workflow_yaml
- assert ctx.code_artifact_metadata[label]["claimed_outcomes"][0]["extraction_schema"] == schema
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "outcome_not_demonstrated"
+ assert required_paths == set()
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_flat_candidate_for_canonical_output_path(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_mixed_runtime_repair_facts_keep_non_judgment_code_return_path(self) -> None:
ctx = _code_only_ctx()
- label = "submit_form_and_extract_confirmation"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["confirmation_number"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["confirmation_number"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ ),
+ ]
+ )
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
{
- "output_path": "output.confirmation_number",
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.login_gate_blocks_target",
"output_root": "output",
- "value_status": "no_typed_value",
+ "criterion_id": "__copilot_requested_output__output_login_gate_blocks_target",
+ "reason_code": "structurally_abstained",
+ "grounding_mode": "judgment_boolean",
+ "value_status": "structural_abstained",
+ },
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.record_id",
+ "output_root": "output",
+ "criterion_id": "record_id",
+ "reason_code": "structurally_abstained",
+ "grounding_mode": "missing",
+ "value_status": "structural_abstained",
},
],
)
- candidate_yaml = _yaml(
- f"""
- title: Form submission
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"confirmation_number": "ABC-123"}}
- """
+
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+
+ assert required_paths == {"output.record_id"}
+ assert source == "runtime_output_repair"
+ assert reason_code == "runtime_output_repair_required"
+
+ def test_runtime_output_facts_record_same_run_null_without_evidence_text_backfill(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "extract_entry_output",
+ "status": "completed",
+ "extracted_data": {"output": {"npi": None}, "evidence_text": "Value 1234567890"},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ output_path="output.npi",
+ grounding_mode="exact_value",
+ expected_output_shape="string",
+ has_exact_value=False,
+ )
+ ],
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ )
- assert result["ok"] is False
- assert "output.confirmation_number" in result["error"]
- assert "satisfies the requested output contract" in result["error"]
- assert "canonical required child paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.runtime_output_repair_facts == [
{
- "output_path": "output.confirmation_number",
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
"output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "grounding_mode": "exact_value",
+ "expected_output_shape": "string",
+ "value_status": "null",
+ "evidence_refs": ["registered_output:extract_entry_output:npi", "output:extract_entry_output"],
}
]
- rendered = agent_module._recorded_build_test_outcome_prompt(ctx)
- assert "output_path=output.confirmation_number" in rendered
+ assert outcome.is_authoritative is True
- def test_required_output_roots_ignore_nested_debug_output_root_without_declared_root(self) -> None:
- label = "extract_provider_profile"
- metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["debug_output"]}]}}
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"debug_output": {{"npi": "1234567890"}}}}
- """
+ def test_runtime_output_facts_record_empty_typed_output_paths_for_next_contract(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "summarize_access_output",
+ "status": "completed",
+ "extracted_data": {
+ "extracted_information": [],
+ "summarize_access_output": {},
+ "evidence_text": "diagnostic page text",
+ },
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[
+ "requested_form_exists",
+ "requested_path_label",
+ "requested_next_action",
+ ],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_form_exists",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.public_form_exists",
+ grounding_mode="missing",
+ expected_output_shape="boolean",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_path_label",
+ state="unsatisfied",
+ reason_code="missing_exact_field",
+ output_path="output.visible_page_path_label",
+ grounding_mode="missing",
+ expected_output_shape="string",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_next_action",
+ state="unsatisfied",
+ reason_code="missing_exact_field",
+ output_path="output.recommended_next_action",
+ grounding_mode="missing",
+ expected_output_shape="string",
+ ),
+ ],
)
- assert workflow_update_module._candidate_missing_required_output_paths(
- candidate_yaml,
- metadata,
- required_paths={"npi"},
- ) == ["npi"]
-
- def test_required_output_roots_ignore_same_label_undeclared_static_root(self) -> None:
- label = "extract_provider_profile"
- metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["address"]}]}}
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"npi": "1234567890"}}
- """
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+ assert outcome is not None
+ assert {fact["output_path"]: fact["value_status"] for fact in outcome.runtime_output_repair_facts} == {
+ "output.public_form_exists": "structural_abstained",
+ "output.recommended_next_action": "no_typed_value",
+ "output.visible_page_path_label": "no_typed_value",
+ }
+ assert all(
+ "evidence_text" not in ref
+ for fact in outcome.runtime_output_repair_facts
+ for ref in fact.get("evidence_refs", [])
)
- assert workflow_update_module._candidate_missing_required_output_paths(
- candidate_yaml,
- metadata,
- required_paths={"npi"},
- ) == ["npi"]
-
- def test_required_output_roots_ignore_static_root_without_label_metadata(self) -> None:
- label = "extract_provider_profile"
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"npi": "1234567890"}}
- """
- )
-
- assert workflow_update_module._candidate_missing_required_output_paths(
- candidate_yaml,
- {},
- required_paths={"npi"},
- ) == ["npi"]
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+ assert required_paths == {
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ }
+ assert source == "runtime_output_repair"
+ assert reason_code == "runtime_output_repair_required"
- def test_required_output_roots_accept_same_label_declared_static_root(self) -> None:
- label = "extract_provider_profile"
- metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["npi"]}]}}
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"npi": "1234567890"}}
- """
+ def test_runtime_output_facts_preserve_satisfied_output_owner_label(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "download_statement",
+ "status": "completed",
+ "extracted_data": {"output": {"statement_pdf": "statement.pdf"}},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_fallback_floor__run", "requested_statement_pdf"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_fallback_floor__run",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_statement_pdf",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ output_path="output.statement_pdf",
+ grounding_mode="exact_value",
+ expected_output_shape="string",
+ has_exact_value=True,
+ evidence_ref="block_outputs:download_statement.output.statement_pdf",
+ ),
+ ],
)
- assert (
- workflow_update_module._candidate_missing_required_output_paths(
- candidate_yaml,
- metadata,
- required_paths={"npi"},
- )
- == []
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
)
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_unprovable_dynamic_output_shape(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "download_statement",
+ "owner_labels": ["download_statement"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "grounding_mode": "exact_value",
+ "expected_output_shape": "string",
+ "value_status": "satisfied",
+ "evidence_refs": ["output:download_statement"],
}
+ ]
+
+ def test_runtime_output_facts_preserve_flat_registered_output_owner_label(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {"label": "apex_portal_login", "status": "completed", "extracted_data": {}},
+ {"label": "apex_open_monthly_statement", "status": "completed", "extracted_data": {}},
+ {"label": "apex_download_invoice_pdf", "status": "completed", "extracted_data": {}},
+ ],
+ },
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_authored_output__output_file_name"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_file_name",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.file_name",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ )
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- result = {{}}
- key = "npi"
- value = await page.locator("#npi").inner_text(timeout=5000)
- result[key] = value
- return result
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
- labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
- assert labels == [label]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "apex_download_invoice_pdf",
+ "output_parameter_key": "apex_download_invoice_pdf_output",
+ "value": {
+ "file_name": "statement.pdf",
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ },
+ }
+ ],
)
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_literal_list_of_output_dicts(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "apex_download_invoice_pdf",
+ "output_path": "output.file_name",
+ "output_root": "output",
+ "criterion_id": "__copilot_authored_output__output_file_name",
+ "reason_code": "no_evidence",
+ "grounding_mode": "missing",
+ "value_status": "no_typed_value",
+ "evidence_refs": ["registered_output:apex_download_invoice_pdf:apex_download_invoice_pdf_output"],
}
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ ]
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: apex_portal_login
code: |
- return [{{"npi": "123"}}]
+ return {"logged_in": True}
+ - block_type: code
+ label: apex_open_monthly_statement
+ code: |
+ return {"matched": True}
+ - block_type: code
+ label: apex_download_invoice_pdf
+ code: |
+ return {"file_name": "statement.pdf"}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
- )
+ assert evaluation is not None
+ assert evaluation.block_label == "apex_download_invoice_pdf"
+ assert evaluation.payload["output_owner_labels"] == ["apex_download_invoice_pdf"]
+ assert "missing_output_owner" not in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_dynamic_list_output_to_abstain(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
+ def test_runtime_output_facts_do_not_infer_registered_owner_from_parameter_key(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {"label": "download_invoice", "status": "completed", "extracted_data": {}},
+ ],
+ },
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_authored_output__output_file_name"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_file_name",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.file_name",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ )
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "output_parameter_key": "download_invoice_output",
+ "value": {"file_name": "statement.pdf"},
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "output_path": "output.file_name",
+ "output_root": "output",
+ "criterion_id": "__copilot_authored_output__output_file_name",
+ "reason_code": "no_evidence",
+ "grounding_mode": "missing",
+ "value_status": "no_typed_value",
+ "evidence_refs": ["registered_output:unknown:download_invoice_output"],
+ }
+ ]
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: download_invoice
code: |
- record = {{"npi": "123"}}
- return [record]
+ return {"file_name": "statement.pdf"}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
- )
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_list_dict_to_abstain(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
+ def test_runtime_output_owner_ignores_unsatisfied_independent_self_emitted_fields(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "login_to_service",
+ "status": "completed",
+ "extracted_data": {"output": {"logged_in": True}},
+ },
+ {
+ "label": "open_statement",
+ "status": "completed",
+ "extracted_data": {
+ "output": {
+ "matched": True,
+ "statement_date": "2026-05",
+ "visible_page_label": "Statement details",
+ }
+ },
+ },
+ ],
+ },
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[
+ "__copilot_authored_output__output_logged_in",
+ "__copilot_authored_output__output_matched",
+ "__copilot_authored_output__output_statement_date",
+ "__copilot_authored_output__output_visible_page_label",
+ "__copilot_authored_output__output_downloaded",
+ ],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_logged_in",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.logged_in",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_ref="block_outputs:login_to_service.output.logged_in",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_matched",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.matched",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_ref="block_outputs:open_statement.output.matched",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_statement_date",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.statement_date",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ evidence_ref="block_outputs:open_statement.output.statement_date",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_visible_page_label",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.visible_page_label",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ evidence_ref="block_outputs:open_statement.output.visible_page_label",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_downloaded",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.downloaded",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement workflow
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: login_to_service
code: |
- key = "npi"
- return [{{key: "123"}}]
+ return {"output": {"logged_in": True}}
+ - block_type: code
+ label: open_statement
+ code: |
+ return {"output": {"statement_date": "2026-05"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
- outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
- )
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_local_dict_to_abstain(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
+ def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- key = "npi"
- result = {{key: "123"}}
- return result
- """
- )
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_previous",
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ )
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
- outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
)
- def test_local_dict_dynamic_key_preserves_literal_roots_while_abstaining(self) -> None:
- produced = workflow_update_module._code_block_produced_output_roots(
- """
- key = "npi"
- result = {"address": "North Carolina", key: "123"}
- return result
- """
+ def test_runtime_output_facts_ignore_unscoped_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
)
- assert produced.roots == {"address"}
- assert produced.abstained is True
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "block_label" not in outcome.runtime_output_repair_facts[0]
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
+ )
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_helper_dynamic_key_local_dict_to_abstain(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
+ def test_runtime_output_facts_ignore_unscoped_fallback_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "registered_output_parameter_values": [
+ {
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
}
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- def build():
- key = "npi"
- result = {{key: "123"}}
- return result
- return build()
- """
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
- outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert all(
- fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
- for fact in outcome.missing_requested_output_facts
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "block_label" not in outcome.runtime_output_repair_facts[0]
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
)
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_unrelated_dynamic_output_shape(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_facts_override_request_policy_contract_paths(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.requested_value",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
+ "output_root": "output",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "value_status": "null",
+ },
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "criterion_id": "requested_location",
+ "reason_code": "structurally_abstained",
+ "grounding_mode": "missing",
+ "value_status": "structural_abstained",
+ },
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: extract_entry_output
code: |
- result = {{}}
- key = "address"
- result[key] = await page.locator("#address").inner_text(timeout=5000)
- return result
+ return {"output": {"requested_value": "wrong"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "npi",
- "output_root": "npi",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
+ assert evaluation is not None
+ assert evaluation.source == "runtime_output_repair"
+ assert evaluation.reason_code == "runtime_output_repair_required"
+ assert evaluation.required_paths == {"output.locations[].address", "output.npi"}
+ assert sorted(
+ evaluation.payload["runtime_output_repair_facts"],
+ key=lambda item: str(item.get("output_path") or ""),
+ ) == sorted(
+ ctx.latest_recorded_build_test_outcome.runtime_output_repair_facts,
+ key=lambda item: str(item.get("output_path") or ""),
+ )
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_when_unrelated_block_abstains(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_owner_selects_current_block_without_metadata(self) -> None:
ctx = _code_only_ctx()
- declared_label = "extract_provider_profile"
- unrelated_label = "extract_unrelated_status"
- ctx.code_artifact_metadata = {
- declared_label: {
- "block_label": declared_label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_statement"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {declared_label}
+ label: login
code: |
- return {{"address": "North Carolina"}}
+ return {"logged_in": True}
- block_type: code
- label: {unrelated_label}
+ label: download_statement
code: |
- result = {{}}
- key = "status"
- result[key] = await page.locator("#status").inner_text(timeout=5000)
- return result
+ return {"output": {"statement_pdf": "statement.pdf"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "npi",
- "output_root": "npi",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
+ assert evaluation is not None
+ assert evaluation.block_label == "download_statement"
+ assert evaluation.payload["output_owner_labels"] == ["download_statement"]
+ assert "missing_output_owner" not in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_keeps_undeclared_missing_roots_with_dynamic_shape(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_zero_owner_does_not_fall_through_to_single_block_default(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
- {"output_path": "status", "output_root": "status", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: only_block
code: |
- result = {{}}
- key = "npi"
- result[key] = await page.locator("#npi").inner_text(timeout=5000)
- return result
+ return {"output": {"statement_pdf": "statement.pdf"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "status",
- "output_root": "status",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_literal_output_missing_required_root(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_multi_owner_rejects_without_picking_metadata_owner(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_a", "download_b"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: download_a
+ code: |
+ return {"output": {"statement_pdf": "a.pdf"}}
+ - block_type: code
+ label: download_b
code: |
- return {{"address": "North Carolina"}}
+ return {"output": {"statement_pdf": "b.pdf"}}
"""
)
+ metadata = [{"block_label": "download_a", "claimed_outcomes": [{"goal_value_paths": ["output.statement_pdf"]}]}]
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata)
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.block_labels == [label]
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "npi",
- "output_root": "npi",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["download_a", "download_b"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_empty_return_despite_assigned_root(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_stale_multi_owner_rejects_when_one_owner_current(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["current_download", "stale_download"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: current_download
code: |
- npi = await page.locator("#npi").inner_text(timeout=5000)
- return {{}}
+ return {"output": {"statement_pdf": "a.pdf"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.block_labels == [label]
- assert outcome.missing_requested_output_facts == []
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["current_download"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
- @pytest.mark.parametrize("return_expression", ["None", "[]", '["not keyed"]'])
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_static_non_keyed_return_with_unrelated_output(
- self, monkeypatch: pytest.MonkeyPatch, return_expression: str
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_multi_owner_all_stale_rejects_as_missing_owner(self) -> None:
ctx = _code_only_ctx()
- declared_label = "extract_provider_profile"
- unrelated_label = "extract_unrelated_status"
- ctx.code_artifact_metadata = {
- declared_label: {
- "block_label": declared_label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {declared_label}
- code: |
- return {return_expression}
- - block_type: code
- label: {unrelated_label}
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["stale_one", "stale_two"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: current_download
code: |
- return {{"status": "active"}}
+ return {"output": {"statement_pdf": "a.pdf"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "npi",
- "output_root": "npi",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_helper_known_empty_output(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_paths_with_disagreeing_single_owners_reject(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_pdf"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ },
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["extract_total"],
+ "output_path": "output.statement_total",
+ "output_root": "output",
+ "criterion_id": "requested_statement_total",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ },
],
)
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: download_pdf
code: |
- def build():
- return {{}}
-
- return build()
+ return {"output": {"statement_pdf": "a.pdf"}}
+ - block_type: code
+ label: extract_total
+ code: |
+ return {"output": {"statement_total": "12.00"}}
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == []
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["download_pdf", "extract_total"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
- @pytest.mark.asyncio
- async def test_metadata_reject_persists_missing_roots_for_next_author_context(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
+ def test_runtime_output_owner_overrides_stale_pin(self) -> None:
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
+ ctx.turn_id = "runtime-owner-pin"
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
{
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["new_owner"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
],
)
- partial_yaml = _yaml(
- f"""
- title: Provider lookup
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: stale_owner
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- return {{"search_options": []}}
+ return {"output": {"statement_pdf": "old.pdf"}}
+ - block_type: code
+ label: new_owner
+ code: |
+ return {"output": {"statement_pdf": "new.pdf"}}
"""
)
+ pin_key = workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, {"output.statement_pdf"})
+ ctx.output_contract_pinned_block_label_by_signature = {pin_key: "stale_owner"}
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "address",
- "output_root": "address",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- ]
- rendered = agent_module._recorded_build_test_outcome_prompt(ctx)
- assert "missing_requested_output_facts:" in rendered
- assert "output_root=address" in rendered
- assert "output_root=credentialing_status" in rendered
- assert "value_status=no_typed_value" in rendered
+ assert evaluation is not None
+ assert evaluation.block_label == "new_owner"
+ assert evaluation.payload["output_owner_labels"] == ["new_owner"]
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_broad_output_for_missing_child_paths(
+ async def test_runtime_output_repair_facts_trigger_first_contact_envelope(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_entry"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["output"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.verified_block_outputs[label] = {
- "output": {"npi": "1234567890", "address": "Example location", "statuses": ["active"]}
- }
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
- {"output_path": "output.address", "output_root": "output", "value_status": "no_typed_value"},
- {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
+ "output_root": "output",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "value_status": "null",
+ }
],
)
- candidate_yaml = _yaml(
- f"""
+ workflow_yaml = _yaml(
+ """
title: Entry lookup
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: extract_entry_output
code: |
- return {{"output": {{"summary": "found"}}}}
+ npi = "1234567890"
+ return npi
"""
)
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is False
- assert "requested output contract" in result["error"]
- assert result["data"]["canonical_required_child_paths"] == [
- "output.address",
- "output.npi",
- "output.statuses",
- ]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.address",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.statuses",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ code = _single_code_block(parsed)["code"]
+ assert 'return {"output": {"npi": npi}}' in code
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
+ "output.npi"
]
+ assert ctx.runtime_output_repair_attempt_by_signature
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_exact_missing_array_child_paths(
+ async def test_update_workflow_applies_metadata_contract_scaffold_for_unambiguous_owner(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_entry"
- child_paths = ["output.npi", "output.locations[].address", "output.locations[].status", "output.statuses"]
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{'
- '"npi":{"type":"string"},'
- '"locations":{"type":"array","items":{"type":"object","properties":{'
- '"address":{"type":"string"},"status":{"type":"string"}}}},'
- '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ _typed_completion_criterion(
+ id="requested_flags",
+ output_path="output.flags",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
)
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
- "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
- {
- "output_path": "output.locations[].address",
- "output_root": "output",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.locations[].status",
- "output_root": "output",
- "value_status": "no_typed_value",
- },
- {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
- ],
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
+ """
)
- candidate_yaml = _yaml(
- f"""
+
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ stored = ctx.code_artifact_metadata["extract_entry_output"]
+ assert stored["artifact_id"] == "code_artifact:extract_entry_output"
+ assert stored["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
+ assert stored["terminal_verifier_expectations"][0]["goal_value_paths"] == [
+ "output.flags",
+ "output.record_id",
+ ]
+ schema = json.loads(stored["claimed_outcomes"][0]["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["record_id"] == {}
+ assert schema["properties"]["output"]["properties"]["flags"] == {}
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_allows_neutral_multi_block_output_owner(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
title: Entry lookup
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: enter_filters
code: |
- return {{
- "output": {{
- "npi": "1234567890",
- "locations": [{{"address": "Example location", "status": "active"}}],
- "statuses": ["active"],
- }}
- }}
+ await page.locator("#filter").fill("ABC123")
+ - block_type: code
+ label: choose_record
+ code: |
+ await page.locator("#choose").click()
+ - block_type: code
+ label: extract_record
+ code: |
+ return {"output": {"record_id": "ABC123"}}
"""
)
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
assert result["ok"] is True
labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
- assert labels == [label]
+ assert labels == ["enter_filters", "choose_record", "extract_record"]
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_ignores_one_block_without_multi_stage_spine(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_record
+ code: |
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is True
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_extraction_schema_wrapper_mismatch(
+ async def test_requested_output_contract_allows_aligned_initial_candidate(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
label = "lookup_entry"
child_paths = ["output.npi", "output.locations[].address", "output.statuses"]
- claimed_schema = (
- '{"type":"object","properties":{'
- '"npi":{"type":"string"},'
- '"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
- '"statuses":{"type":"array","items":{"type":"string"}}}}'
- )
- terminal_schema = (
+ schema = (
'{"type":"object","properties":{"output":{"type":"object","properties":{'
'"npi":{"type":"string"},'
'"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
'"statuses":{"type":"array","items":{"type":"string"}}}}}}'
)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id=f"requested_{index}",
+ output_path=path,
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ for index, path in enumerate(child_paths)
+ ]
+ )
ctx.code_artifact_metadata = {
label: {
"block_label": label,
- "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": claimed_schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": child_paths, "extraction_schema": terminal_schema}
- ],
+ "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
- {
- "output_path": "output.locations[].address",
- "output_root": "output",
- "value_status": "no_typed_value",
- },
- {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
- ],
- )
candidate_yaml = _yaml(
f"""
title: Entry lookup
@@ -5275,12753 +4829,5177 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is False
- assert "extraction_schema" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.locations[].address",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.statuses",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- ]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "metadata_reject"
- assert repair_context["block_label"] == label
- assert repair_context["runtime_failure_class"] == "recorded_outcome_missing_output_coverage"
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
@pytest.mark.asyncio
- async def test_requested_output_contract_rejects_collapsed_candidate_before_recorded_outcome(
+ async def test_authoritative_outcome_not_demonstrated_allows_static_return_for_required_root(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_entry"
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_locations",
- output_path="output.locations",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_statuses",
- output_path="output.statuses",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
+ label = "extract_top_hn_post"
ctx.code_artifact_metadata = {
label: {
"block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["output"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
+ "claimed_outcomes": [{"goal_value_paths": ["top_post"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["top_post"]}],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},
+ ],
+ )
candidate_yaml = _yaml(
f"""
- title: Entry lookup
+ title: Hacker News lookup
workflow_definition:
blocks:
- block_type: code
label: {label}
code: |
- return {{"output": {{"summary": "found"}}}}
+ return {{"top_post": "Claude Sonnet 5", "rank": 1}}
"""
)
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is False
- assert "requested output contract" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.locations",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.statuses",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- ]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "metadata_reject"
- assert repair_context["block_label"] == label
- assert repair_context["runtime_failure_class"] == "requested_output_contract_missing_output_coverage"
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
- def test_single_output_contract_evaluator_reports_all_deficiency_classes(self) -> None:
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_candidate_covering_missing_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [
+ {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
+ ],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
+ ],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ ],
)
- workflow_yaml = _yaml(
- """
- title: Entry lookup
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: value
blocks:
- block_type: code
- label: extract_record
- parameter_keys:
- - value
+ label: {label}
code: |
- _scout_entry_target = page.locator("#filter")
- try:
- await _scout_entry_target.wait_for(state="visible", timeout=1000)
- except Exception:
- await page.goto("https://example.com/records", wait_until="domcontentloaded")
- await _scout_entry_target.wait_for(state="visible")
- await page.locator("#filter").fill(str(value))
- await page.locator("#choose").click()
- await page.wait_for_load_state("domcontentloaded")
- return {"output": {"summary": "found"}}
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ address = "North Carolina, USA"
+ credentialing_status = "unknown"
+ locations = []
+ statuses = []
"""
)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.has_deficiencies is True
- assert evaluation.block_label == "extract_record"
- assert evaluation.missing_metadata_paths == ["output.record_id"]
- assert evaluation.missing_schema_paths == ["output.record_id"]
- assert evaluation.missing_return_paths == ["output.record_id"]
- assert evaluation.shape_violations == ["separated_spine_shape_required"]
- assert evaluation.payload["satisfying_templates"]["code_artifact_metadata"]["block_label"] == "extract_record"
- assert evaluation.payload["satisfying_templates"]["return_skeleton"] == (
- 'return {"output": {"record_id": record_id}}'
- )
-
- def test_schema_template_derives_nested_array_paths_without_semantic_values(self) -> None:
- schema = workflow_update_module._schema_template_for_required_paths(
- {"output.npi", "output.locations[].address", "output.locations[].status"}
- )
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- output = schema["properties"]["output"]
- assert output["type"] == "object"
- assert output["required"] == ["locations", "npi"]
- locations = output["properties"]["locations"]
- assert locations["type"] == "array"
- assert locations["items"]["required"] == ["address", "status"]
- assert locations["items"]["properties"]["address"] == {}
- assert locations["items"]["properties"]["status"] == {}
- assert "description" not in json.dumps(schema)
- assert "example" not in json.dumps(schema)
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
@pytest.mark.asyncio
- async def test_update_and_run_preflight_and_update_workflow_share_contract_signature(
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_returning_missing_output_roots(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- workflow_yaml = _yaml(
- """
- title: Entry lookup
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
blocks:
- block_type: code
- label: extract_entry_output
+ label: {label}
code: |
- return {"output": {"summary": "found"}}
+ async def extract():
+ return {{"address": "North Carolina", "credentialing_status": "unknown"}}
+
+ return await extract()
"""
)
- repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- update_ctx = _code_only_ctx()
- update_ctx.turn_id = "shared-contract-turn"
- update_ctx.last_code_authoring_repair_context = repair_context
- run_ctx = _code_only_ctx()
- run_ctx.turn_id = "shared-contract-turn"
- run_ctx.last_code_authoring_repair_context = repair_context
- update_result = await _update_workflow(
- {"workflow_yaml": workflow_yaml}, update_ctx, allow_missing_credentials=True
- )
- run_result = workflow_update_module._metadata_contract_run_preflight_reject(run_ctx, workflow_yaml, [])
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert run_result is not None
- assert update_result["ok"] is False
- assert run_result["ok"] is False
- assert (
- update_result["data"]["canonical_output_contract_signature"]
- == (run_result["data"]["canonical_output_contract_signature"])
- )
- assert (
- update_result["data"]["canonical_required_child_paths"]
- == (run_result["data"]["canonical_required_child_paths"])
- )
- assert update_result["data"]["satisfying_templates"] == run_result["data"]["satisfying_templates"]
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
- def test_static_return_uncertainty_requires_typed_advisory_grant(self) -> None:
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_local_dict_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- schema = workflow_update_module._schema_template_text_for_required_paths({"output.record_id"})
- metadata = [
- {
- "block_label": "extract_entry_output",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
}
- ]
- workflow_yaml = _yaml(
- """
- title: Entry lookup
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
blocks:
- block_type: code
- label: extract_entry_output
+ label: {label}
code: |
- return {"output": build_output()}
+ async def extract():
+ result = {{"address": "North Carolina", "credentialing_status": "unknown"}}
+ return result
+
+ return await extract()
"""
)
- update_eval = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, allow_static_return_advisory=False
- )
- run_eval = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, allow_static_return_advisory=True
- )
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert update_eval is not None
- assert update_eval.missing_return_paths == ["output.record_id"]
- assert update_eval.can_attempt_run is False
- assert run_eval is not None
- assert run_eval.missing_return_paths == ["output.record_id"]
- assert run_eval.can_attempt_run is False
-
- workflow_update_module._grant_output_contract_advisory_run(ctx, run_eval.canonical_signature)
- granted_eval = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, allow_static_return_advisory=True
- )
-
- assert granted_eval is not None
- assert granted_eval.missing_return_paths == []
- assert granted_eval.payload["static_return_advisory_paths"] == ["output.record_id"]
- assert granted_eval.payload["actuated_static_return_advisory"] is True
- assert granted_eval.can_attempt_run is True
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
- def test_complete_metadata_without_code_return_is_run_preflight_advisory(self) -> None:
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_top_level_literal_key_dict_updates(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- required_paths = {"output.account_number", "output.confirmation_number"}
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_account",
- output_path="output.account_number",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_confirmation",
- output_path="output.confirmation_number",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
- schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
- metadata = [
- {
- "block_label": "submit_service_request",
- "claimed_outcomes": [{"goal_value_paths": sorted(required_paths), "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": sorted(required_paths), "extraction_schema": schema}
- ],
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
}
- ]
- workflow_yaml = _yaml(
- """
- title: Service request
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
blocks:
- block_type: code
- label: submit_service_request
+ label: {label}
code: |
- await page.get_by_role("button", name="Submit").click()
+ result = {{"address": "North Carolina"}}
+ result["credentialing_status"] = "unknown"
+ return result
"""
)
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
- workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, allow_static_return_advisory=True
- )
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert evaluation is not None
- assert evaluation.can_attempt_run is True
- assert evaluation.missing_return_paths == []
- assert evaluation.payload["static_return_advisory_paths"] == sorted(required_paths)
- assert evaluation.payload["actuated_static_return_advisory"] is True
- assert result is None
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
@pytest.mark.asyncio
- async def test_save_only_update_rejects_static_return_gap_without_typed_grant(
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_literal_key_dict_updates(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
)
- workflow_yaml = _yaml(
- """
- title: Entry lookup
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
blocks:
- block_type: code
- label: extract_entry_output
+ label: {label}
code: |
- return "not a structured output"
+ async def extract():
+ result = {{}}
+ result["address"] = "North Carolina"
+ result["credentialing_status"] = "unknown"
+ return result
+
+ return await extract()
"""
)
- ctx.turn_id = "save-only-static-return-gap"
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml},
- ctx,
- allow_missing_credentials=True,
- )
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "output_contract_required"
- assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
- assert result["data"]["static_return_advisory_paths"] == []
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
@pytest.mark.asyncio
- async def test_run_path_allows_typed_advisory_declared_output_return_shape_gap(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_captured_select_option_allows_authored_select_api(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- workflow_yaml = _yaml(
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans",
+ "value": "gold",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
"""
- title: Entry lookup
+ title: Plan lookup
workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
blocks:
- block_type: code
- label: extract_entry_output
+ label: plan_selection
+ parameter_keys:
+ - plan
code: |
- return "not a structured output"
+ await page.locator("#planSelect").select_option(label=str(plan))
+ return {"selected_plan": str(plan)}
"""
)
- ctx.turn_id = "run-static-return-gap"
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths={"output.record_id"})
- workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
- "output.record_id"
- ]
- assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["extraction_schema"]
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- block = _single_code_block(parsed)
- schema = json.loads(block["extraction_schema"])
- assert schema["properties"]["output"]["properties"]["record_id"] == {}
+ assert ctx.workflow_yaml == submitted
+ assert ctx.last_code_authoring_repair_context is None
@pytest.mark.asyncio
- async def test_run_path_persists_effective_output_contract_for_readback(
+ async def test_captured_select_option_allows_later_unrelated_text_click(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- required_paths = {"output.record_id"}
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
- metadata = [
+ ctx.scout_trajectory = [
{
- "block_label": "extract_entry_output",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans",
+ "value": "gold",
+ "trajectory_index": 0,
}
]
- workflow_yaml = _yaml(
+ submitted = _yaml(
"""
- title: Entry lookup
+ title: Plan lookup
workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
blocks:
- block_type: code
- label: extract_entry_output
+ label: plan_selection
+ parameter_keys:
+ - plan
code: |
- return {"output": {"record_id": "ABC123"}}
+ await page.locator("#planSelect").select_option(label=str(plan))
+ return {"selected_plan": str(plan)}
+ - block_type: code
+ label: continue_from_plan
+ code: |
+ await page.get_by_text("Continue", exact=True).click()
+ return {"continued": True}
"""
)
- def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
- return SimpleNamespace(
- title="Entry lookup",
- description=None,
- workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
- proxy_location=None,
- webhook_callback_url=None,
- totp_verification_url=None,
- totp_identifier=None,
- persist_browser_session=False,
- pin_saved_session_ip=False,
- browser_profile_id=None,
- browser_profile_key=None,
- model=None,
- max_screenshot_scrolls=None,
- extra_http_headers=None,
- cdp_connect_headers=None,
- run_with=None,
- ai_fallback=None,
- cache_key=None,
- adaptive_caching=None,
- enable_self_healing=None,
- code_version=None,
- run_sequentially=False,
- sequential_key=None,
- )
-
- persisted: dict[str, object] = {}
-
- async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
- return fake_workflow(blocks=[])
-
- async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
- return fake_workflow(blocks=[SimpleNamespace(label="extract_entry_output")])
-
- async def fake_update_workflow_definition(**kwargs: object) -> None:
- persisted.update(kwargs)
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
- monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
- monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
- monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(
- workflow_update_module.app.WORKFLOW_SERVICE,
- "update_workflow_definition",
- fake_update_workflow_definition,
- )
- monkeypatch.setattr(
- workflow_update_module,
- "resolve_copilot_created_by_stamp",
- AsyncMock(return_value="copilot"),
- )
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+ assert ctx.last_code_authoring_repair_context is None
+ @pytest.mark.asyncio
+ async def test_credential_scout_reject_is_not_classified_as_progress(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ {
+ "workflow_yaml": _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ """
+ )
+ },
ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
)
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["data"].get("surface_kind") is None
- assert result["ok"] is True
- assert persisted["workflow_id"] == "w"
- definition = persisted["workflow_definition"]
- assert definition.blocks == [SimpleNamespace(label="extract_entry_output")]
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- block = _single_code_block(parsed)
- assert json.loads(block["extraction_schema"])["properties"]["output"]["properties"]["record_id"] == {}
- @pytest.mark.asyncio
- async def test_run_path_persists_blocks_for_api_readback_when_metadata_missing(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _code_only_ctx()
- workflow_yaml = _yaml(
- """
- title: Public path validation
- workflow_definition:
- blocks:
- - block_type: code
- label: validate_public_path
- code: |
- return {"public_form_exists": False}
- """
- )
+_SPINE_SYNTH_CODE = 'await page.locator("#stage-a").click()\nawait page.locator("#stage-b").click()'
- def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
- return SimpleNamespace(
- title="Public path validation",
- description=None,
- workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
- proxy_location=None,
- webhook_callback_url=None,
- totp_verification_url=None,
- totp_identifier=None,
- persist_browser_session=False,
- pin_saved_session_ip=False,
- browser_profile_id=None,
- browser_profile_key=None,
- model=None,
- max_screenshot_scrolls=None,
- extra_http_headers=None,
- cdp_connect_headers=None,
- run_with=None,
- ai_fallback=None,
- cache_key=None,
- adaptive_caching=None,
- enable_self_healing=None,
- code_version=None,
- run_sequentially=False,
- sequential_key=None,
- )
- persisted: dict[str, object] = {}
+def _fake_spine_synthesized(
+ *,
+ parameters: list[dict[str, str]] | None = None,
+ steps: list[dict[str, int]] | None = None,
+ code: str | None = None,
+ diagnostics: SynthesisDiagnostics | None = None,
+) -> SynthesizedCodeBlock:
+ return SynthesizedCodeBlock(
+ code=code if code is not None else _SPINE_SYNTH_CODE,
+ parameters=parameters if parameters is not None else [],
+ steps=steps if steps is not None else [{"line_start": 1, "line_end": 1}, {"line_start": 2, "line_end": 2}],
+ diagnostics=diagnostics if diagnostics is not None else SynthesisDiagnostics(),
+ )
- async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
- return fake_workflow(blocks=[])
- async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
- return fake_workflow(blocks=[SimpleNamespace(label="validate_public_path")])
+def _spine_emission_diagnostics() -> SynthesisDiagnostics:
+ return SynthesisDiagnostics(
+ emitted_interaction_count=2,
+ emitted_interactions=[
+ {
+ "trajectory_index": 0,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-a",
+ "locator": 'page.locator("#stage-a")',
+ },
+ {
+ "trajectory_index": 1,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-b",
+ "locator": 'page.locator("#stage-b")',
+ },
+ ],
+ )
- async def fake_update_workflow_definition(**kwargs: object) -> None:
- persisted.update(kwargs)
- monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
- monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
- monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(
- workflow_update_module.app.WORKFLOW_SERVICE,
- "update_workflow_definition",
- fake_update_workflow_definition,
- )
- monkeypatch.setattr(
- workflow_update_module,
- "resolve_copilot_created_by_stamp",
- AsyncMock(return_value="copilot"),
- )
+def _spine_actuation_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "t-spine"
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#stage-a", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#stage-b", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ return ctx
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
- assert result["ok"] is True
- assert persisted["workflow_id"] == "w"
- assert persisted["workflow_definition"].blocks == [SimpleNamespace(label="validate_public_path")]
+def _collapsed_spine_yaml(code_body: str) -> str:
+ indented = textwrap.indent(textwrap.dedent(code_body).strip(), " " * 10)
+ return _yaml(
+ "title: Entry lookup\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: extract_record\n"
+ " code: |\n"
+ f"{indented}\n"
+ )
- @pytest.mark.asyncio
- async def test_output_contract_reject_budget_counts_same_signature_across_classes(
+
+class TestSeparatedSpineViolationActuation:
+ def test_branch_a_split_replaces_collapsed_owner_with_stages_plus_extraction(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
- """
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
)
- result = None
- for _ in range(4):
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+ new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
- assert result is not None
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["output_contract_reject_count"] == 4
- assert result["data"]["canonical_required_child_paths"] == ["output.record_id"]
+ assert applied is True
+ blocks = workflow_blocks(parse_workflow_yaml(new_yaml))
+ assert [block.get("label") for block in blocks] == [
+ "extract_record_browser_stage_1",
+ "extract_record_browser_stage_2",
+ "extract_record",
+ ]
+ retained_code = str(blocks[-1].get("code") or "")
+ assert "inner_text" in retained_code
+ assert ".click()" not in retained_code
- def test_output_contract_signature_uses_stable_scope_and_required_path_identity(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "contract-scope-a"
- first = workflow_update_module._output_contract_signature(
- ctx=ctx,
- required_paths={"output.locations[].address", "output.statuses"},
- )
- second = workflow_update_module._output_contract_signature(
- ctx=ctx,
- required_paths={"output.statuses", "output.locations[].address"},
- )
- other_ctx = _code_only_ctx()
- other_ctx.turn_id = "contract-scope-b"
- other_scope = workflow_update_module._output_contract_signature(
- ctx=other_ctx,
- required_paths={"output.statuses", "output.locations[].address"},
- )
-
- assert first == second
- assert other_scope != first
- assert (
- workflow_update_module._output_contract_pin_key(
- ctx,
- "title: First\nworkflow_definition:\n blocks: []\n",
- {"output.locations[].address", "output.statuses"},
- )
- == first
+ def test_branch_a_result_is_idempotent_no_resplit(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
)
- def test_output_contract_family_counter_aggregates_reject_classes(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "contract-budget-scope"
- required_paths = {"output.record_id"}
- first = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="metadata_reject",
- )
- second = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="synthesized_parameter_binding_ambiguous",
+ split_yaml, _meta, _applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
- third = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="declared_output_return_shape",
+ again_yaml, _meta2, _applied2 = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, split_yaml, []
)
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
- assert (first, second, third) == (1, 2, 3)
- assert ctx.output_contract_reject_count_by_signature[signature] == 3
-
- def test_output_contract_family_counter_is_scoped(self) -> None:
- ctx = _code_only_ctx()
- required_paths = {"output.status"}
+ assert [block.get("label") for block in workflow_blocks(parse_workflow_yaml(again_yaml))] == [
+ "extract_record_browser_stage_1",
+ "extract_record_browser_stage_2",
+ "extract_record",
+ ]
+ assert not ctx.output_contract_spine_directive_blockers_by_attempt_key
- ctx.turn_id = "first-build-goal"
- first_scope_count = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="metadata_reject",
- )
- ctx.turn_id = "second-build-goal"
- second_scope_count = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="metadata_reject",
- )
- ctx.turn_id = "first-build-goal"
- first_scope_again = workflow_update_module._record_output_contract_family_reject(
- ctx,
- required_paths,
- reject_family="output_contract_required",
+ def test_branch_b_arms_directive_and_returns_unchanged_yaml(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
)
- assert (first_scope_count, second_scope_count, first_scope_again) == (1, 1, 2)
- assert len(ctx.output_contract_reject_count_by_signature) == 2
-
- def test_output_contract_owner_pin_is_scoped(self) -> None:
- ctx = _code_only_ctx()
- workflow_yaml = _yaml(
- """
- title: Shared output shape
- workflow_definition:
- blocks:
- - block_type: code
- label: first_goal_output
- code: |
- return {"output": {"status": "one"}}
- - block_type: code
- label: second_goal_output
- code: |
- return {"output": {"status": "two"}}
- """
+ new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
- required_paths = {"output.status"}
- ctx.turn_id = "first-build-goal"
- workflow_update_module._pin_output_contract_block_label(
- ctx,
- workflow_yaml,
- required_paths,
- "first_goal_output",
- )
- ctx.turn_id = "second-build-goal"
- workflow_update_module._pin_output_contract_block_label(
- ctx,
- workflow_yaml,
- required_paths,
- "second_goal_output",
+ assert applied is False
+ assert new_yaml == workflow_yaml
+ signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
+ attempt_key = workflow_update_module._output_contract_spine_directive_attempt_key(
+ signature=signature, block_label="extract_record", workflow_yaml=workflow_yaml
)
+ assert ctx.output_contract_spine_directive_blockers_by_attempt_key[attempt_key] == [
+ "extraction_boundary_ambiguous"
+ ]
- assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
- "second_goal_output"
- )
- ctx.turn_id = "first-build-goal"
- assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
- "first_goal_output"
+ @pytest.mark.parametrize(
+ "code_body, synth_kwargs, expected_blocker",
+ [
+ (
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}',
+ {},
+ "extraction_boundary_ambiguous",
+ ),
+ (
+ _SPINE_SYNTH_CODE + '\nawait page.locator("#extra").click()\nreturn {"output": {"record_id": "X"}}',
+ {},
+ "extraction_suffix_contains_browser_actions",
+ ),
+ (
+ _SPINE_SYNTH_CODE + '\nvalue = await page.locator("#result").inner_text()',
+ {},
+ "static_return_envelope_unavailable",
+ ),
+ (
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}',
+ {"steps": [{"line_start": 1, "line_end": 2}]},
+ "insufficient_durable_stages",
+ ),
+ (
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}',
+ {"parameters": [{"key": "alpha"}, {"key": "alpha"}]},
+ "parameter_reconciliation_failed",
+ ),
+ ],
+ )
+ def test_branch_b_precondition_failures_arm_matching_blocker(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ code_body: str,
+ synth_kwargs: dict[str, object],
+ expected_blocker: str,
+ ) -> None:
+ monkeypatch.setattr(
+ workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized(**synth_kwargs)
)
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(code_body)
- def test_metadata_contract_recorded_key_stable_across_reject_count_and_reason_drift(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "stable-contract-key"
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
+ _new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
- workflow_yaml = _yaml(
- """
- title: First candidate title
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
- """
+
+ assert applied is False
+ armed = list(ctx.output_contract_spine_directive_blockers_by_attempt_key.values())
+ assert armed == [[expected_blocker]]
+
+ def test_evaluation_enriches_repair_context_and_payload_after_directive_armed(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
)
+
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
assert evaluation is not None
+ assert evaluation.shape_violations == ["separated_spine_shape_required"]
+ assert evaluation.repair_context is not None
+ assert evaluation.repair_context.required_block_structure == "separated_browser_spine_plus_extraction"
+ assert evaluation.repair_context.spine_split_blockers == ["extraction_boundary_ambiguous"]
+ assert evaluation.payload["spine_structure_directive"]["spine_split_blockers"] == [
+ "extraction_boundary_ambiguous"
+ ]
- workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="First reject wording.",
+ def test_directive_renders_into_next_authoring_prompt(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
)
- first_key = ctx.latest_recorded_build_test_outcome.structural_key
- assert first_key is not None
- evaluation.payload["reason_code"] = "declared_output_return_shape"
- workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Changed reject wording and reason site.",
- )
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ assert evaluation is not None
+ ctx.last_code_authoring_repair_context = evaluation.repair_context
- assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
- assert latest_recorded_build_test_outcome_repeated(ctx) is True
- assert ctx.latest_recorded_build_test_outcome.missing_requested_output_facts
+ rendered = agent_module._code_authoring_repair_context_prompt(ctx)
- def test_metadata_contract_recorded_key_keeps_distinct_scope_and_required_paths(self) -> None:
- workflow_yaml = _yaml(
- """
- title: Shared shape
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
- """
+ assert "required_block_structure: separated_browser_spine_plus_extraction" in rendered
+ assert "spine_split_blockers:" in rendered
+ assert "one browser-stage code block per scouted mutation stage" in rendered
+
+ def test_cosmetic_churn_escalates_to_advisory_instead_of_rearming(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ first_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
+ )
+ cosmetic_yaml = _collapsed_spine_yaml(
+ "_setup = 2\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
)
- def recorded_key(turn_id: str, output_path: str) -> str:
- ctx = _code_only_ctx()
- ctx.turn_id = turn_id
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path=output_path,
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert evaluation is not None
- workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="Rejected.")
- key = ctx.latest_recorded_build_test_outcome.structural_key
- assert key is not None
- return key
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, first_yaml, [])
+ assert len(ctx.output_contract_spine_directive_blockers_by_attempt_key) == 1
+ assert not ctx.output_contract_actuation_by_signature
+ assert ctx.turn_halt is None
- first_scope_key = recorded_key("first-build-goal", "output.status")
- second_scope_key = recorded_key("second-build-goal", "output.status")
- other_path_key = recorded_key("first-build-goal", "output.npi")
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, cosmetic_yaml, [])
+ assert list(ctx.output_contract_actuation_by_signature.values()) == [OutputContractAdvisoryState.GRANTED]
+ assert ctx.turn_halt is None
+ assert len(ctx.output_contract_spine_directive_blockers_by_attempt_key) == 1
- assert first_scope_key != second_scope_key
- assert first_scope_key != other_path_key
- @pytest.mark.asyncio
- async def test_output_contract_first_contact_imposes_keyed_return_envelope(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_flags",
- output_path="output.flags",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- record_id = "ABC123"
- flags = ["enabled"]
- """
- )
+def _dual_output_owner_yaml() -> str:
+ return _yaml(
+ "title: Entry lookup\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: extract_a\n"
+ " code: |\n"
+ ' return {"output": {"record_id": "A"}}\n'
+ " - block_type: code\n"
+ " label: extract_b\n"
+ " code: |\n"
+ ' return {"output": {"record_id": "B"}}\n'
+ )
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code = str(_single_code_block(parsed)["code"])
- assert 'return {"output": {"flags": flags, "record_id": record_id}}' in code
- metadata = ctx.code_artifact_metadata["extract_entry_output"]
- assert metadata["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
- assert (
- json.loads(metadata["claimed_outcomes"][0]["extraction_schema"])["properties"]["output"]["properties"][
- "record_id"
- ]
- == {}
- )
+class TestAmbiguousOutputOwnerActuation:
+ def _signature(self) -> str:
+ return workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- @pytest.mark.asyncio
- async def test_requested_output_contract_routes_ambiguous_output_target_to_typed_repair(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- ctx.code_artifact_metadata = {
- "lookup_entry": {
- "block_label": "lookup_entry",
- "claimed_outcomes": [{"goal_value_paths": ["output"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- candidate_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: open_entry
- code: |
- await page.goto("https://example.com/search")
- - block_type: code
- label: lookup_entry
- code: |
- return {"output": {"summary": "found"}}
- """
- )
+ def test_ambiguous_owner_arms_directive_instead_of_bailing(self) -> None:
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _dual_output_owner_yaml()
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
- assert result["ok"] is False
- assert "requested output contract" in result["error"]
- assert result["data"]["authoring_repair_context"]["reason_code"] == "output_owner_ambiguous"
- assert ctx.last_code_authoring_repair_context is not None
- assert ctx.last_code_authoring_repair_context.reason_code == "output_owner_ambiguous"
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- }
+ assert applied is False
+ assert new_yaml == workflow_yaml
+ assert ctx.output_contract_output_owner_directive_candidates_by_signature[self._signature()] == [
+ "extract_a",
+ "extract_b",
]
- assert result["data"]["metadata_repair_contract"] is None
- assert result["data"]["block_label"] == ""
- assert result["data"]["shape_violations"] == ["missing_output_owner"]
- @pytest.mark.asyncio
- async def test_requested_output_contract_omits_repair_context_when_schema_target_is_ambiguous(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- child_paths = ["output.npi"]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- ctx.code_artifact_metadata = {
- "lookup_entry": {
- "block_label": "lookup_entry",
- "claimed_outcomes": [
- {
- "goal_value_paths": child_paths,
- "extraction_schema": '{"type":"object","properties":{"npi":{"type":"string"}}}',
- }
- ],
- "terminal_verifier_expectations": [{"goal_value_paths": child_paths}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- candidate_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: open_entry
- code: |
- await page.goto("https://example.com/search")
- - block_type: code
- label: lookup_entry
- code: |
- return {"output": {"npi": "1234567890"}}
- """
- )
+ def test_evaluation_enriches_owner_ambiguity_repair_context_after_directive_armed(self) -> None:
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _dual_output_owner_yaml()
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert result["ok"] is False
- assert "extraction_schema" in result["error"]
- assert result["data"]["authoring_repair_context"]["block_label"] == "lookup_entry"
- assert ctx.last_code_authoring_repair_context is not None
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- }
+ assert evaluation is not None
+ assert "ambiguous_output_owner" in evaluation.shape_violations
+ assert evaluation.repair_context is not None
+ assert evaluation.repair_context.reason_code == "output_owner_ambiguous"
+ assert evaluation.repair_context.output_owner_candidate_labels == ["extract_a", "extract_b"]
+ assert evaluation.payload["output_owner_directive"]["output_owner_candidate_labels"] == [
+ "extract_a",
+ "extract_b",
]
- assert result["data"]["metadata_repair_contract"]["block_label"] == "lookup_entry"
- def test_metadata_contract_run_preflight_blocks_missing_metadata_before_run(self) -> None:
- ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id", "output.flags"],
- required_extraction_schema_paths=["output.record_id", "output.flags"],
- required_code_return_paths=["output.record_id", "output.flags"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
- """
- )
+ def test_owner_directive_renders_into_next_authoring_prompt(self) -> None:
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _dual_output_owner_yaml()
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ assert evaluation is not None
+ ctx.last_code_authoring_repair_context = evaluation.repair_context
- assert result is not None
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in result["data"]
- assert result["data"]["value_bearing_convergence_directive"]["rung"] == 1
- assert result["data"]["block_label"] == "extract_entry_output"
- assert result["data"]["canonical_required_child_paths"] == ["output.flags", "output.record_id"]
- assert result["data"]["missing_goal_value_paths"] == ["output.flags", "output.record_id"]
- assert result["data"]["missing_extraction_schema_paths"] == ["output.flags", "output.record_id"]
- assert result["data"]["missing_code_return_paths"] == ["output.flags", "output.record_id"]
- assert result["data"]["metadata_contract_source"] == "requested_output_contract"
- assert result["data"]["metadata_contract_reason_code"] == ("requested_output_contract_missing_output_coverage")
- assert result["data"]["metadata_repair_contract"] == {
- "block_label": "extract_entry_output",
- "required_goal_value_paths": ["output.flags", "output.record_id"],
- "required_extraction_schema_paths": ["output.flags", "output.record_id"],
- "required_code_return_paths": ["output.flags", "output.record_id"],
- "source": "requested_output_contract",
- "reason_code": "requested_output_contract_missing_output_coverage",
- }
- assert result["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
- "output.flags",
- "output.record_id",
- ]
- assert result["data"]["missing_requested_output_facts"] == [
- {
- "output_path": "output.flags",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.record_id",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- ]
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
- assert sanitized["data"]["authoring_repair_context"]["metadata_contract_source"] == (
- "requested_output_contract"
- )
- assert sanitized["data"]["metadata_repair_contract"] == result["data"]["metadata_repair_contract"]
+ rendered = agent_module._code_authoring_repair_context_prompt(ctx)
- def test_metadata_contract_run_preflight_preserves_run_backed_convergence_terminal(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_account",
- output_path="output.account_number",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- workflow_yaml = _yaml(
- """
- title: Service request
- workflow_definition:
- blocks:
- - block_type: code
- label: submit_service_request
- code: |
- await page.get_by_role("button", name="Submit").click()
- """
- )
- metadata = [
- {
- "artifact_id": "code_artifact:submit_service_request",
- "block_label": "submit_service_request",
- "claimed_outcomes": [
- {
- "goal_value_paths": ["output.account_number"],
- "extraction_schema": (
- '{"type":"object","properties":{"output":{"type":"object","properties":'
- '{"account_number":{}}}}}'
- ),
- }
- ],
- }
- ]
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_recorded",
- block_labels=["submit_service_request"],
- structural_failure_identity="completion:typed-output",
- authored_structure_signature="previous-authoring-signature",
- )
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_recorded"
- recorded_sig = authored_block_signatures_from_workflow(workflow_yaml, metadata)["submit_service_request"]
- ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=ctx.latest_recorded_build_test_outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="outcome_not_demonstrated",
- frontier_facet="value_shape",
- owning_block_labels=["submit_service_request"],
- diagnostic_reason="empty_page",
- workflow_run_id="wr_recorded",
- recorded_block_signatures={"submit_service_request": recorded_sig},
- )
+ assert "output_owner_candidate_labels: extract_a, extract_b" in rendered
+ assert "sole output owner" in rendered
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+ def test_owner_directive_emits_fingerprint_once_per_signature(self) -> None:
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _dual_output_owner_yaml()
- assert result is not None
- assert result["ok"] is False
- assert "frontier" in result["error"]
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "unchanged_after_recorded_outcome"
- assert outcome.workflow_run_id is None
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- def test_metadata_contract_run_preflight_budget_remains_repair_before_run(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "preflight-no-run-contract"
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ assert list(ctx.output_contract_output_owner_directive_candidates_by_signature.keys()) == [self._signature()]
+
+
+def _already_split_spine_yaml(extra_sibling_code: str | None = None, *, base: str | None = None) -> str:
+ base = base or workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ extra_block = ""
+ if extra_sibling_code is not None:
+ extra_block = (
+ f" - block_type: code\n label: {base}_browser_stage_extra\n code: |\n"
+ + textwrap.indent(textwrap.dedent(extra_sibling_code).strip(), " " * 6)
+ + "\n"
)
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"summary": "found"}}
- """
+ return _yaml(
+ "title: Entry lookup\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ f" - block_type: code\n label: {base}_browser_stage_1\n code: |\n"
+ ' await page.locator("#stage-a").click()\n'
+ f" - block_type: code\n label: {base}_browser_stage_2\n code: |\n"
+ ' await page.locator("#stage-b").click()\n'
+ f"{extra_block}"
+ f" - block_type: code\n label: {base}\n code: |\n"
+ ' value = await page.locator("#result").inner_text()\n'
+ ' return {"output": {"record_id": value}}\n'
+ )
+
+
+def _imposition_split_ctx() -> CopilotContext:
+ ctx = _spine_actuation_ctx()
+ ctx.impose_synthesized_code_block = True
+ ctx.raw_code_artifact_metadata = [
+ {
+ "block_label": workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": '{"type":"object"}'}],
+ }
+ ]
+ return ctx
+
+
+class TestSeparatedSpineImpositionRunEligibility:
+ def test_imposition_accepts_scouted_browser_stage_siblings(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _imposition_split_ctx()
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_already_split_spine_yaml(), ctx)
+
+ assert result.violations == []
+
+ def test_imposition_still_flags_unscouted_sibling_mutation(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _imposition_split_ctx()
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(
+ _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
)
- results = [
- workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, []) for _ in range(4)
- ]
+ assert any("unscouted browser action" in violation for violation in result.violations)
+ assert any("#hallucinated" in violation for violation in result.violations)
+ flagged_actions = [violation.split(" Provenance: ")[0] for violation in result.violations]
+ assert not any("#stage-a" in flagged for flagged in flagged_actions)
- assert results[0] is not None
- assert results[0]["ok"] is False
- assert results[0]["data"]["reason_code"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in results[0]["data"]
- assert results[0]["data"]["value_bearing_convergence_directive"]["rung"] == 1
- assert results[0]["data"]["output_contract_reject_count"] == 1
- assert results[-1] is not None
- assert results[-1]["data"]["output_contract_reject_count"] == 4
- assert results[-1]["data"]["reason_code"] == "metadata_contract_required_before_run"
- assert results[-1]["data"]["output_contract_reason_code"] == "value_bearing_output_required"
- assert all(
- result is None or result["data"]["reason_code"] != "output_contract_reject_budget_exhausted"
- for result in results
+ def test_whole_trajectory_validation_exempts_spine_covered_sibling_mutations(self) -> None:
+ parsed = parse_workflow_yaml(_already_split_spine_yaml())
+ blocks = workflow_update_module._workflow_code_blocks(parsed)
+ extraction_block = next(
+ block for block in blocks if block.get("label") == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
)
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- def test_metadata_contract_run_preflight_accepts_complete_submitted_metadata(self) -> None:
- ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id", "output.flags"],
- required_extraction_schema_paths=["output.record_id", "output.flags"],
- required_code_return_paths=["output.record_id", "output.flags"],
- metadata_contract_source="recorded_outcome",
- metadata_contract_reason_code="recorded_outcome_missing_output_coverage",
+ validation = workflow_update_module._whole_trajectory_browser_surface_violations(
+ code_blocks=blocks,
+ selected_code_block=extraction_block,
+ submitted_selected_code=str(extraction_block.get("code") or ""),
+ synthesized_code=_SPINE_SYNTH_CODE,
)
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{'
- '"record_id":{"type":"string"},'
- '"flags":{"type":"array","items":{"type":"string"}}}}}}'
+
+ assert validation.violations == []
+
+ def test_armed_attempt_key_survives_scaffold_metadata_owner_shift(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
+ )
+
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
+ armed_keys = set(ctx.output_contract_spine_directive_blockers_by_attempt_key)
+ assert len(armed_keys) == 1
+
+ scaffolded_metadata, _applied = workflow_update_module._scaffold_metadata_contract_for_update(
+ ctx, workflow_yaml, []
+ )
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ required_paths = contract.union
+ read_label, _owner_labels = workflow_update_module._target_output_contract_block_label(
+ ctx, workflow_yaml, scaffolded_metadata, required_paths
+ )
+ read_signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
+ read_key = workflow_update_module._output_contract_spine_directive_attempt_key(
+ signature=read_signature, block_label=read_label, workflow_yaml=workflow_yaml
+ )
+
+ assert read_key in armed_keys
+
+ def test_granted_structural_advisory_lifts_output_contract_run_gate_on_unsplit_draft(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _spine_actuation_ctx()
+ collapsed_yaml = _collapsed_spine_yaml(
+ "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
+ )
+
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, collapsed_yaml, [])
+ blocked = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, collapsed_yaml, [], allow_static_return_advisory=True
+ )
+ assert blocked is not None and blocked.can_attempt_run is False
+
+ workflow_update_module._impose_output_contract_envelope_after_steering(ctx, collapsed_yaml, [])
+ assert list(ctx.output_contract_actuation_by_signature.values()) == [OutputContractAdvisoryState.GRANTED]
+
+ scaffolded_metadata, _applied = workflow_update_module._scaffold_metadata_contract_for_update(
+ ctx, collapsed_yaml, []
+ )
+ granted = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, collapsed_yaml, scaffolded_metadata, allow_static_return_advisory=True
)
+ assert granted is not None and granted.can_attempt_run is True
+
+
+def _budget_evaluation(ctx: CopilotContext) -> _OutputContractEvaluation:
+ workflow_yaml = _collapsed_spine_yaml(
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ assert evaluation is not None
+ return evaluation
+
+
+class TestCodeBlockParameterPersistSeam:
+ def test_declared_credential_key_is_adopted_for_unresolved_name(self) -> None:
workflow_yaml = _yaml(
"""
- title: Entry lookup
+ title: Portal login
workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: portal_credentials
+ default_value: cred_123
blocks:
- block_type: code
- label: extract_entry_output
+ label: sign_in
code: |
- return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
+ await page.locator("#user").fill(portal_credentials.username)
"""
)
- metadata = [
- {
- "block_label": "extract_entry_output",
- "claimed_outcomes": [
- {"goal_value_paths": ["output.record_id", "output.flags"], "extraction_schema": schema}
- ],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id", "output.flags"], "extraction_schema": schema}
- ],
- }
- ]
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+ adopted = workflow_update_module._adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
- assert result is None
+ parsed = parse_workflow_yaml(adopted)
+ blocks = workflow_blocks(parsed)
+ assert blocks[0]["parameter_keys"] == ["portal_credentials"]
- def test_metadata_contract_run_preflight_rejects_root_only_output(self) -> None:
- ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
+ def test_output_contract_graph_preserves_branch_isolation(self) -> None:
workflow_yaml = _yaml(
"""
- title: Entry lookup
workflow_definition:
blocks:
- block_type: code
- label: extract_entry_output
+ label: start_search
code: |
- return {"output": {"summary": "found"}}
+ return {"ok": True}
+ - block_type: task
+ label: choose_path
+ branch_conditions:
+ - blocks:
+ - block_type: code
+ label: branch_a
+ code: |
+ return start_search_output
+ - blocks:
+ - block_type: code
+ label: branch_b
+ code: |
+ return start_search_output
+ - block_type: code
+ label: after_choice
+ code: |
+ return choose_path_output
"""
)
- metadata = [
- {
- "block_label": "extract_entry_output",
- "claimed_outcomes": [
- {
- "goal_value_paths": ["output"],
- "extraction_schema": '{"type":"object","properties":{"output":{"type":"object"}}}',
- }
- ],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
- }
- ]
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+ available_by_label = code_block_available_binding_keys_by_label(workflow_yaml)
- assert result is not None
- assert result["ok"] is False
- assert result["data"]["missing_goal_value_paths"] == ["output.record_id"]
- assert result["data"]["missing_extraction_schema_paths"] == ["output.record_id"]
- assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
+ assert available_by_label["branch_a"] == ["start_search_output"]
+ assert available_by_label["branch_b"] == ["start_search_output"]
+ assert "branch_a_output" not in available_by_label["branch_b"]
+ assert available_by_label["after_choice"] == ["choose_path_output", "start_search_output"]
- def _run_preflight_metadata_reject_ctx(self) -> CopilotContext:
+ @pytest.mark.asyncio
+ async def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="validate_start",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.record_id"],
- required_extraction_schema_paths=["output.record_id"],
- required_code_return_paths=["output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
- return ctx
-
- _RUN_PREFLIGHT_CLICK_YAML = _yaml(
- """
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: validate_start
- code: |
- await page.locator("#start").click()
- """
- )
-
- def test_run_preflight_metadata_reject_arms_directive_and_emits_once(self) -> None:
- ctx = self._run_preflight_metadata_reject_ctx()
- with capture_logs() as logs:
- result = workflow_update_module._metadata_contract_run_preflight_reject(
- ctx, self._RUN_PREFLIGHT_CLICK_YAML, []
- )
- assert result is not None and result["ok"] is False
- directive = result["data"]["metadata_convergence_directive"]
- assert directive["rung"] == 1
- assert "declared_goal" in directive["missing_fields_by_label"]["validate_start"]
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
-
- def test_run_preflight_metadata_reject_repeat_does_not_reissue_emit(self) -> None:
- ctx = self._run_preflight_metadata_reject_ctx()
- with capture_logs() as first_logs:
- first = workflow_update_module._metadata_contract_run_preflight_reject(
- ctx, self._RUN_PREFLIGHT_CLICK_YAML, []
- )
- ctx.output_contract_bail_actuated_this_call = False
- with capture_logs() as second_logs:
- workflow_update_module._metadata_contract_run_preflight_reject(ctx, self._RUN_PREFLIGHT_CLICK_YAML, [])
- assert first is not None and first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert len([log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]) == 1
- assert [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT] == []
-
- def test_run_preflight_root_only_value_required_emits_without_payload_rewrite(self) -> None:
- ctx = _live_root_value_bearing_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
+ submitted = _code_yaml(
+ "await page.locator('#query').fill(str(provider_query))",
+ parameter_keys=["provider_query"],
+ workflow_param=True,
)
- assert evaluation is not None
- assert evaluation.required_paths == {"output"}
- assert evaluation.payload["canonical_required_child_paths"] == ["output"]
- assert evaluation.payload["declaration_only_child_paths"] == ["output.blocker"]
- assert evaluation.payload["shape_violations"] == ["value_bearing_output_required"]
- assert evaluation.payload["satisfying_templates"]
- assert evaluation.payload["satisfying_templates"]["return_skeleton"] == (
- 'return {"output": {"blocker": None, **output_value}}'
- )
- assert evaluation.payload["satisfying_templates"]["extraction_schema"] == {
- "type": "object",
- "properties": {
- "output": {
- "type": "object",
- "properties": {"blocker": {"type": ["string", "null"]}},
- "required": ["blocker"],
- }
- },
- "required": ["output"],
- }
- metadata_template = evaluation.payload["satisfying_templates"]["code_artifact_metadata"]
- assert metadata_template["claimed_outcomes"][0]["goal_value_paths"] == ["output"]
- assert metadata_template["terminal_verifier_expectations"][0]["goal_value_paths"] == ["output"]
- assert evaluation.canonical_signature == workflow_update_module._output_contract_signature(
- ctx=ctx, required_paths={"output"}
- )
- with capture_logs() as logs:
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
- assert result is not None and result["ok"] is False
- assert "metadata_convergence_directive" not in result["data"]
- assert result["data"]["value_bearing_convergence_directive"] == {
- "rung": 1,
- "required_output_paths": ["output"],
- "shape_violations": ["value_bearing_output_required"],
- "satisfying_templates": result["data"]["satisfying_templates"],
- }
- assert result["data"]["canonical_output_contract_signature"] == evaluation.canonical_signature
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert emits[0]["directive_family"] == "value_bearing_output_required"
- assert emits[0]["canonical_required_child_paths"] == ["output"]
-
- def test_cross_seam_write_arm_then_metadata_less_preflight_escalates(self) -> None:
- ctx = self._run_preflight_metadata_reject_ctx()
- first = workflow_update_module._metadata_contract_run_preflight_reject(ctx, self._RUN_PREFLIGHT_CLICK_YAML, [])
- assert first is not None and first["data"]["metadata_convergence_directive"]["rung"] == 1
- ctx.output_contract_bail_actuated_this_call = False
- directive = workflow_update_module._adjudicate_metadata_reject_ladder(
- ctx,
- workflow_yaml=self._RUN_PREFLIGHT_CLICK_YAML,
- raw_metadata=[],
- missing_labels=["validate_start"],
- required_paths={"output.record_id"},
- )
- assert directive is not None
- assert directive["rung"] == 2
- assert "metadata_fill_in_skeleton" in directive
-
- def test_metadata_less_preflight_never_grants_advisory_run(self) -> None:
- ctx = self._run_preflight_metadata_reject_ctx()
- directive = workflow_update_module._adjudicate_metadata_reject_ladder(
- ctx,
- workflow_yaml=self._RUN_PREFLIGHT_CLICK_YAML,
- raw_metadata=[],
- missing_labels=["validate_start"],
- required_paths={"output.record_id"},
- )
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths={"output.record_id"})
- assert directive is not None and directive["rung"] == 1
- assert workflow_update_module._output_contract_advisory_granted(ctx, signature) is False
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- def test_metadata_contract_scaffold_uses_recorded_paths_before_request_policy(self) -> None:
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+
+ @pytest.mark.asyncio
+ async def test_new_declared_workflow_parameter_key_is_accepted_on_later_edit(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.requested_value",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:typed-output",
- missing_requested_output_facts=[
- {"output_path": "output.recorded_value", "output_root": "output", "value_status": "no_typed_value"},
- ],
+ ctx.workflow_yaml = _code_yaml(
+ "await page.locator('#query').fill(str(provider_query))",
+ parameter_keys=["provider_query"],
+ workflow_param=True,
)
- workflow_yaml = _yaml(
+ submitted = _yaml(
"""
- title: Entry lookup
+ title: Registry lookup
workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample Search}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: Example City}
blocks:
- block_type: code
- label: extract_entry_output
+ label: search_registry
+ parameter_keys: [provider_query, search_location]
code: |
- return {"output": {"recorded_value": "ABC123"}}
+ await page.locator("#query").fill(str(provider_query))
+ await page.locator("#location").fill(str(search_location))
"""
)
- required_paths, source, reason_code = workflow_update_module._required_child_output_paths_for_authoring(ctx)
- scaffolded = workflow_update_module._apply_metadata_contract_scaffold(
- ctx,
- workflow_yaml,
- [],
- required_paths=required_paths,
- source=source,
- reason_code=reason_code,
- )
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert scaffolded[0]["artifact_id"] == "code_artifact:extract_entry_output"
- assert scaffolded[0]["block_label"] == "extract_entry_output"
- assert scaffolded[0]["claimed_outcomes"][0]["goal_value_paths"] == ["output.recorded_value"]
- schema = json.loads(scaffolded[0]["claimed_outcomes"][0]["extraction_schema"])
- assert schema["properties"]["output"]["properties"]["recorded_value"] == {}
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
- def test_independent_judgment_output_is_not_required_as_code_return_path(self) -> None:
- judgment_criterion = CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- judgment_truth_condition=JudgmentTruthCondition(
- predicate="login_gate_blocks_target",
- polarity_when_holds=True,
- ),
+ @pytest.mark.asyncio
+ async def test_prior_block_output_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _directory_blocks_yaml(
+ """
+ - block_type: code
+ label: search_registry
+ code: |
+ return {"records": []}
+ - block_type: code
+ label: summarize_registry
+ parameter_keys: [search_registry_output]
+ code: |
+ print(search_registry_output)
+ """
)
- judgment_only_ctx = _code_only_ctx()
- judgment_only_ctx.request_policy = RequestPolicy(completion_criteria=[judgment_criterion])
- judgment_only_paths = workflow_update_module._output_contract_required_paths_source(judgment_only_ctx).union
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert judgment_only_paths == set()
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+ @pytest.mark.parametrize("marker", ["<<<<<<< HEAD", "======="])
+ @pytest.mark.asyncio
+ async def test_raw_workflow_yaml_conflict_marker_rejects_before_persist(
+ self, monkeypatch: pytest.MonkeyPatch, marker: str
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- judgment_criterion,
- CompletionCriterion(
- id="record_id",
- outcome="the record id is returned",
- output_path="output.record_id",
- ),
- ]
- )
-
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+ ctx.workflow_yaml = _SAFE_CODE_YAML
+ submitted = f"{marker}\n{_SAFE_CODE_YAML}"
- assert required_paths == {"output.record_id"}
- assert source == "requested_output_contract"
- assert reason_code == "requested_output_contract_missing_output_coverage"
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- def test_independent_judgment_shape_output_is_not_required_as_code_return_path(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- )
- ]
- )
+ assert result["ok"] is False
+ assert f"conflict marker `{marker}`" in result["error"]
+ assert "line 1" in result["error"]
+ assert ctx.workflow_yaml == _SAFE_CODE_YAML
- required_paths = workflow_update_module._output_contract_required_paths_source(ctx).union
- assert required_paths == set()
+class TestCompiledAuthoringImposition:
+ def _provider_search_ctx(self) -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#provInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 13,
+ "role": "textbox",
+ "accessible_name": "Provider Name",
+ "trajectory_index": 0,
+ }
+ ]
+ return ctx
- def test_independent_judgment_repair_context_is_not_rehydrated_as_code_return_path(self) -> None:
+ def _typed_default_ctx(self) -> CopilotContext:
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- judgment_truth_condition=JudgmentTruthCondition(
- predicate="login_gate_blocks_target",
- polarity_when_holds=True,
- ),
- )
- ]
- )
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="judge_login_gate_blocks_target",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.login_gate_blocks_target"],
- required_extraction_schema_paths=["output.login_gate_blocks_target"],
- required_code_return_paths=["output.login_gate_blocks_target"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#search",
+ "source_url": "https://example.com/catalog",
+ "typed_length": 15,
+ "typed_value": "example_sku_123",
+ "role": "textbox",
+ "accessible_name": "Search",
+ "trajectory_index": 0,
+ }
+ ]
+ return ctx
- required_paths = workflow_update_module._output_contract_required_paths_source(ctx).union
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, _SAFE_CODE_YAML, [])
+ @staticmethod
+ def _cafe_search_capture() -> dict[str, object]:
+ return {
+ "tool_name": "type_text",
+ "selector": "#café-search",
+ "source_url": "https://example.com/catalog",
+ "typed_length": 15,
+ "role": "textbox",
+ "accessible_name": "Catalog search",
+ "trajectory_index": 0,
+ }
- assert required_paths == set()
- assert result is None
+ @pytest.mark.asyncio
+ async def test_imposes_strict_scout_selector_and_lifts_singleton_literal(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
- def test_mixed_repair_context_keeps_non_judgment_code_return_path(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- judgment_truth_condition=JudgmentTruthCondition(
- predicate="login_gate_blocks_target",
- polarity_when_holds=True,
- ),
- )
- ]
- )
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="extract_entry_output",
- reason_code="metadata_reject",
- required_goal_value_paths=["output.login_gate_blocks_target", "output.record_id"],
- required_extraction_schema_paths=["output.login_gate_blocks_target", "output.record_id"],
- required_code_return_paths=["output.login_gate_blocks_target", "output.record_id"],
- metadata_contract_source="requested_output_contract",
- metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
- )
-
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
- assert required_paths == {"output.record_id"}
- assert source == "requested_output_contract"
- assert reason_code == "requested_output_contract_missing_output_coverage"
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+ assert "input[placeholder='Search']" not in block["code"]
+ assert block["parameter_keys"] == ["provider_name"]
+ parameters = parsed["workflow_definition"]["parameters"]
+ assert parameters == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Sample Search",
+ }
+ ]
+ assert result["data"]["imposed_substitutions"] == {
+ "block_label": "search_registry",
+ "source_trajectory_count": 1,
+ "parameter_keys": ["provider_name"],
+ "credential_parameter_keys": [],
+ "selector_provenance": [
+ {
+ "trajectory_index": 0,
+ "selector": "#provInput",
+ "emitted_literal": "#provInput",
+ "source": "selector",
+ }
+ ],
+ "prior_source": "workflow_yaml",
+ }
- def test_runtime_output_judgment_is_excluded_from_scalar_bind_set(self) -> None:
+ @pytest.mark.asyncio
+ async def test_enter_directory_location_output_block_metadata_passes_before_persist(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="runtime_output",
- ),
- CompletionCriterion(
- id="record_id",
- outcome="the record id is returned",
- output_path="output.record_id",
- ),
- ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: enter_directory_location
+ prompt: Enter the directory location and output the selected address.
+ code: |
+ await page.locator("#locInput").fill("North Carolina, USA")
+ address = "North Carolina, USA"
+ """
)
+ metadata = [
+ {
+ "block_label": "enter_directory_location",
+ "declared_goal": "Enter the directory location and return the address.",
+ "claimed_outcomes": [
+ {
+ "id": "claim:address",
+ "status": "observed_not_verified",
+ "goal_value_paths": ["address"],
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "id": "expectation:address",
+ "goal_value_paths": ["address"],
+ }
+ ],
+ }
+ ]
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
- assert contract.observation_paths == {"output.record_id"}
- assert contract.union == {"output.record_id"}
- assert "output.login_gate_blocks_target" not in contract.declaration_paths
+ assert result["ok"] is True
+ assert ctx.workflow_yaml
+ assert "enter_directory_location" in ctx.code_artifact_metadata
+ assert ctx.latest_recorded_build_test_outcome is None
- def test_runtime_output_judgment_alone_yields_empty_bind_set(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="runtime_output",
- )
- ]
+ @pytest.mark.asyncio
+ async def test_imposition_preserves_submitted_extraction_suffix(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = synthesized.code.rstrip() + '\nrecords = [{"number": "REC-001", "status": "credentialed"}]\n'
+ submitted = yaml.safe_dump(
+ {
+ "title": "Provider lookup",
+ "workflow_definition": {
+ "parameters": [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Sample Search",
+ }
+ ],
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "search_registry",
+ "code": submitted_code,
+ }
+ ],
+ },
+ },
+ sort_keys=False,
)
+ metadata = [_terminal_metadata("search_registry", "search the registry")]
- assert workflow_update_module._output_contract_required_paths_source(ctx).union == set()
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
- def test_non_judgment_scalar_bind_set_is_unchanged(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="record_id",
- outcome="the record id is returned",
- output_path="output.record_id",
- )
- ]
- )
+ assert result["ok"] is True
+ assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ output_block = _code_blocks(parsed)["search_registry"]
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ output_code = str(output_block["code"])
+ assert "<fill" not in browser_code
+ assert "<fill: captured value>" not in browser_code
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in browser_code
+ assert 'records = [{"number": "REC-001", "status": "credentialed"}]' in output_code
+ assert 'return {"records": records}' in output_code
- assert workflow_update_module._output_contract_required_paths_source(ctx).observation_paths == {
- "output.record_id"
- }
+ def _download_ctx(self) -> CopilotContext:
+ from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
- def test_judgment_output_paths_widens_independent_predicate(self) -> None:
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="independent_gate",
- outcome="the independent path is blocked by a login gate",
- output_path="output.independent_gate_blocks_target",
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- ),
- CompletionCriterion(
- id="runtime_gate",
- outcome="the runtime path is blocked by a login gate",
- output_path="output.runtime_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="runtime_output",
- ),
- ]
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#statement-row",
+ "source_url": "https://example.com/billing",
+ "trajectory_index": 0,
+ }
+ ]
+ ctx.reached_download_target = ReachedDownloadTarget(
+ selector='a[href="/billing/statement.pdf"]',
+ affordance_text="View Printable Statement",
+ download_kind="attribute",
+ source_step="trajectory_recency",
+ already_registered=False,
)
+ return ctx
- all_judgment = workflow_update_module._judgment_output_paths(ctx)
+ @pytest.mark.asyncio
+ async def test_imposition_forwards_reached_target_and_emits_download_terminal(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._download_ctx()
+ submitted = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_statement
+ code: |
+ await page.locator("#statement-row").click()
+ """
+ )
- assert all_judgment == {
- "output.independent_gate_blocks_target",
- "output.runtime_gate_blocks_target",
- }
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- def test_independent_judgment_runtime_repair_fact_is_not_rehydrated_as_code_return_path(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- )
- ]
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert "async with page.expect_download()" in block["code"]
+ assert "/billing/statement.pdf" in block["code"]
+ assert "save_as" not in block["code"]
+ assert result["data"]["imposed_substitutions"]["block_label"] == "download_statement"
+
+ @pytest.mark.asyncio
+ async def test_imposed_download_terminal_clears_binding_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._download_ctx()
+ ctx.flow_evidence = [
+ {
+ "evidence": {"source_tool": "scout_interaction", "interaction_selector": "#statement-row"},
+ "reached_via": "interaction",
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_statement
+ code: |
+ await page.locator("#statement-row").click()
+ """
)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "judge_login_gate_blocks_target",
- "output_path": "output.login_gate_blocks_target",
- "output_root": "output",
- "criterion_id": "__copilot_requested_output__output_login_gate_blocks_target",
- "reason_code": "structurally_abstained",
- "grounding_mode": "judgment_boolean",
- "value_status": "structural_abstained",
- }
- ],
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))
+ assert "async with page.expect_download()" in block["code"]
+
+ @pytest.mark.asyncio
+ async def test_reached_download_target_still_imposes_after_prior_update_attempt(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._download_ctx()
+ ctx.update_workflow_called = True
+ submitted = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_statement
+ code: |
+ await page.locator("#statement-row").click()
+ """
)
- required_paths = workflow_update_module._output_contract_required_paths_source(ctx).union
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert required_paths == set()
+ assert result["ok"] is True
+ block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))
+ assert "async with page.expect_download()" in block["code"]
- def test_mixed_runtime_repair_facts_keep_non_judgment_code_return_path(self) -> None:
+ @pytest.mark.asyncio
+ async def test_imposition_carries_reached_download_target_to_synthesized_code(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="login_gate",
- outcome="the target path is blocked by a login gate",
- output_path="output.login_gate_blocks_target",
- expected_output_value=True,
- expected_output_shape="goal_judgment_boolean",
- requested_output_evidence_source="independent_run_evidence",
- ),
- CompletionCriterion(
- id="record_id",
- outcome="the record id is returned",
- output_path="output.record_id",
- ),
- ]
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "div.stmt-row",
+ "source_url": "https://example.com/bills",
+ "trajectory_index": 0,
+ }
+ ]
+ ctx.reached_download_target = ReachedDownloadTarget(
+ selector='[href="/files/report.pdf"]',
+ affordance_text="Download PDF",
+ download_kind="extension",
+ source_step="trajectory_recency",
+ already_registered=False,
)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.login_gate_blocks_target",
- "output_root": "output",
- "criterion_id": "__copilot_requested_output__output_login_gate_blocks_target",
- "reason_code": "structurally_abstained",
- "grounding_mode": "judgment_boolean",
- "value_status": "structural_abstained",
- },
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.record_id",
- "output_root": "output",
- "criterion_id": "record_id",
- "reason_code": "structurally_abstained",
- "grounding_mode": "missing",
- "value_status": "structural_abstained",
- },
- ],
+ workflow_yaml = _yaml(
+ """
+ title: Download report
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_report
+ code: |
+ await page.goto("https://example.com/bills")
+ await page.locator("div.stmt-row").click()
+ """
)
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
- assert required_paths == {"output.record_id"}
- assert source == "runtime_output_repair"
- assert reason_code == "runtime_output_repair_required"
+ assert result.violations == []
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert "expect_download" in block["code"]
+ assert 'await page.locator("[href=\\"/files/report.pdf\\"]").click()' in block["code"]
- def test_runtime_output_facts_record_same_run_null_without_evidence_text_backfill(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {
- "label": "extract_entry_output",
- "status": "completed",
- "extracted_data": {"output": {"npi": None}, "evidence_text": "Value 1234567890"},
- }
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["requested_npi"],
- verdicts=[
- CriterionVerdict(
- criterion_id="requested_npi",
- state="unsatisfied",
- reason_code="evidence_contradicts",
- output_path="output.npi",
- grounding_mode="exact_value",
- expected_output_shape="string",
- has_exact_value=False,
- )
- ],
+ def test_imposition_targets_synthesized_label_in_multi_code_workflow(self) -> None:
+ ctx = self._download_ctx()
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: summarize_statement
+ code: |
+ return {"status": "ready"}
+ - block_type: code
+ label: scout_synthesized_browser_steps
+ code: |
+ await page.locator("#statement-row").click()
+ """
)
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- registered_output_parameter_payloads=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_parameter_key": "npi",
- "value": None,
- }
- ],
- )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
- assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert outcome.runtime_output_repair_facts == [
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.npi",
- "output_root": "output",
- "criterion_id": "requested_npi",
- "reason_code": "evidence_contradicts",
- "grounding_mode": "exact_value",
- "expected_output_shape": "string",
- "value_status": "null",
- "evidence_refs": ["registered_output:extract_entry_output:npi", "output:extract_entry_output"],
- }
- ]
- assert outcome.is_authoritative is True
-
- def test_runtime_output_facts_record_empty_typed_output_paths_for_next_contract(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {
- "label": "summarize_access_output",
- "status": "completed",
- "extracted_data": {
- "extracted_information": [],
- "summarize_access_output": {},
- "evidence_text": "diagnostic page text",
- },
- }
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=[
- "requested_form_exists",
- "requested_path_label",
- "requested_next_action",
- ],
- verdicts=[
- CriterionVerdict(
- criterion_id="requested_form_exists",
- state="unsatisfied",
- reason_code="structurally_abstained",
- output_path="output.public_form_exists",
- grounding_mode="missing",
- expected_output_shape="boolean",
- ),
- CriterionVerdict(
- criterion_id="requested_path_label",
- state="unsatisfied",
- reason_code="missing_exact_field",
- output_path="output.visible_page_path_label",
- grounding_mode="missing",
- expected_output_shape="string",
- ),
- CriterionVerdict(
- criterion_id="requested_next_action",
- state="unsatisfied",
- reason_code="missing_exact_field",
- output_path="output.recommended_next_action",
- grounding_mode="missing",
- expected_output_shape="string",
- ),
- ],
- )
-
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- )
- assert outcome is not None
- assert {fact["output_path"]: fact["value_status"] for fact in outcome.runtime_output_repair_facts} == {
- "output.public_form_exists": "structural_abstained",
- "output.recommended_next_action": "no_typed_value",
- "output.visible_page_path_label": "no_typed_value",
- }
- assert all(
- "evidence_text" not in ref
- for fact in outcome.runtime_output_repair_facts
- for ref in fact.get("evidence_refs", [])
- )
-
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = outcome
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
- assert required_paths == {
- "output.public_form_exists",
- "output.recommended_next_action",
- "output.visible_page_path_label",
- }
- assert source == "runtime_output_repair"
- assert reason_code == "runtime_output_repair_required"
-
- def test_runtime_output_facts_preserve_satisfied_output_owner_label(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {
- "label": "download_statement",
- "status": "completed",
- "extracted_data": {"output": {"statement_pdf": "statement.pdf"}},
- }
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["__copilot_fallback_floor__run", "requested_statement_pdf"],
- verdicts=[
- CriterionVerdict(
- criterion_id="__copilot_fallback_floor__run",
- state="unsatisfied",
- reason_code="no_evidence",
- ),
- CriterionVerdict(
- criterion_id="requested_statement_pdf",
- state="satisfied",
- reason_code="evidence_confirms",
- output_path="output.statement_pdf",
- grounding_mode="exact_value",
- expected_output_shape="string",
- has_exact_value=True,
- evidence_ref="block_outputs:download_statement.output.statement_pdf",
- ),
- ],
- )
-
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- )
-
- assert outcome is not None
- assert outcome.reason_code == "outcome_not_demonstrated"
- assert outcome.runtime_output_repair_facts == [
- {
- "workflow_run_id": "wr_current",
- "block_label": "download_statement",
- "owner_labels": ["download_statement"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "grounding_mode": "exact_value",
- "expected_output_shape": "string",
- "value_status": "satisfied",
- "evidence_refs": ["output:download_statement"],
- }
- ]
-
- def test_runtime_output_facts_preserve_flat_registered_output_owner_label(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {"label": "apex_portal_login", "status": "completed", "extracted_data": {}},
- {"label": "apex_open_monthly_statement", "status": "completed", "extracted_data": {}},
- {"label": "apex_download_invoice_pdf", "status": "completed", "extracted_data": {}},
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["__copilot_authored_output__output_file_name"],
- verdicts=[
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_file_name",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.file_name",
- grounding_mode="missing",
- requested_output_evidence_source="runtime_output",
- )
- ],
- )
+ assert result.violations == []
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert blocks["summarize_statement"]["code"].strip() == 'return {"status": "ready"}'
+ assert "async with page.expect_download()" in blocks["scout_synthesized_browser_steps"]["code"]
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- registered_output_parameter_payloads=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "apex_download_invoice_pdf",
- "output_parameter_key": "apex_download_invoice_pdf_output",
- "value": {
- "file_name": "statement.pdf",
- "downloaded_files": [{"filename": "statement.pdf"}],
- },
- }
- ],
+ def test_imposition_targets_recorded_outcome_owner_in_multi_code_workflow(self) -> None:
+ ctx = self._download_ctx()
+ ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key="recorded-download",
+ phase="persisted_block_run",
+ reason_code="outcome_not_demonstrated",
+ frontier_facet="value_shape",
+ owning_block_labels=["download_matching_invoice_pdf"],
)
-
- assert outcome is not None
- assert outcome.runtime_output_repair_facts == [
- {
- "workflow_run_id": "wr_current",
- "block_label": "apex_download_invoice_pdf",
- "output_path": "output.file_name",
- "output_root": "output",
- "criterion_id": "__copilot_authored_output__output_file_name",
- "reason_code": "no_evidence",
- "grounding_mode": "missing",
- "value_status": "no_typed_value",
- "evidence_refs": ["registered_output:apex_download_invoice_pdf:apex_download_invoice_pdf_output"],
- }
- ]
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = outcome
workflow_yaml = _yaml(
"""
title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: apex_portal_login
+ label: login_to_apex_business
code: |
return {"logged_in": True}
- block_type: code
- label: apex_open_monthly_statement
- code: |
- return {"matched": True}
- - block_type: code
- label: apex_download_invoice_pdf
+ label: download_matching_invoice_pdf
code: |
- return {"file_name": "statement.pdf"}
+ async with page.expect_download() as download_info:
+ await page.locator("a[href='/billing/statement.pdf']").click()
+ download = await download_info.value
+ return {"downloaded_files": [download.suggested_filename]}
"""
)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
- assert evaluation is not None
- assert evaluation.block_label == "apex_download_invoice_pdf"
- assert evaluation.payload["output_owner_labels"] == ["apex_download_invoice_pdf"]
- assert "missing_output_owner" not in evaluation.shape_violations
+ assert result.violations == []
+ assert result.substitutions is not None
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert blocks["login_to_apex_business"]["code"].strip() == 'return {"logged_in": True}'
+ assert "async with page.expect_download()" in blocks["download_matching_invoice_pdf"]["code"]
- def test_runtime_output_facts_do_not_infer_registered_owner_from_parameter_key(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {"label": "download_invoice", "status": "completed", "extracted_data": {}},
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["__copilot_authored_output__output_file_name"],
- verdicts=[
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_file_name",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.file_name",
- grounding_mode="missing",
- requested_output_evidence_source="runtime_output",
- )
- ],
- )
-
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- registered_output_parameter_payloads=[
- {
- "workflow_run_id": "wr_current",
- "output_parameter_key": "download_invoice_output",
- "value": {"file_name": "statement.pdf"},
- }
- ],
+ def test_imposition_ignores_unchanged_persisted_browser_sibling(self) -> None:
+ ctx = self._download_ctx()
+ prior_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login_to_apex_business
+ code: |
+ await page.locator("#email").fill(str(login_credentials["username"]))
+ await page.locator("#password").fill(str(login_credentials["password"]))
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: download_matching_invoice_pdf
+ code: |
+ return {"status": "ready"}
+ """
)
-
- assert outcome is not None
- assert outcome.runtime_output_repair_facts == [
- {
- "workflow_run_id": "wr_current",
- "output_path": "output.file_name",
- "output_root": "output",
- "criterion_id": "__copilot_authored_output__output_file_name",
- "reason_code": "no_evidence",
- "grounding_mode": "missing",
- "value_status": "no_typed_value",
- "evidence_refs": ["registered_output:unknown:download_invoice_output"],
- }
- ]
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = outcome
- workflow_yaml = _yaml(
+ submitted_yaml = _yaml(
"""
title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: download_invoice
+ label: login_to_apex_business
code: |
- return {"file_name": "statement.pdf"}
+ await page.locator("#email").fill(str(login_credentials["username"]))
+ await page.locator("#password").fill(str(login_credentials["password"]))
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: download_matching_invoice_pdf
+ code: |
+ await page.locator("#statement-row").click()
"""
)
+ ctx.workflow_yaml = prior_yaml
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == []
- assert "missing_output_owner" in evaluation.shape_violations
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted_yaml, ctx)
- def test_runtime_output_owner_ignores_unsatisfied_independent_self_emitted_fields(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [
- {
- "label": "login_to_service",
- "status": "completed",
- "extracted_data": {"output": {"logged_in": True}},
- },
- {
- "label": "open_statement",
- "status": "completed",
- "extracted_data": {
- "output": {
- "matched": True,
- "statement_date": "2026-05",
- "visible_page_label": "Statement details",
- }
- },
- },
- ],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=[
- "__copilot_authored_output__output_logged_in",
- "__copilot_authored_output__output_matched",
- "__copilot_authored_output__output_statement_date",
- "__copilot_authored_output__output_visible_page_label",
- "__copilot_authored_output__output_downloaded",
- ],
- verdicts=[
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_logged_in",
- state="unsatisfied",
- reason_code="structurally_abstained",
- output_path="output.logged_in",
- grounding_mode="missing",
- requested_output_evidence_source="independent_run_evidence",
- evidence_ref="block_outputs:login_to_service.output.logged_in",
- ),
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_matched",
- state="unsatisfied",
- reason_code="structurally_abstained",
- output_path="output.matched",
- grounding_mode="missing",
- requested_output_evidence_source="independent_run_evidence",
- evidence_ref="block_outputs:open_statement.output.matched",
- ),
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_statement_date",
- state="unsatisfied",
- reason_code="structurally_abstained",
- output_path="output.statement_date",
- grounding_mode="missing",
- requested_output_evidence_source="runtime_output",
- evidence_ref="block_outputs:open_statement.output.statement_date",
- ),
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_visible_page_label",
- state="unsatisfied",
- reason_code="structurally_abstained",
- output_path="output.visible_page_label",
- grounding_mode="missing",
- requested_output_evidence_source="runtime_output",
- evidence_ref="block_outputs:open_statement.output.visible_page_label",
- ),
- CriterionVerdict(
- criterion_id="__copilot_authored_output__output_downloaded",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.downloaded",
- grounding_mode="missing",
- requested_output_evidence_source="independent_run_evidence",
- ),
- ],
- )
+ assert result.violations == []
+ assert result.substitutions is not None
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert "#sign-in" in str(blocks["login_to_apex_business"]["code"])
+ assert "async with page.expect_download()" in str(blocks["download_matching_invoice_pdf"]["code"])
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
+ def test_prior_synthesized_label_prevents_unrelated_multi_code_imposition(self) -> None:
+ ctx = self._download_ctx()
+ prior_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: summarize_statement
+ code: |
+ return {"status": "ready"}
+ - block_type: code
+ label: scout_synthesized_browser_steps
+ code: |
+ await page.locator("#statement-row").click()
+ """
)
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = outcome
- workflow_yaml = _yaml(
+ submitted_yaml = _yaml(
"""
- title: Statement workflow
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: login_to_service
+ label: summarize_statement
code: |
- return {"output": {"logged_in": True}}
+ return {"status": "edited"}
- block_type: code
- label: open_statement
+ label: scout_synthesized_browser_steps
code: |
- return {"output": {"statement_date": "2026-05"}}
+ await page.locator("#statement-row").click()
"""
)
+ ctx.workflow_yaml = prior_yaml
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert outcome is not None
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == []
- assert "missing_output_owner" in evaluation.shape_violations
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted_yaml, ctx)
- def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["requested_npi"],
- verdicts=[
- CriterionVerdict(
- criterion_id="requested_npi",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.npi",
- grounding_mode="missing",
- )
- ],
+ assert result.violations == []
+ assert result.substitutions is None
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert blocks["summarize_statement"]["code"].strip() == 'return {"status": "edited"}'
+ assert blocks["scout_synthesized_browser_steps"]["code"].strip() == (
+ 'await page.locator("#statement-row").click()'
)
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- registered_output_parameter_payloads=[
- {
- "workflow_run_id": "wr_previous",
- "block_label": "extract_entry_output",
- "output_parameter_key": "npi",
- "value": None,
- }
- ],
- )
-
- assert outcome is not None
- assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
- assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
- "evidence_refs", []
- )
-
- def test_runtime_output_facts_ignore_unscoped_registered_values(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["requested_npi"],
- verdicts=[
- CriterionVerdict(
- criterion_id="requested_npi",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.npi",
- grounding_mode="missing",
- )
- ],
- )
-
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- registered_output_parameter_payloads=[
- {
- "block_label": "extract_entry_output",
- "output_parameter_key": "npi",
- "value": None,
- }
- ],
- )
-
- assert outcome is not None
- assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
- assert "block_label" not in outcome.runtime_output_repair_facts[0]
- assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
- "evidence_refs", []
- )
-
- def test_runtime_output_facts_ignore_unscoped_fallback_registered_values(self) -> None:
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_current",
- "registered_output_parameter_values": [
- {
- "block_label": "extract_entry_output",
- "output_parameter_key": "npi",
- "value": None,
- }
- ],
- "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
- },
- }
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["requested_npi"],
- verdicts=[
- CriterionVerdict(
- criterion_id="requested_npi",
- state="unsatisfied",
- reason_code="no_evidence",
- output_path="output.npi",
- grounding_mode="missing",
- )
- ],
- )
-
- outcome = recorded_outcome_from_run_blocks_result(
- result,
- recorded_run_outcome=RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- ),
- completion_verification=verification,
- )
-
- assert outcome is not None
- assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
- assert "block_label" not in outcome.runtime_output_repair_facts[0]
- assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
- "evidence_refs", []
- )
-
- def test_runtime_output_facts_override_request_policy_contract_paths(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.requested_value",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.npi",
- "output_root": "output",
- "criterion_id": "requested_npi",
- "reason_code": "evidence_contradicts",
- "value_status": "null",
- },
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.locations[].address",
- "output_root": "output",
- "criterion_id": "requested_location",
- "reason_code": "structurally_abstained",
- "grounding_mode": "missing",
- "value_status": "structural_abstained",
- },
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"requested_value": "wrong"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.source == "runtime_output_repair"
- assert evaluation.reason_code == "runtime_output_repair_required"
- assert evaluation.required_paths == {"output.locations[].address", "output.npi"}
- assert sorted(
- evaluation.payload["runtime_output_repair_facts"],
- key=lambda item: str(item.get("output_path") or ""),
- ) == sorted(
- ctx.latest_recorded_build_test_outcome.runtime_output_repair_facts,
- key=lambda item: str(item.get("output_path") or ""),
- )
-
- def test_runtime_output_owner_selects_current_block_without_metadata(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["download_statement"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: login
- code: |
- return {"logged_in": True}
- - block_type: code
- label: download_statement
- code: |
- return {"output": {"statement_pdf": "statement.pdf"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == "download_statement"
- assert evaluation.payload["output_owner_labels"] == ["download_statement"]
- assert "missing_output_owner" not in evaluation.shape_violations
-
- def test_runtime_output_zero_owner_does_not_fall_through_to_single_block_default(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: only_block
- code: |
- return {"output": {"statement_pdf": "statement.pdf"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == []
- assert "missing_output_owner" in evaluation.shape_violations
-
- def test_runtime_output_multi_owner_rejects_without_picking_metadata_owner(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["download_a", "download_b"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_a
- code: |
- return {"output": {"statement_pdf": "a.pdf"}}
- - block_type: code
- label: download_b
- code: |
- return {"output": {"statement_pdf": "b.pdf"}}
- """
- )
- metadata = [{"block_label": "download_a", "claimed_outcomes": [{"goal_value_paths": ["output.statement_pdf"]}]}]
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata)
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == ["download_a", "download_b"]
- assert "ambiguous_output_owner" in evaluation.shape_violations
-
- def test_runtime_output_stale_multi_owner_rejects_when_one_owner_current(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["current_download", "stale_download"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: current_download
- code: |
- return {"output": {"statement_pdf": "a.pdf"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == ["current_download"]
- assert "ambiguous_output_owner" in evaluation.shape_violations
-
- def test_runtime_output_multi_owner_all_stale_rejects_as_missing_owner(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["stale_one", "stale_two"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: current_download
- code: |
- return {"output": {"statement_pdf": "a.pdf"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == []
- assert "missing_output_owner" in evaluation.shape_violations
-
- def test_runtime_output_paths_with_disagreeing_single_owners_reject(self) -> None:
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["download_pdf"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- },
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["extract_total"],
- "output_path": "output.statement_total",
- "output_root": "output",
- "criterion_id": "requested_statement_total",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- },
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_pdf
- code: |
- return {"output": {"statement_pdf": "a.pdf"}}
- - block_type: code
- label: extract_total
- code: |
- return {"output": {"statement_total": "12.00"}}
- """
- )
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == ""
- assert evaluation.payload["output_owner_labels"] == ["download_pdf", "extract_total"]
- assert "ambiguous_output_owner" in evaluation.shape_violations
-
- def test_runtime_output_owner_overrides_stale_pin(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "runtime-owner-pin"
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "owner_labels": ["new_owner"],
- "output_path": "output.statement_pdf",
- "output_root": "output",
- "criterion_id": "requested_statement_pdf",
- "reason_code": "evidence_confirms",
- "value_status": "satisfied",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: stale_owner
- code: |
- return {"output": {"statement_pdf": "old.pdf"}}
- - block_type: code
- label: new_owner
- code: |
- return {"output": {"statement_pdf": "new.pdf"}}
- """
- )
- pin_key = workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, {"output.statement_pdf"})
- ctx.output_contract_pinned_block_label_by_signature = {pin_key: "stale_owner"}
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.block_label == "new_owner"
- assert evaluation.payload["output_owner_labels"] == ["new_owner"]
-
- @pytest.mark.asyncio
- async def test_runtime_output_repair_facts_trigger_first_contact_envelope(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- workflow_run_id="wr_current",
- structural_failure_identity="completion:runtime-output",
- runtime_output_repair_facts=[
- {
- "workflow_run_id": "wr_current",
- "block_label": "extract_entry_output",
- "output_path": "output.npi",
- "output_root": "output",
- "criterion_id": "requested_npi",
- "reason_code": "evidence_contradicts",
- "value_status": "null",
- }
- ],
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- npi = "1234567890"
- return npi
- """
- )
-
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- code = _single_code_block(parsed)["code"]
- assert 'return {"output": {"npi": npi}}' in code
- assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
- "output.npi"
- ]
- assert ctx.runtime_output_repair_attempt_by_signature
-
- @pytest.mark.asyncio
- async def test_update_workflow_applies_metadata_contract_scaffold_for_unambiguous_owner(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_flags",
- output_path="output.flags",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_entry_output
- code: |
- return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- stored = ctx.code_artifact_metadata["extract_entry_output"]
- assert stored["artifact_id"] == "code_artifact:extract_entry_output"
- assert stored["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
- assert stored["terminal_verifier_expectations"][0]["goal_value_paths"] == [
- "output.flags",
- "output.record_id",
- ]
- schema = json.loads(stored["claimed_outcomes"][0]["extraction_schema"])
- assert schema["properties"]["output"]["properties"]["record_id"] == {}
- assert schema["properties"]["output"]["properties"]["flags"] == {}
- assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
-
- @pytest.mark.asyncio
- async def test_separated_spine_shape_allows_neutral_multi_block_output_owner(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: enter_filters
- code: |
- await page.locator("#filter").fill("ABC123")
- - block_type: code
- label: choose_record
- code: |
- await page.locator("#choose").click()
- - block_type: code
- label: extract_record
- code: |
- return {"output": {"record_id": "ABC123"}}
- """
- )
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
- labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
- assert labels == ["enter_filters", "choose_record", "extract_record"]
-
- @pytest.mark.asyncio
- async def test_formation_prepared_p4_two_block_output_owner_ignores_generic_non_owner_label_reject(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_npi",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- workflow_yaml = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: search_directory
- prompt: Search the directory and extract provider result data.
- code: |
- await page.locator("#npi").fill("1234567890")
- await page.locator("#search_by").click()
- - block_type: code
- label: extract_directory_result
- prompt: Read the provider card and return the NPI.
- code: |
- await page.locator("#coastalCard").wait_for(state="visible")
- return {"output": {"npi": "1234567890"}}
- """
- )
- schema = '{"type":"object","properties":{"output":{"type":"object","properties":{"npi":{"type":"string"}}}}}'
- metadata = [
- {
- "block_label": "extract_directory_result",
- "claimed_outcomes": [{"goal_value_paths": ["output.npi"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output.npi"], "extraction_schema": schema}],
- }
- ]
-
- save_only = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert save_only["ok"] is False
- assert "search_directory" in save_only["error"]
-
- run_ctx = _code_only_ctx()
- run_ctx.request_policy = ctx.request_policy
- prepared_run = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- run_ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- formation_prepared=True,
- )
-
- assert prepared_run["ok"] is True
- assert run_ctx.latest_recorded_build_test_outcome is None
- assert workflow_update_module._artifact_declares_goal_values(
- run_ctx.code_artifact_metadata["extract_directory_result"]
- )
- assert not workflow_update_module._artifact_declares_goal_values(
- run_ctx.code_artifact_metadata["search_directory"]
- )
-
- @pytest.mark.asyncio
- async def test_separated_spine_shape_rejects_collapsed_output_owner(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: value
- blocks:
- - block_type: code
- label: extract_record
- parameter_keys:
- - value
- code: |
- _scout_entry_target = page.locator("#filter")
- try:
- await _scout_entry_target.wait_for(state="visible", timeout=1000)
- except Exception:
- await page.goto("https://example.com/records", wait_until="domcontentloaded")
- await _scout_entry_target.wait_for(state="visible")
- await page.locator("#filter").fill(str(value))
- await page.locator("#choose").click()
- await page.wait_for_load_state("domcontentloaded")
- return {"output": {"record_id": "ABC123"}}
- """
- )
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "output_contract_required"
- assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
-
- @pytest.mark.asyncio
- async def test_separated_spine_shape_rejects_collapsed_output_owner_with_sibling_block(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: prepare_session
- code: |
- context["ready"] = True
- - block_type: code
- label: extract_record
- code: |
- _scout_entry_target = page.locator("#filter")
- try:
- await _scout_entry_target.wait_for(state="visible", timeout=1000)
- except Exception:
- await page.goto("https://example.com/records", wait_until="domcontentloaded")
- await _scout_entry_target.wait_for(state="visible")
- await page.locator("#filter").fill(str(value))
- await page.locator("#choose").click()
- await page.wait_for_load_state("domcontentloaded")
- return {"output": {"record_id": "ABC123"}}
- """
- )
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "output_contract_required"
- assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
- assert result["data"]["block_label"] == "extract_record"
- assert result["data"]["output_owner_labels"] == ["extract_record"]
-
- @pytest.mark.asyncio
- async def test_separated_spine_shape_rejects_collapsed_output_owner_with_extra_browser_action(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: value
- blocks:
- - block_type: code
- label: extract_record
- parameter_keys:
- - value
- code: |
- _scout_entry_target = page.locator("#filter")
- try:
- await _scout_entry_target.wait_for(state="visible", timeout=1000)
- except Exception:
- await page.goto("https://example.com/records", wait_until="domcontentloaded")
- await _scout_entry_target.wait_for(state="visible")
- await page.locator("#filter").fill(str(value))
- await page.locator("#choose").click()
- await page.wait_for_load_state("domcontentloaded")
- await page.locator("#extra").click()
- return {"output": {"record_id": "ABC123"}}
- """
- )
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "output_contract_required"
- assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
- assert result["data"]["block_label"] == "extract_record"
-
- @pytest.mark.asyncio
- async def test_separated_spine_shape_ignores_one_block_without_multi_stage_spine(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- workflow_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: extract_record
- code: |
- return {"output": {"record_id": "ABC123"}}
- """
- )
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
-
- @pytest.mark.asyncio
- async def test_requested_output_contract_allows_aligned_initial_candidate(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_entry"
- child_paths = ["output.npi", "output.locations[].address", "output.statuses"]
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{'
- '"npi":{"type":"string"},'
- '"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
- '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
- )
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id=f"requested_{index}",
- output_path=path,
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- for index, path in enumerate(child_paths)
- ]
- )
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
- "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- candidate_yaml = _yaml(
- f"""
- title: Entry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{
- "output": {{
- "npi": "1234567890",
- "locations": [{{"address": "Example location"}}],
- "statuses": ["active"],
- }}
- }}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
- assert labels == [label]
-
- @pytest.mark.asyncio
- async def test_requested_output_contract_missing_metadata_records_child_path_facts(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- _typed_completion_criterion(
- id="requested_statuses",
- output_path="output.statuses",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
- candidate_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: lookup_result}
- blocks:
- - block_type: code
- label: extract_entry_output
- prompt: Extract structured fields from the entry.
- code: |
- await page.goto("https://example.com/search")
- return {"output": {"summary": "found"}}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is False
- assert "Required requested output paths: output.npi, output.statuses" in result["error"]
- expected_facts = [
- {
- "output_path": "output.npi",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "output.statuses",
- "output_root": "output",
- "reason_code": "requested_output_contract_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- ]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == expected_facts
- assert result["data"]["missing_requested_output_facts"] == expected_facts
- assert result["data"]["metadata_repair_contract"] == {
- "block_label": "extract_entry_output",
- "required_goal_value_paths": ["output.npi", "output.statuses"],
- "required_extraction_schema_paths": ["output.npi", "output.statuses"],
- "required_code_return_paths": ["output.npi", "output.statuses"],
- "source": "requested_output_contract",
- "reason_code": "requested_output_contract_missing_output_coverage",
- }
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "metadata_reject"
- assert repair_context["block_label"] == "extract_entry_output"
- assert repair_context["runtime_failure_class"] == "requested_output_contract_missing_output_coverage"
- assert repair_context["required_goal_value_paths"] == ["output.npi", "output.statuses"]
- assert repair_context["required_extraction_schema_paths"] == ["output.npi", "output.statuses"]
- assert repair_context["required_code_return_paths"] == ["output.npi", "output.statuses"]
- assert repair_context["metadata_contract_source"] == "requested_output_contract"
- assert repair_context["metadata_contract_reason_code"] == ("requested_output_contract_missing_output_coverage")
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
- sanitized = sanitize_tool_result_for_llm("update_workflow", result)
- assert sanitized["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
- "output.npi",
- "output.statuses",
- ]
- assert sanitized["data"]["authoring_repair_context"]["required_extraction_schema_paths"] == [
- "output.npi",
- "output.statuses",
- ]
- assert sanitized["data"]["authoring_repair_context"]["required_code_return_paths"] == [
- "output.npi",
- "output.statuses",
- ]
- assert sanitized["data"]["authoring_repair_context"]["metadata_contract_source"] == "requested_output_contract"
- assert sanitized["data"]["metadata_repair_contract"] == result["data"]["metadata_repair_contract"]
-
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{'
- '"npi":{"type":"string"},'
- '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
- )
- corrected_metadata = {
- "extract_entry_output": {
- "block_label": "extract_entry_output",
- "claimed_outcomes": [
- {"goal_value_paths": ["output.npi", "output.statuses"], "extraction_schema": schema}
- ],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.npi", "output.statuses"], "extraction_schema": schema}
- ],
- }
- }
- corrected_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: lookup_result}
- blocks:
- - block_type: code
- label: extract_entry_output
- prompt: Extract structured fields from the entry.
- code: |
- await page.goto("https://example.com/search")
- return {"output": {"npi": "1234567890", "statuses": ["active"]}}
- """
- )
-
- corrected = await _update_workflow(
- {"workflow_yaml": corrected_yaml, "code_artifact_metadata": corrected_metadata},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert corrected["ok"] is True
- accepted_yaml = parse_workflow_yaml(ctx.workflow_yaml)
- accepted_block = _single_code_block(accepted_yaml)
- assert accepted_block["label"] == "extract_entry_output"
- assert (
- accepted_block["code"].strip().endswith('return {"output": {"npi": "1234567890", "statuses": ["active"]}}')
- )
- assert accepted_block["extraction_schema"] == schema
- stored_metadata = ctx.code_artifact_metadata["extract_entry_output"]
- assert stored_metadata["block_label"] == corrected_metadata["extract_entry_output"]["block_label"]
- assert stored_metadata["claimed_outcomes"][0]["goal_value_paths"] == ["output.npi", "output.statuses"]
- assert stored_metadata["claimed_outcomes"][0]["extraction_schema"] == schema
- assert stored_metadata["terminal_verifier_expectations"][0]["goal_value_paths"] == [
- "output.npi",
- "output.statuses",
- ]
- assert stored_metadata["terminal_verifier_expectations"][0]["extraction_schema"] == schema
- assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
-
- @pytest.mark.asyncio
- async def test_recorded_outcome_missing_metadata_records_recorded_child_path_facts(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.requested_only",
- level="run",
- method_mandated=False,
- kind="outcome",
- ),
- ]
- )
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "output.recorded", "output_root": "output", "value_status": "no_typed_value"},
- ],
- )
- candidate_yaml = _yaml(
- """
- title: Entry lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: lookup_result}
- blocks:
- - block_type: code
- label: extract_entry_output
- prompt: Extract structured fields from the entry.
- code: |
- await page.goto("https://example.com/search")
- return {"output": {"summary": "found"}}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is False
- assert "Required requested output paths: output.recorded" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "output.recorded",
- "output_root": "output",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
- assert result["data"]["missing_requested_output_facts"] == outcome.missing_requested_output_facts
- assert result["data"]["metadata_repair_contract"] == {
- "block_label": "extract_entry_output",
- "required_goal_value_paths": ["output.recorded"],
- "required_extraction_schema_paths": ["output.recorded"],
- "required_code_return_paths": ["output.recorded"],
- "source": "recorded_outcome",
- "reason_code": "recorded_outcome_missing_output_coverage",
- }
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "metadata_reject"
- assert repair_context["block_label"] == "extract_entry_output"
- assert repair_context["runtime_failure_class"] == "recorded_outcome_missing_output_coverage"
- assert repair_context["required_goal_value_paths"] == ["output.recorded"]
- assert repair_context["required_extraction_schema_paths"] == ["output.recorded"]
- assert repair_context["required_code_return_paths"] == ["output.recorded"]
- assert repair_context["metadata_contract_source"] == "recorded_outcome"
- assert repair_context["metadata_contract_reason_code"] == "recorded_outcome_missing_output_coverage"
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_when_only_last_run_produced_required_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "extract_top_hn_post"
- ctx.verified_block_outputs[label] = {
- "top_post": "Claude Sonnet 5",
- "rank": 1,
- "url": "https://news.ycombinator.com/item?id=1",
- }
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Hacker News lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"rank": 1, "url": "https://news.ycombinator.com/item?id=1"}}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is False
- assert "does not cover the missing requested output paths" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.reason_code == "metadata_reject"
- assert outcome.missing_requested_output_facts == [
- {
- "output_path": "top_post",
- "output_root": "top_post",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- ]
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_static_return_for_required_root(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "extract_top_hn_post"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["top_post"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["top_post"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Hacker News lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- return {{"top_post": "Claude Sonnet 5", "rank": 1}}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_still_rejects_empty_last_run_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.verified_block_outputs[label] = {
- "address": "",
- "credentialing_status": "",
- "locations": [],
- "statuses": [],
- }
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is False
- assert "does not return any keyed output" in result["error"]
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_candidate_covering_missing_output_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [
- {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
- ],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
- ],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- address = "North Carolina, USA"
- credentialing_status = "unknown"
- locations = []
- statuses = []
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_helper_returning_missing_output_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- async def extract():
- return {{"address": "North Carolina", "credentialing_status": "unknown"}}
-
- return await extract()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_helper_local_dict_output_roots(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- async def extract():
- result = {{"address": "North Carolina", "credentialing_status": "unknown"}}
- return result
-
- return await extract()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_top_level_literal_key_dict_updates(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- result = {{"address": "North Carolina"}}
- result["credentialing_status"] = "unknown"
- return result
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_helper_literal_key_dict_updates(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
- }
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- authored_structure_signature="authored:previous-failed-candidate",
- missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- ],
- )
- candidate_yaml = _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
- async def extract():
- result = {{}}
- result["address"] = "North Carolina"
- result["credentialing_status"] = "unknown"
- return result
-
- return await extract()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
-
- @pytest.mark.asyncio
- async def test_captured_select_option_rejects_authored_text_click_plan_selection(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "select_option",
- "selector": "#planSelect",
- "source_url": "https://example.com/plans?session=private",
- "value": "gold",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Plan lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: plan
- blocks:
- - block_type: code
- label: plan_selection
- parameter_keys:
- - plan
- code: |
- main = page.locator("main")
- await main.get_by_text(str(plan), exact=True).first.click()
- return {"selected_plan": str(plan)}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is False
- assert "select-option interaction with a text click" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "select_option_interaction_mismatch"
- assert repair_context["block_label"] == "plan_selection"
- assert repair_context["selector"] == "#planSelect"
- assert repair_context["source_url"] == "https://example.com"
- assert "private" not in str(repair_context)
- assert ctx.last_code_authoring_repair_context is not None
- assert ctx.last_code_authoring_repair_context.reason_code == "select_option_interaction_mismatch"
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "code_safety_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- assert ctx.has_staged_proposal is False
-
- @pytest.mark.asyncio
- async def test_captured_select_option_allows_authored_select_api(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "select_option",
- "selector": "#planSelect",
- "source_url": "https://example.com/plans",
- "value": "gold",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Plan lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: plan
- blocks:
- - block_type: code
- label: plan_selection
- parameter_keys:
- - plan
- code: |
- await page.locator("#planSelect").select_option(label=str(plan))
- return {"selected_plan": str(plan)}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
- assert ctx.last_code_authoring_repair_context is None
-
- @pytest.mark.asyncio
- async def test_captured_select_option_allows_later_unrelated_text_click(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "select_option",
- "selector": "#planSelect",
- "source_url": "https://example.com/plans",
- "value": "gold",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Plan lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: plan
- blocks:
- - block_type: code
- label: plan_selection
- parameter_keys:
- - plan
- code: |
- await page.locator("#planSelect").select_option(label=str(plan))
- return {"selected_plan": str(plan)}
- - block_type: code
- label: continue_from_plan
- code: |
- await page.get_by_text("Continue", exact=True).click()
- return {"continued": True}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
- assert ctx.last_code_authoring_repair_context is None
-
- @pytest.mark.asyncio
- async def test_runtime_hidden_option_failure_rejects_repeated_text_click_structure(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "select_option",
- "selector": "#planSelect",
- "source_url": "https://example.com/plans",
- "value": "gold",
- "trajectory_index": 0,
- }
- ]
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="runtime_block_failure",
- structural_failure_identity="runtime:hidden-option-click",
- authored_structure_signature="authored:previous-hidden-option-click",
- )
-
- def submitted_yaml(return_value: str) -> str:
- return _yaml(
- f"""
- title: Plan lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: plan
- blocks:
- - block_type: code
- label: plan_selection
- parameter_keys:
- - plan
- code: |
- main = page.locator("main")
- await main.get_by_text(str(plan), exact=True).first.click()
- return {{"selected_plan": "{return_value}"}}
- """
- )
-
- first = await _update_workflow({"workflow_yaml": submitted_yaml("first")}, ctx, allow_missing_credentials=True)
- first_key = ctx.latest_recorded_build_test_outcome.structural_key
- first_count = ctx.code_authoring_guardrail_reject_count
- second = await _update_workflow(
- {"workflow_yaml": submitted_yaml("second")}, ctx, allow_missing_credentials=True
- )
-
- assert first["ok"] is False
- assert second["ok"] is False
- assert first_key is not None
- assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
- assert first_count == 1
- assert ctx.code_authoring_guardrail_reject_count == 2
- assert ctx.has_staged_proposal is False
-
- @pytest.mark.asyncio
- async def test_raw_conflict_marker_reject_carries_progress_surface_kind(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- result = await _update_workflow({"workflow_yaml": f"<<<<<<< HEAD\n{_SAFE_CODE_YAML}"}, ctx)
- assert result["ok"] is False
- assert result["data"]["surface_kind"] == "code_repair_progress"
- assert result["data"]["progress_text"]
-
- @pytest.mark.asyncio
- async def test_credential_scout_reject_is_not_classified_as_progress(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- result = await _update_workflow(
- {
- "workflow_yaml": _credential_code_yaml(
- code="""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(login_credential.totp)
- await page.locator("input[type='submit']").click()
- await page.wait_for_load_state("load")
- """
- )
- },
- ctx,
- )
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["data"].get("surface_kind") is None
-
-
-_SPINE_SYNTH_CODE = 'await page.locator("#stage-a").click()\nawait page.locator("#stage-b").click()'
-
-
-def _fake_spine_synthesized(
- *,
- parameters: list[dict[str, str]] | None = None,
- steps: list[dict[str, int]] | None = None,
- code: str | None = None,
- diagnostics: SynthesisDiagnostics | None = None,
-) -> SynthesizedCodeBlock:
- return SynthesizedCodeBlock(
- code=code if code is not None else _SPINE_SYNTH_CODE,
- parameters=parameters if parameters is not None else [],
- steps=steps if steps is not None else [{"line_start": 1, "line_end": 1}, {"line_start": 2, "line_end": 2}],
- diagnostics=diagnostics if diagnostics is not None else SynthesisDiagnostics(),
- )
-
-
-def _spine_emission_diagnostics() -> SynthesisDiagnostics:
- return SynthesisDiagnostics(
- emitted_interaction_count=2,
- emitted_interactions=[
- {
- "trajectory_index": 0,
- "tool_name": "click",
- "method": "click",
- "selector": "#stage-a",
- "locator": 'page.locator("#stage-a")',
- },
- {
- "trajectory_index": 1,
- "tool_name": "click",
- "method": "click",
- "selector": "#stage-b",
- "locator": 'page.locator("#stage-b")',
- },
- ],
- )
-
-
-def _spine_actuation_ctx() -> CopilotContext:
- ctx = _code_only_ctx()
- ctx.turn_id = "t-spine"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#stage-a", "source_url": "https://example.com/records"},
- {"tool_name": "click", "selector": "#stage-b", "source_url": "https://example.com/records"},
- ]
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_value",
- output_path="output.record_id",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- return ctx
-
-
-def _collapsed_spine_yaml(code_body: str) -> str:
- indented = textwrap.indent(textwrap.dedent(code_body).strip(), " " * 10)
- return _yaml(
- "title: Entry lookup\n"
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: extract_record\n"
- " code: |\n"
- f"{indented}\n"
- )
-
-
-class TestSeparatedSpineViolationActuation:
- def test_branch_a_split_replaces_collapsed_owner_with_stages_plus_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
- )
-
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
-
- assert applied is True
- blocks = workflow_blocks(parse_workflow_yaml(new_yaml))
- assert [block.get("label") for block in blocks] == [
- "extract_record_browser_stage_1",
- "extract_record_browser_stage_2",
- "extract_record",
- ]
- retained_code = str(blocks[-1].get("code") or "")
- assert "inner_text" in retained_code
- assert ".click()" not in retained_code
-
- def test_branch_a_result_is_idempotent_no_resplit(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
- )
-
- split_yaml, _meta, _applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- again_yaml, _meta2, _applied2 = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, split_yaml, []
- )
-
- assert [block.get("label") for block in workflow_blocks(parse_workflow_yaml(again_yaml))] == [
- "extract_record_browser_stage_1",
- "extract_record_browser_stage_2",
- "extract_record",
- ]
- assert not ctx.output_contract_spine_directive_blockers_by_attempt_key
-
- def test_branch_b_arms_directive_and_returns_unchanged_yaml(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
-
- assert applied is False
- assert new_yaml == workflow_yaml
- signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- attempt_key = workflow_update_module._output_contract_spine_directive_attempt_key(
- signature=signature, block_label="extract_record", workflow_yaml=workflow_yaml
- )
- assert ctx.output_contract_spine_directive_blockers_by_attempt_key[attempt_key] == [
- "extraction_boundary_ambiguous"
- ]
-
- @pytest.mark.parametrize(
- "code_body, synth_kwargs, expected_blocker",
- [
- (
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}',
- {},
- "extraction_boundary_ambiguous",
- ),
- (
- _SPINE_SYNTH_CODE + '\nawait page.locator("#extra").click()\nreturn {"output": {"record_id": "X"}}',
- {},
- "extraction_suffix_contains_browser_actions",
- ),
- (
- _SPINE_SYNTH_CODE + '\nvalue = await page.locator("#result").inner_text()',
- {},
- "static_return_envelope_unavailable",
- ),
- (
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}',
- {"steps": [{"line_start": 1, "line_end": 2}]},
- "insufficient_durable_stages",
- ),
- (
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}',
- {"parameters": [{"key": "alpha"}, {"key": "alpha"}]},
- "parameter_reconciliation_failed",
- ),
- ],
- )
- def test_branch_b_precondition_failures_arm_matching_blocker(
- self,
- monkeypatch: pytest.MonkeyPatch,
- code_body: str,
- synth_kwargs: dict[str, object],
- expected_blocker: str,
- ) -> None:
- monkeypatch.setattr(
- workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized(**synth_kwargs)
- )
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(code_body)
-
- _new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
-
- assert applied is False
- armed = list(ctx.output_contract_spine_directive_blockers_by_attempt_key.values())
- assert armed == [[expected_blocker]]
-
- def test_evaluation_enriches_repair_context_and_payload_after_directive_armed(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert evaluation.shape_violations == ["separated_spine_shape_required"]
- assert evaluation.repair_context is not None
- assert evaluation.repair_context.required_block_structure == "separated_browser_spine_plus_extraction"
- assert evaluation.repair_context.spine_split_blockers == ["extraction_boundary_ambiguous"]
- assert evaluation.payload["spine_structure_directive"]["spine_split_blockers"] == [
- "extraction_boundary_ambiguous"
- ]
-
- def test_directive_renders_into_next_authoring_prompt(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert evaluation is not None
- ctx.last_code_authoring_repair_context = evaluation.repair_context
-
- rendered = agent_module._code_authoring_repair_context_prompt(ctx)
-
- assert "required_block_structure: separated_browser_spine_plus_extraction" in rendered
- assert "spine_split_blockers:" in rendered
- assert "one browser-stage code block per scouted mutation stage" in rendered
-
- def test_preflight_reject_persists_directive_repair_context(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- prepared_yaml, prepared_metadata, _ = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- prepared_metadata, _ = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, prepared_yaml, prepared_metadata
- )
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, prepared_yaml, prepared_metadata)
-
- assert result is not None
- assert result["ok"] is False
- assert ctx.last_code_authoring_repair_context is not None
- assert (
- ctx.last_code_authoring_repair_context.required_block_structure == "separated_browser_spine_plus_extraction"
- )
-
- def test_cosmetic_churn_escalates_to_advisory_instead_of_rearming(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- first_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
- cosmetic_yaml = _collapsed_spine_yaml(
- "_setup = 2\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, first_yaml, [])
- assert len(ctx.output_contract_spine_directive_blockers_by_attempt_key) == 1
- assert not ctx.output_contract_actuation_by_signature
- assert ctx.turn_halt is None
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, cosmetic_yaml, [])
- assert list(ctx.output_contract_actuation_by_signature.values()) == [OutputContractAdvisoryState.GRANTED]
- assert ctx.turn_halt is None
- assert len(ctx.output_contract_spine_directive_blockers_by_attempt_key) == 1
-
- def test_granted_advisory_arms_run_evidence_without_preflight_consume_when_run_attemptable(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _spine_actuation_ctx()
- signature = "sig_preflight_dispatch"
- ctx.output_contract_actuation_by_signature[signature] = OutputContractAdvisoryState.GRANTED
- evaluation = workflow_update_module._OutputContractEvaluation(
- block_label="extract_record",
- artifact_id="art-1",
- required_paths={"output.confirmation_number"},
- observation_paths={"output.confirmation_number"},
- declaration_paths=set(),
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- missing_metadata_paths=[],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=["separated_spine_shape_violation"],
- canonical_signature=signature,
- payload={
- "reason_code": "requested_output_contract_missing_output_coverage",
- "missing_requested_output_facts": [],
- },
- repair_context=None,
- can_attempt_run=True,
- )
- monkeypatch.setattr(
- workflow_update_module,
- "_impose_output_contract_envelope_after_steering",
- lambda ctx, wf, meta: (wf, meta, False),
- )
- monkeypatch.setattr(
- workflow_update_module,
- "_scaffold_metadata_contract_for_update",
- lambda ctx, wf, meta: (meta, False),
- )
- monkeypatch.setattr(
- workflow_update_module,
- "_evaluate_output_contract_for_code_block",
- lambda *a, **k: evaluation,
- )
-
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, "title: x", [])
-
- assert result is None
- assert ctx.output_contract_pending_run_evidence.get(signature) == ["output.confirmation_number"]
- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
-
- run_result = {
- "data": {"blocks": [{"label": "extract_record", "extracted_data": {"confirmation_number": "R-9"}}]}
- }
- workflow_update_module.record_output_contract_run_output_evidence(ctx, run_result)
- assert ctx.output_contract_run_output_observed_by_signature[signature] is True
- assert ctx.output_contract_run_bound_required_path_by_signature[signature] is True
-
-
-def _dual_output_owner_yaml() -> str:
- return _yaml(
- "title: Entry lookup\n"
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: extract_a\n"
- " code: |\n"
- ' return {"output": {"record_id": "A"}}\n'
- " - block_type: code\n"
- " label: extract_b\n"
- " code: |\n"
- ' return {"output": {"record_id": "B"}}\n'
- )
-
-
-class TestAmbiguousOutputOwnerActuation:
- def _signature(self) -> str:
- return workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
-
- def test_ambiguous_owner_arms_directive_instead_of_bailing(self) -> None:
- ctx = _spine_actuation_ctx()
- workflow_yaml = _dual_output_owner_yaml()
-
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
-
- assert applied is False
- assert new_yaml == workflow_yaml
- assert ctx.output_contract_output_owner_directive_candidates_by_signature[self._signature()] == [
- "extract_a",
- "extract_b",
- ]
-
- def test_evaluation_enriches_owner_ambiguity_repair_context_after_directive_armed(self) -> None:
- ctx = _spine_actuation_ctx()
- workflow_yaml = _dual_output_owner_yaml()
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
-
- assert evaluation is not None
- assert "ambiguous_output_owner" in evaluation.shape_violations
- assert evaluation.repair_context is not None
- assert evaluation.repair_context.reason_code == "output_owner_ambiguous"
- assert evaluation.repair_context.output_owner_candidate_labels == ["extract_a", "extract_b"]
- assert evaluation.payload["output_owner_directive"]["output_owner_candidate_labels"] == [
- "extract_a",
- "extract_b",
- ]
-
- def test_owner_directive_renders_into_next_authoring_prompt(self) -> None:
- ctx = _spine_actuation_ctx()
- workflow_yaml = _dual_output_owner_yaml()
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert evaluation is not None
- ctx.last_code_authoring_repair_context = evaluation.repair_context
-
- rendered = agent_module._code_authoring_repair_context_prompt(ctx)
-
- assert "output_owner_candidate_labels: extract_a, extract_b" in rendered
- assert "sole output owner" in rendered
-
- def test_preflight_reject_persists_owner_ambiguity_repair_context(self) -> None:
- ctx = _spine_actuation_ctx()
- workflow_yaml = _dual_output_owner_yaml()
-
- prepared_yaml, prepared_metadata, _ = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- prepared_metadata, _ = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, prepared_yaml, prepared_metadata
- )
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, prepared_yaml, prepared_metadata)
-
- assert result is not None
- assert result["ok"] is False
- assert ctx.last_code_authoring_repair_context is not None
- assert ctx.last_code_authoring_repair_context.reason_code == "output_owner_ambiguous"
-
- def test_owner_directive_emits_fingerprint_once_per_signature(self) -> None:
- ctx = _spine_actuation_ctx()
- workflow_yaml = _dual_output_owner_yaml()
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
-
- assert list(ctx.output_contract_output_owner_directive_candidates_by_signature.keys()) == [self._signature()]
-
-
-def _already_split_spine_yaml(extra_sibling_code: str | None = None, *, base: str | None = None) -> str:
- base = base or workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- extra_block = ""
- if extra_sibling_code is not None:
- extra_block = (
- f" - block_type: code\n label: {base}_browser_stage_extra\n code: |\n"
- + textwrap.indent(textwrap.dedent(extra_sibling_code).strip(), " " * 6)
- + "\n"
- )
- return _yaml(
- "title: Entry lookup\n"
- "workflow_definition:\n"
- " blocks:\n"
- f" - block_type: code\n label: {base}_browser_stage_1\n code: |\n"
- ' await page.locator("#stage-a").click()\n'
- f" - block_type: code\n label: {base}_browser_stage_2\n code: |\n"
- ' await page.locator("#stage-b").click()\n'
- f"{extra_block}"
- f" - block_type: code\n label: {base}\n code: |\n"
- ' value = await page.locator("#result").inner_text()\n'
- ' return {"output": {"record_id": value}}\n'
- )
-
-
-def _imposition_split_ctx() -> CopilotContext:
- ctx = _spine_actuation_ctx()
- ctx.impose_synthesized_code_block = True
- ctx.raw_code_artifact_metadata = [
- {
- "block_label": workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": '{"type":"object"}'}],
- }
- ]
- return ctx
-
-
-class TestSeparatedSpineImpositionRunEligibility:
- def test_imposition_accepts_scouted_browser_stage_siblings(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _imposition_split_ctx()
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(_already_split_spine_yaml(), ctx)
-
- assert result.violations == []
-
- def test_imposition_still_flags_unscouted_sibling_mutation(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _imposition_split_ctx()
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(
- _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
- )
-
- assert any("unscouted browser action" in violation for violation in result.violations)
- assert any("#hallucinated" in violation for violation in result.violations)
- flagged_actions = [violation.split(" Provenance: ")[0] for violation in result.violations]
- assert not any("#stage-a" in flagged for flagged in flagged_actions)
-
- def test_whole_trajectory_validation_exempts_spine_covered_sibling_mutations(self) -> None:
- parsed = parse_workflow_yaml(_already_split_spine_yaml())
- blocks = workflow_update_module._workflow_code_blocks(parsed)
- extraction_block = next(
- block for block in blocks if block.get("label") == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- )
-
- validation = workflow_update_module._whole_trajectory_browser_surface_violations(
- code_blocks=blocks,
- selected_code_block=extraction_block,
- submitted_selected_code=str(extraction_block.get("code") or ""),
- synthesized_code=_SPINE_SYNTH_CODE,
- )
-
- assert validation.violations == []
-
- def test_split_imposed_yaml_clears_output_contract_run_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
- )
-
- split_yaml, split_metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- assert applied is True
-
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, split_yaml, split_metadata, allow_static_return_advisory=True
- )
- assert evaluation is not None
- assert workflow_update_module._SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE not in evaluation.shape_violations
- assert evaluation.can_attempt_run or not evaluation.has_deficiencies
-
- assert workflow_update_module._metadata_contract_run_preflight_reject(ctx, split_yaml, split_metadata) is None
-
- def test_directive_satisfying_reauthor_passes_preflight(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- collapsed_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- _yaml_out, _meta, armed = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, collapsed_yaml, []
- )
- assert armed is False
- assert ctx.output_contract_spine_directive_blockers_by_attempt_key
-
- schema = (
- '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
- )
- reauthored_yaml = _already_split_spine_yaml(base="extract_record")
- metadata = [
- {
- "block_label": "extract_record",
- "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
- "terminal_verifier_expectations": [
- {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
- ],
- }
- ]
-
- assert workflow_update_module._metadata_contract_run_preflight_reject(ctx, reauthored_yaml, metadata) is None
-
- def test_armed_attempt_key_survives_scaffold_metadata_owner_shift(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, workflow_yaml, [])
- armed_keys = set(ctx.output_contract_spine_directive_blockers_by_attempt_key)
- assert len(armed_keys) == 1
-
- scaffolded_metadata, _applied = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, workflow_yaml, []
- )
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths = contract.union
- read_label, _owner_labels = workflow_update_module._target_output_contract_block_label(
- ctx, workflow_yaml, scaffolded_metadata, required_paths
- )
- read_signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
- read_key = workflow_update_module._output_contract_spine_directive_attempt_key(
- signature=read_signature, block_label=read_label, workflow_yaml=workflow_yaml
- )
-
- assert read_key in armed_keys
-
- def test_granted_structural_advisory_lifts_output_contract_run_gate_on_unsplit_draft(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- collapsed_yaml = _collapsed_spine_yaml(
- "_setup = 1\n" + _SPINE_SYNTH_CODE + '\nreturn {"output": {"record_id": "X"}}'
- )
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, collapsed_yaml, [])
- blocked = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, collapsed_yaml, [], allow_static_return_advisory=True
- )
- assert blocked is not None and blocked.can_attempt_run is False
-
- workflow_update_module._impose_output_contract_envelope_after_steering(ctx, collapsed_yaml, [])
- assert list(ctx.output_contract_actuation_by_signature.values()) == [OutputContractAdvisoryState.GRANTED]
-
- scaffolded_metadata, _applied = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, collapsed_yaml, []
- )
- granted = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, collapsed_yaml, scaffolded_metadata, allow_static_return_advisory=True
- )
- assert granted is not None and granted.can_attempt_run is True
-
-
-def _budget_evaluation(ctx: CopilotContext) -> _OutputContractEvaluation:
- workflow_yaml = _collapsed_spine_yaml(
- _SPINE_SYNTH_CODE
- + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
- )
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
- assert evaluation is not None
- return evaluation
-
-
-class TestOutputContractBudgetRunGuard:
- def _prime(self, monkeypatch: pytest.MonkeyPatch) -> tuple[CopilotContext, _OutputContractEvaluation]:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- ctx.output_contract_reject_count_by_signature = {
- signature: workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS - 1
- }
- evaluation = _budget_evaluation(ctx)
- return ctx, evaluation
-
- def test_budget_rewrite_deferred_without_run_evidence(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx, evaluation = self._prime(monkeypatch)
-
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
-
- assert payload["output_contract_reject_count"] >= workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS
- assert payload["reason_code"] == "output_contract_required"
-
- def test_budget_rewrite_deferred_for_author_time_reject_outcome(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx, evaluation = self._prime(monkeypatch)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="author_time_reject", verdict="authoring_rejected", reason_code="metadata_reject"
- )
-
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
-
- assert payload["reason_code"] == "output_contract_required"
-
- def test_budget_rewrite_fires_with_persisted_run_outcome(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx, evaluation = self._prime(monkeypatch)
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- verdict="repairable_failure",
- reason_code="failed_run",
- workflow_run_id="wr_1",
- structural_failure_identity="runtime:failed",
- )
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"
-
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
-
- assert payload["reason_code"] == "output_contract_reject_budget_exhausted"
-
- def _prime_at_reject_count(
- self, monkeypatch: pytest.MonkeyPatch, *, reject_count: int, deferral_count: int
- ) -> tuple[CopilotContext, _OutputContractEvaluation]:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- ctx.output_contract_reject_count_by_signature = {signature: reject_count}
- ctx.output_contract_deferral_count_by_signature = {signature: deferral_count}
- evaluation = _budget_evaluation(ctx)
- return ctx, evaluation
-
- def test_zero_run_reject_loop_terminalizes_at_deferral_cap(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx, evaluation = self._prime_at_reject_count(
- monkeypatch,
- reject_count=workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS,
- deferral_count=workflow_update_module._MAX_OUTPUT_CONTRACT_DEFERRALS - 1,
- )
-
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
-
- assert payload["reason_code"] == "output_contract_reject_budget_exhausted"
-
- def test_zero_run_reject_below_deferral_cap_still_defers(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx, evaluation = self._prime_at_reject_count(
- monkeypatch,
- reject_count=workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS,
- deferral_count=0,
- )
-
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
-
- assert payload["reason_code"] == "output_contract_required"
-
- def test_deferral_cap_matches_max_rejects_plus_two(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
- ctx = _spine_actuation_ctx()
- signature = workflow_update_module._stable_output_contract_key("turn:t-spine", {"output.record_id"})
- ctx.output_contract_reject_count_by_signature = {
- signature: workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS - 1
- }
- ctx.output_contract_deferral_count_by_signature = {}
-
- reasons: list[str] = []
- for _ in range(6):
- evaluation = _budget_evaluation(ctx)
- payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
- reasons.append(str(payload["reason_code"]))
- if payload["reason_code"] == "output_contract_reject_budget_exhausted":
- break
-
- assert reasons[-1] == "output_contract_reject_budget_exhausted"
- assert reasons[:-1] == ["output_contract_required"] * (len(reasons) - 1)
- assert (
- ctx.output_contract_reject_count_by_signature[signature]
- == workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS + 2
- )
-
-
-class TestCodeBlockParameterPersistSeam:
- def test_declared_credential_key_is_adopted_for_unresolved_name(self) -> None:
- workflow_yaml = _yaml(
- """
- title: Portal login
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: credential_id
- key: portal_credentials
- default_value: cred_123
- blocks:
- - block_type: code
- label: sign_in
- code: |
- await page.locator("#user").fill(portal_credentials.username)
- """
- )
-
- adopted = workflow_update_module._adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
-
- parsed = parse_workflow_yaml(adopted)
- blocks = workflow_blocks(parsed)
- assert blocks[0]["parameter_keys"] == ["portal_credentials"]
-
- @pytest.mark.asyncio
- async def test_undeclared_parameter_key_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _SAFE_CODE_YAML
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
- submitted = _directory_blocks_yaml(
- """
- - block_type: code
- label: search_directory
- parameter_keys: [address_or_postal_code]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "search_directory" in result["error"]
- assert "`address_or_postal_code`" in result["error"]
- assert "workflow_definition.parameters" in result["error"]
- assert ctx.workflow_yaml == _SAFE_CODE_YAML
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
-
- @pytest.mark.asyncio
- async def test_nested_code_block_undeclared_parameter_key_rejects(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _directory_blocks_yaml(
- """
- - block_type: loop
- label: retry_search
- loop_blocks:
- - block_type: code
- label: nested_directory_search
- parameter_keys: [address_or_postal_code]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "nested_directory_search" in result["error"]
- assert "`address_or_postal_code`" in result["error"]
- assert ctx.workflow_yaml == ""
-
- def test_deeply_nested_parameter_contract_returns_validation_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
- block: dict[str, object] = {
- "block_type": "code",
- "label": "search_directory",
- "parameter_keys": ["search_location"],
- "code": "print(search_location)",
- }
- for index in range(1100):
- block = {
- "block_type": "loop",
- "label": f"loop_{index}",
- "loop_blocks": [block],
- }
- parsed = {
- "workflow_definition": {
- "parameters": [{"key": "search_location"}],
- "blocks": [block],
- }
- }
- monkeypatch.setattr(workflow_update_module, "parse_workflow_yaml", lambda _workflow_yaml: parsed)
-
- assert (
- workflow_update_module._code_block_parameter_contract_error("workflow_definition: {}")
- == "Workflow YAML nesting is too deep to validate."
- )
-
- def test_output_contract_graph_preserves_branch_isolation(self) -> None:
- workflow_yaml = _yaml(
- """
- workflow_definition:
- blocks:
- - block_type: code
- label: start_search
- code: |
- return {"ok": True}
- - block_type: task
- label: choose_path
- branch_conditions:
- - blocks:
- - block_type: code
- label: branch_a
- code: |
- return start_search_output
- - blocks:
- - block_type: code
- label: branch_b
- code: |
- return start_search_output
- - block_type: code
- label: after_choice
- code: |
- return choose_path_output
- """
- )
-
- available_by_label = code_block_available_binding_keys_by_label(workflow_yaml)
-
- assert available_by_label["branch_a"] == ["start_search_output"]
- assert available_by_label["branch_b"] == ["start_search_output"]
- assert "branch_a_output" not in available_by_label["branch_b"]
- assert available_by_label["after_choice"] == ["choose_path_output", "start_search_output"]
-
- @pytest.mark.asyncio
- async def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _code_yaml(
- "await page.locator('#query').fill(str(provider_query))",
- parameter_keys=["provider_query"],
- workflow_param=True,
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
-
- @pytest.mark.asyncio
- async def test_new_declared_workflow_parameter_key_is_accepted_on_later_edit(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _code_yaml(
- "await page.locator('#query').fill(str(provider_query))",
- parameter_keys=["provider_query"],
- workflow_param=True,
- )
- submitted = _yaml(
- """
- title: Registry lookup
- workflow_definition:
- parameters:
- - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample Search}
- - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: Example City}
- blocks:
- - block_type: code
- label: search_registry
- parameter_keys: [provider_query, search_location]
- code: |
- await page.locator("#query").fill(str(provider_query))
- await page.locator("#location").fill(str(search_location))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
-
- @pytest.mark.asyncio
- async def test_prior_block_output_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _directory_blocks_yaml(
- """
- - block_type: code
- label: search_registry
- code: |
- return {"records": []}
- - block_type: code
- label: summarize_registry
- parameter_keys: [search_registry_output]
- code: |
- print(search_registry_output)
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
-
- @pytest.mark.parametrize(
- ("submitted", "label", "key"),
- [
- (
- _directory_blocks_yaml(
- """
- - block_type: code
- label: summarize_registry
- parameter_keys: [search_registry_output]
- code: |
- print(search_registry_output)
- - block_type: code
- label: search_registry
- code: |
- return {"records": []}
- """
- ),
- "summarize_registry",
- "search_registry_output",
- ),
- (
- _directory_blocks_yaml(
- """
- - block_type: loop
- label: retry_search
- loop_blocks:
- - block_type: code
- label: nested_directory_search
- parameter_keys: [retry_search_output]
- code: |
- print(retry_search_output)
- """
- ),
- "nested_directory_search",
- "retry_search_output",
- ),
- ],
- )
- @pytest.mark.asyncio
- async def test_unavailable_output_parameter_key_rejects(
- self, monkeypatch: pytest.MonkeyPatch, submitted: str, label: str, key: str
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert label in result["error"]
- assert f"`{key}`" in result["error"]
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.parametrize("marker", ["<<<<<<< HEAD", "======="])
- @pytest.mark.asyncio
- async def test_raw_workflow_yaml_conflict_marker_rejects_before_persist(
- self, monkeypatch: pytest.MonkeyPatch, marker: str
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _SAFE_CODE_YAML
- submitted = f"{marker}\n{_SAFE_CODE_YAML}"
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert f"conflict marker `{marker}`" in result["error"]
- assert "line 1" in result["error"]
- assert ctx.workflow_yaml == _SAFE_CODE_YAML
-
- @pytest.mark.asyncio
- async def test_conflict_marker_inside_code_block_has_marker_specific_error(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: search_registry
- code: |
- <<<<<<< HEAD
- await page.goto("https://example.com/search")
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Code block `search_registry` contains unresolved conflict marker `<<<<<<< HEAD`" in result["error"]
- assert "not valid Python" not in result["error"]
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_registered_download_output_keys_are_not_parameter_keys(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- parameter_keys: [downloaded_files]
- code: |
- print(downloaded_files)
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "download_statement" in result["error"]
- assert "`downloaded_files`" in result["error"]
- assert "execution layer injects registered download output keys" in result["error"]
-
-
-class TestCompiledAuthoringImposition:
- def _provider_search_ctx(self) -> CopilotContext:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#provInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 13,
- "role": "textbox",
- "accessible_name": "Provider Name",
- "trajectory_index": 0,
- }
- ]
- return ctx
-
- def _typed_default_ctx(self) -> CopilotContext:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#search",
- "source_url": "https://example.com/catalog",
- "typed_length": 15,
- "typed_value": "example_sku_123",
- "role": "textbox",
- "accessible_name": "Search",
- "trajectory_index": 0,
- }
- ]
- return ctx
-
- @staticmethod
- def _cafe_search_capture() -> dict[str, object]:
- return {
- "tool_name": "type_text",
- "selector": "#café-search",
- "source_url": "https://example.com/catalog",
- "typed_length": 15,
- "role": "textbox",
- "accessible_name": "Catalog search",
- "trajectory_index": 0,
- }
-
- @pytest.mark.asyncio
- async def test_imposes_strict_scout_selector_and_lifts_singleton_literal(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
- assert "input[placeholder='Search']" not in block["code"]
- assert block["parameter_keys"] == ["provider_name"]
- parameters = parsed["workflow_definition"]["parameters"]
- assert parameters == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- "default_value": "Sample Search",
- }
- ]
- assert result["data"]["imposed_substitutions"] == {
- "block_label": "search_registry",
- "source_trajectory_count": 1,
- "parameter_keys": ["provider_name"],
- "credential_parameter_keys": [],
- "selector_provenance": [
- {
- "trajectory_index": 0,
- "selector": "#provInput",
- "emitted_literal": "#provInput",
- "source": "selector",
- }
- ],
- "prior_source": "workflow_yaml",
- }
-
- @pytest.mark.asyncio
- async def test_imposition_appends_no_fill_skeleton_and_emits_no_skeleton_substitution(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- metadata = [_terminal_metadata("search_registry", "search the registry")]
-
- result = await _update_workflow(
- {"workflow_yaml": _SUBMITTED_LITERAL_YAML, "code_artifact_metadata": metadata}, ctx
- )
-
- assert result["ok"] is True
- substitutions = result["data"]["imposed_substitutions"]
- assert substitutions["scrubbed_stale_selected_goal_value_paths"] is True
- assert "preserved_submitted_extraction_code" not in substitutions
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
- assert not workflow_update_module._artifact_declares_goal_values(ctx.code_artifact_metadata["search_registry"])
-
- @pytest.mark.asyncio
- async def test_output_intent_requires_artifact_metadata_before_persist(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
- submitted = _yaml(
- """
- title: Provider lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: provider_result}
- blocks:
- - block_type: code
- label: search_registry
- prompt: Search the registry and return structured provider result data.
- code: |
- await page.locator("input[placeholder='Search']").fill("Sample Search")
- await page.locator("button.lookup").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "must pass `code_artifact_metadata`" in result["error"]
- assert "goal_value_paths" in result["error"]
- assert ctx.workflow_yaml == ""
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.last_code_authoring_reject_was_credential_priority is False
-
- @pytest.mark.asyncio
- async def test_missing_metadata_with_credential_gap_uses_credential_priority_counter(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- submitted = _yaml(
- """
- title: Credentialed lookup
- workflow_definition:
- parameters:
- - {parameter_type: workflow, workflow_parameter_type: credential_id, key: login_credential, default_value: cred_missing}
- - {parameter_type: output, key: lookup_result}
- blocks:
- - block_type: code
- label: lookup_with_saved_credential
- parameter_keys: [login_credential]
- prompt: Sign in and return the lookup result.
- code: |
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- return {"lookup_result": "sample"}
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "must pass `code_artifact_metadata`" in result["error"]
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.last_code_authoring_reject_was_credential_priority is True
- assert ctx.blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_output_intent_rejects_partial_metadata_without_output_label(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- submitted = _yaml(
- """
- title: Provider lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: provider_result}
- blocks:
- - block_type: code
- label: open_registry
- prompt: Open the registry.
- code: |
- await page.goto("https://example.com/search")
- - block_type: code
- label: extract_registry
- prompt: Return structured provider result data.
- code: |
- records = [{"number": "REC-001"}]
- """
- )
-
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [_terminal_metadata("open_registry", "open the registry")],
- },
- ctx,
- )
-
- assert result["ok"] is False
- assert "extract_registry" in result["error"]
- assert "must pass `code_artifact_metadata`" in result["error"]
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_enter_directory_location_output_block_metadata_passes_before_persist(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: enter_directory_location
- prompt: Enter the directory location and output the selected address.
- code: |
- await page.locator("#locInput").fill("North Carolina, USA")
- address = "North Carolina, USA"
- """
- )
- metadata = [
- {
- "block_label": "enter_directory_location",
- "declared_goal": "Enter the directory location and return the address.",
- "claimed_outcomes": [
- {
- "id": "claim:address",
- "status": "observed_not_verified",
- "goal_value_paths": ["address"],
- }
- ],
- "terminal_verifier_expectations": [
- {
- "id": "expectation:address",
- "goal_value_paths": ["address"],
- }
- ],
- }
- ]
-
- result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml
- assert "enter_directory_location" in ctx.code_artifact_metadata
- assert ctx.latest_recorded_build_test_outcome is None
-
- @pytest.mark.asyncio
- async def test_invalid_metadata_records_typed_outcome_without_prose_payload(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- submitted = _yaml(
- """
- title: Provider lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: search_registry
- code: |
- records = [{"number": "REC-001"}]
- """
- )
- metadata = [
- {
- "block_label": "search_registry",
- "claimed_outcomes": [{"id": "claim:rows", "status": "satisfied"}],
- }
- ]
-
- result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
-
- assert result["ok"] is False
- assert "Artifact metadata" in result["error"]
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- structural_payload = workflow_update_module._code_artifact_metadata_reject_payload(
- workflow_yaml=submitted,
- raw_metadata=metadata,
- offending_labels=["search_registry"],
- violation_categories=["missing_declared_goal", "missing_required_list", "missing_artifact_refs"],
- )
- assert structural_payload is not None
- assert structural_payload["offending_labels"] == ["search_registry"]
- assert structural_payload["missing_fields_by_label"] == {
- "search_registry": [
- "declared_goal",
- "page_dependencies",
- "completion_criteria",
- "terminal_verifier_expectations",
- "evidence_refs_or_observation_refs",
- ]
- }
- assert structural_payload["code_block_output_status"] == {
- "search_registry": {
- "block_type": "code",
- "has_code": True,
- "declares_output_intent": False,
- "declares_output_roots": [],
- "has_meaningful_output": False,
- }
- }
- payload_text = str(structural_payload)
- assert "requires non-empty" not in payload_text
- assert "claim:rows" not in payload_text
-
- @pytest.mark.asyncio
- async def test_imposition_preserves_submitted_extraction_suffix(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = synthesized.code.rstrip() + '\nrecords = [{"number": "REC-001", "status": "credentialed"}]\n'
- submitted = yaml.safe_dump(
- {
- "title": "Provider lookup",
- "workflow_definition": {
- "parameters": [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- "default_value": "Sample Search",
- }
- ],
- "blocks": [
- {
- "block_type": "code",
- "label": "search_registry",
- "code": submitted_code,
- }
- ],
- },
- },
- sort_keys=False,
- )
- metadata = [_terminal_metadata("search_registry", "search the registry")]
-
- result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
-
- assert result["ok"] is True
- assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- output_block = _code_blocks(parsed)["search_registry"]
- browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
- output_code = str(output_block["code"])
- assert "<fill" not in browser_code
- assert "<fill: captured value>" not in browser_code
- assert 'await page.locator("#provInput").fill(str(provider_name))' in browser_code
- assert 'records = [{"number": "REC-001", "status": "credentialed"}]' in output_code
- assert 'return {"records": records}' in output_code
-
- @pytest.mark.asyncio
- async def test_imposition_preserves_custom_extraction_code_with_goal_metadata(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- synthesized_code = textwrap.dedent(synthesized.code).lstrip()
- submitted_code = (
- "records = []\n"
- + synthesized_code.rstrip()
- + '\nrecords.append({"number": "REC-001", "status": "credentialed"})\n'
- )
- submitted = yaml.safe_dump(
- {
- "title": "Provider lookup",
- "workflow_definition": {
- "parameters": [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- "default_value": "Sample Search",
- }
- ],
- "blocks": [
- {
- "block_type": "code",
- "label": "search_registry",
- "code": submitted_code,
- }
- ],
- },
- },
- sort_keys=False,
- )
- metadata = [_terminal_metadata("search_registry", "search the registry")]
-
- result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
-
- assert result["ok"] is False
- assert "selected output extraction boundary is ambiguous" in result["error"]
-
- def _download_ctx(self) -> CopilotContext:
- from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
-
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": "#statement-row",
- "source_url": "https://example.com/billing",
- "trajectory_index": 0,
- }
- ]
- ctx.reached_download_target = ReachedDownloadTarget(
- selector='a[href="/billing/statement.pdf"]',
- affordance_text="View Printable Statement",
- download_kind="attribute",
- source_step="trajectory_recency",
- already_registered=False,
- )
- return ctx
-
- @pytest.mark.asyncio
- async def test_imposition_forwards_reached_target_and_emits_download_terminal(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._download_ctx()
- submitted = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert "async with page.expect_download()" in block["code"]
- assert "/billing/statement.pdf" in block["code"]
- assert "save_as" not in block["code"]
- assert result["data"]["imposed_substitutions"]["block_label"] == "download_statement"
-
- @pytest.mark.asyncio
- async def test_imposed_download_terminal_clears_binding_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._download_ctx()
- ctx.flow_evidence = [
- {
- "evidence": {"source_tool": "scout_interaction", "interaction_selector": "#statement-row"},
- "reached_via": "interaction",
- }
- ]
- submitted = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))
- assert "async with page.expect_download()" in block["code"]
-
- @pytest.mark.asyncio
- async def test_reached_download_target_still_imposes_after_prior_update_attempt(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._download_ctx()
- ctx.update_workflow_called = True
- submitted = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))
- assert "async with page.expect_download()" in block["code"]
-
- @pytest.mark.asyncio
- async def test_imposition_carries_reached_download_target_to_synthesized_code(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": "div.stmt-row",
- "source_url": "https://example.com/bills",
- "trajectory_index": 0,
- }
- ]
- ctx.reached_download_target = ReachedDownloadTarget(
- selector='[href="/files/report.pdf"]',
- affordance_text="Download PDF",
- download_kind="extension",
- source_step="trajectory_recency",
- already_registered=False,
- )
- workflow_yaml = _yaml(
- """
- title: Download report
- workflow_definition:
- blocks:
- - block_type: code
- label: download_report
- code: |
- await page.goto("https://example.com/bills")
- await page.locator("div.stmt-row").click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
-
- assert result.violations == []
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert "expect_download" in block["code"]
- assert 'await page.locator("[href=\\"/files/report.pdf\\"]").click()' in block["code"]
-
- def test_imposition_targets_synthesized_label_in_multi_code_workflow(self) -> None:
- ctx = self._download_ctx()
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: summarize_statement
- code: |
- return {"status": "ready"}
- - block_type: code
- label: scout_synthesized_browser_steps
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
-
- assert result.violations == []
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
- assert blocks["summarize_statement"]["code"].strip() == 'return {"status": "ready"}'
- assert "async with page.expect_download()" in blocks["scout_synthesized_browser_steps"]["code"]
-
- def test_imposition_targets_recorded_outcome_owner_in_multi_code_workflow(self) -> None:
- ctx = self._download_ctx()
- ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key="recorded-download",
- phase="persisted_block_run",
- reason_code="outcome_not_demonstrated",
- frontier_facet="value_shape",
- owning_block_labels=["download_matching_invoice_pdf"],
- )
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: login_to_apex_business
- code: |
- return {"logged_in": True}
- - block_type: code
- label: download_matching_invoice_pdf
- code: |
- async with page.expect_download() as download_info:
- await page.locator("a[href='/billing/statement.pdf']").click()
- download = await download_info.value
- return {"downloaded_files": [download.suggested_filename]}
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
-
- assert result.violations == []
- assert result.substitutions is not None
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
- assert blocks["login_to_apex_business"]["code"].strip() == 'return {"logged_in": True}'
- assert "async with page.expect_download()" in blocks["download_matching_invoice_pdf"]["code"]
-
- def test_imposition_ignores_unchanged_persisted_browser_sibling(self) -> None:
- ctx = self._download_ctx()
- prior_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: login_to_apex_business
- code: |
- await page.locator("#email").fill(str(login_credentials["username"]))
- await page.locator("#password").fill(str(login_credentials["password"]))
- await page.locator("#sign-in").click()
- - block_type: code
- label: download_matching_invoice_pdf
- code: |
- return {"status": "ready"}
- """
- )
- submitted_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: login_to_apex_business
- code: |
- await page.locator("#email").fill(str(login_credentials["username"]))
- await page.locator("#password").fill(str(login_credentials["password"]))
- await page.locator("#sign-in").click()
- - block_type: code
- label: download_matching_invoice_pdf
- code: |
- await page.locator("#statement-row").click()
- """
- )
- ctx.workflow_yaml = prior_yaml
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted_yaml, ctx)
-
- assert result.violations == []
- assert result.substitutions is not None
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
- assert "#sign-in" in str(blocks["login_to_apex_business"]["code"])
- assert "async with page.expect_download()" in str(blocks["download_matching_invoice_pdf"]["code"])
-
- def test_prior_synthesized_label_prevents_unrelated_multi_code_imposition(self) -> None:
- ctx = self._download_ctx()
- prior_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: summarize_statement
- code: |
- return {"status": "ready"}
- - block_type: code
- label: scout_synthesized_browser_steps
- code: |
- await page.locator("#statement-row").click()
- """
- )
- submitted_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: summarize_statement
- code: |
- return {"status": "edited"}
- - block_type: code
- label: scout_synthesized_browser_steps
- code: |
- await page.locator("#statement-row").click()
- """
- )
- ctx.workflow_yaml = prior_yaml
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted_yaml, ctx)
-
- assert result.violations == []
- assert result.substitutions is None
- parsed = parse_workflow_yaml(result.workflow_yaml)
- assert isinstance(parsed, dict)
- blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
- assert blocks["summarize_statement"]["code"].strip() == 'return {"status": "edited"}'
- assert blocks["scout_synthesized_browser_steps"]["code"].strip() == (
- 'await page.locator("#statement-row").click()'
- )
-
- @pytest.mark.asyncio
- async def test_unchanged_prior_code_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "input[placeholder='Search']",
- "source_url": "https://example.com/find-care",
- "typed_length": 13,
- "role": "textbox",
- "accessible_name": "Search",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "button.lookup",
- "source_url": "https://example.com/find-care",
- "role": "button",
- "accessible_name": "Lookup",
- "trajectory_index": 1,
- },
- ]
- ctx.workflow_yaml = _SUBMITTED_LITERAL_YAML
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- assert "imposed_substitutions" not in result["data"]
- assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
-
- @pytest.mark.asyncio
- async def test_flag_off_code_only_mode_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.impose_synthesized_code_block = False
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- assert "imposed_substitutions" not in result["data"]
- assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
-
- @pytest.mark.asyncio
- async def test_flag_off_code_only_mode_does_not_promote_scouted_typed_defaults(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._typed_default_ctx()
- ctx.impose_synthesized_code_block = False
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_TYPED_LITERAL_REWRITE_YAML}, ctx)
-
- assert result["ok"] is True
- assert ctx.workflow_yaml == _SUBMITTED_TYPED_LITERAL_REWRITE_YAML
-
- @pytest.mark.asyncio
- async def test_single_direct_synthesized_parameter_reference_creates_required_input(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["provider_name"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- }
- ]
- expected_selector = ctx.scout_trajectory[0]["selector"]
- assert f'await page.locator("{expected_selector}").fill(str(provider_name))' in block["code"]
-
- @pytest.mark.asyncio
- async def test_repeated_direct_synthesized_parameter_reference_creates_required_input(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_REPEATED_COMPUTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["provider_name"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- }
- ]
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
-
- @pytest.mark.asyncio
- async def test_mixed_non_locator_fill_does_not_hide_direct_synthesized_parameter_reference(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_FILL_COMPUTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["provider_name"]
- expected_selector = ctx.scout_trajectory[0]["selector"]
- assert f'await page.locator("{expected_selector}").fill(str(provider_name))' in block["code"]
-
- @pytest.mark.asyncio
- async def test_mixed_locator_fill_rejects_direct_synthesized_parameter_reference(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is False
- assert "submitted code mixes direct fills using `provider_name`" in result["error"]
- assert ctx.workflow_yaml == ""
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["provider_name"]
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
-
- @pytest.mark.asyncio
- async def test_typed_default_contract_reject_clears_stale_repair_context(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._typed_default_ctx()
- submitted = _yaml(
- """
- title: Product lookup
- workflow_definition:
- parameters: invalid
- blocks:
- - block_type: code
- label: search_catalog
- code: |
- await page.locator("#search").fill("example_sku_123")
- """
- )
- ctx.workflow_yaml = submitted
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "workflow_definition.parameters must be a list" in result["error"]
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
-
- @pytest.mark.asyncio
- async def test_unknown_computed_parameter_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["provider_name"]
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_selector_join_aliases_synthesized_parameter_to_authored_fill_parameter(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- submitted = _yaml(
- """
- title: Provider lookup
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: search_term
- blocks:
- - block_type: code
- label: search_registry
- code: |
- await page.locator("#provInput").fill(search_term)
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["search_term"]
- assert 'await page.locator("#provInput").fill(str(search_term))' in block["code"]
- assert "provider_name" not in block["code"]
-
- @pytest.mark.asyncio
- async def test_single_synthesized_key_without_bindable_fill_records_repair_context(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#location",
- "source_url": "https://example.com/directory",
- "typed_length": 5,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: search_directory
- code: |
- await page.locator("#submit").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["selector"] == "#location"
- assert repair_context["source_url"] == "https://example.com"
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- @pytest.mark.parametrize(
- "workflow_yaml",
- [_SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML, _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML],
- )
- async def test_ambiguous_parameter_binding_rejects_without_count_authority(
- self, monkeypatch: pytest.MonkeyPatch, workflow_yaml: str
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.output_contract_reject_count_by_signature = {"unrelated": 99}
-
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["authoring_repair_context"]["reason_code"] == ("synthesized_parameter_binding_ambiguous")
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_synthesized_parameter_repair_context_uses_safe_selector_atom(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- raw_selector = "#location-" + ("x" * 180)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": raw_selector,
- "source_url": "https://example.com/directory?query=raw",
- "typed_length": 5,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: search_directory
- code: |
- await page.locator("#submit").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["selector"] in (None, "")
- assert repair_context["source_url"] == "https://example.com"
- assert raw_selector not in str(repair_context)
-
- @pytest.mark.asyncio
- async def test_multi_input_missing_synthesized_parameter_rejects_with_repair_context(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#confirmation",
- "source_url": "https://example.com/orders",
- "typed_length": 10,
- "role": "textbox",
- "accessible_name": "Enter Confirmation",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#postal",
- "source_url": "https://example.com/orders",
- "typed_length": 5,
- "role": "textbox",
- "accessible_name": "ZIP Code",
- "trajectory_index": 1,
- },
- ]
- submitted = _yaml(
- """
- title: Order lookup
- workflow_definition:
- parameters:
- - {parameter_type: workflow, workflow_parameter_type: string, key: order_lookup}
- blocks:
- - block_type: code
- label: order_status
- parameter_keys: [order_lookup]
- code: |
- await page.locator("#confirmation").fill(str(order_lookup))
- await page.locator("#postal").fill(str(order_lookup))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "authored selector-join alias is reused by another synthesized input" in result["error"]
- assert ctx.workflow_yaml == ""
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["block_label"] == "order_status"
- assert repair_context["unresolved_names"] == ["zip_code"]
- assert repair_context["available_parameter_keys"] == ["order_lookup"]
- assert repair_context["binding_candidates"] == ["zip_code", "order_lookup"]
- assert "update_and_run_blocks" in repair_context["repair_instruction"]
- assert "secret" not in str(repair_context).lower()
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
-
- @pytest.mark.asyncio
- async def test_synthesized_internal_parameter_aliases_to_existing_declared_default(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 17,
- "typed_value": "Example City, USA",
- "role": "textbox",
- "accessible_name": "Address or postal code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: search_location, default_value: "Example City, USA"}
- - {key: address_or_postal_code, default_value: "Example City, USA"}
- blocks:
- - block_type: code
- label: search_directory
- parameter_keys: [address_or_postal_code]
- parameters:
- - {key: address_or_postal_code, default_value: "Example City, USA"}
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["search_location"]
- assert "parameters" not in block
- assert "str(search_location)" in block["code"]
- assert "address_or_postal_code" not in block["code"]
- parameters = parsed["workflow_definition"]["parameters"]
- assert [parameter["key"] for parameter in parameters] == ["search_location"]
- assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
- "address_or_postal_code": "search_location"
- }
-
- @pytest.mark.asyncio
- async def test_multi_input_synthesized_parameter_aliases_before_ambiguity_guard(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 17,
- "typed_value": "Example City, USA",
- "role": "textbox",
- "accessible_name": "Address or postal code",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#firstName",
- "source_url": "https://example.com/find-care",
- "typed_length": 5,
- "typed_value": "Given",
- "role": "textbox",
- "accessible_name": "Provider First Name",
- "trajectory_index": 1,
- },
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: search_location, default_value: "Example City, USA"}
- - {key: provider_first_name, default_value: "Given"}
- blocks:
- - block_type: code
- label: search_directory
- parameter_keys: [address_or_postal_code, provider_first_name]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- await page.locator("#firstName").fill(str(provider_first_name))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["search_location", "provider_first_name"]
- assert "str(search_location)" in block["code"]
- assert "str(provider_first_name)" in block["code"]
- assert "address_or_postal_code" not in block["code"]
- assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
- "address_or_postal_code": "search_location"
- }
-
- @pytest.mark.asyncio
- async def test_synthesized_parameter_does_not_alias_by_typed_length_to_declared_default(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 17,
- "role": "textbox",
- "accessible_name": "Address or postal code",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#firstName",
- "source_url": "https://example.com/find-care",
- "typed_length": 5,
- "role": "textbox",
- "accessible_name": "Provider First Name",
- "trajectory_index": 1,
- },
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: search_location, default_value: "Example City, USA"}
- - {key: provider_first_name, default_value: "Given"}
- blocks:
- - block_type: code
- label: search_directory
- parameter_keys: [address_or_postal_code, provider_first_name]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- await page.locator("#firstName").fill(str(provider_first_name))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_or_postal_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_or_postal_code"]
- assert ctx.workflow_yaml == ""
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_synthesized_provider_search_key_does_not_compose_declared_first_last_inputs(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 17,
- "typed_value": "Example City, USA",
- "role": "textbox",
- "accessible_name": "Address or postal code",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#providerSearch",
- "source_url": "https://example.com/find-care",
- "typed_length": 12,
- "role": "textbox",
- "accessible_name": "Provider name or identifier",
- "trajectory_index": 1,
- },
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: search_location, default_value: "Example City, USA"}
- - {key: provider_first_name, default_value: "Given"}
- - {key: provider_last_name, default_value: "Family"}
- blocks:
- - block_type: code
- label: search_directory
- parameter_keys:
- - address_or_postal_code
- - provider_name_or_identifier
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- await page.locator("#providerSearch").fill(str(provider_name_or_identifier))
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `provider_name_or_identifier`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["provider_name_or_identifier"]
- assert ctx.workflow_yaml == ""
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- def test_identifier_rewrite_skips_string_literals_and_comments(self) -> None:
- source = (
- 'await page.locator("#providerSearch").fill(str(provider_query))\n'
- "# provider_query should stay readable in comments\n"
- 'message = "provider_query should stay readable in strings"\n'
- )
-
- rewritten = workflow_update_module._replace_python_identifier(
- source,
- "provider_query",
- "provider_name",
- )
-
- ast.parse(rewritten)
- assert 'await page.locator("#providerSearch").fill(str(provider_name))' in rewritten
- assert "# provider_query should stay readable in comments" in rewritten
- assert '"provider_query should stay readable in strings"' in rewritten
-
- def test_identifier_rewrite_preserves_multiline_block_shape(self) -> None:
- source = (
- "if provider_query:\n"
- ' await page.locator("#providerSearch").fill(str(provider_query))\n'
- "else:\n"
- ' await page.locator("#providerSearch").fill("")\n'
- )
-
- rewritten = workflow_update_module._replace_python_identifier(source, "provider_query", "provider_name")
-
- assert rewritten == (
- "if provider_name:\n"
- ' await page.locator("#providerSearch").fill(str(provider_name))\n'
- "else:\n"
- ' await page.locator("#providerSearch").fill("")\n'
- )
-
- @pytest.mark.asyncio
- async def test_single_synthesized_location_key_rejects_unique_declared_search_location_without_exact_default(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 19,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: plan, default_value: "Coastal Complete Health"}
- - {key: provider_first_name, default_value: "Jordan"}
- - {key: provider_last_name, default_value: "Avery"}
- - {key: provider_npi, default_value: "1457803926"}
- - {key: search_location, default_value: "North Carolina, USA"}
- blocks:
- - block_type: code
- label: open_directory_plan_selection
- parameter_keys:
- - address_city_county_or_zip_code
- code: |
- await page.locator("#planContinue").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["available_parameter_keys"] == [
- "plan",
- "provider_first_name",
- "provider_last_name",
- "provider_npi",
- "search_location",
- ]
- assert ctx.workflow_yaml == ""
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_single_synthesized_location_key_rejects_unique_declared_search_location_without_default(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 19,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {parameter_type: workflow, workflow_parameter_type: string, key: plan}
- - {parameter_type: workflow, workflow_parameter_type: string, key: provider_first_name}
- - {parameter_type: workflow, workflow_parameter_type: string, key: provider_last_name}
- - {parameter_type: workflow, workflow_parameter_type: string, key: provider_npi}
- - {parameter_type: workflow, workflow_parameter_type: string, key: search_location}
- blocks:
- - block_type: code
- label: open_directory_plan_selection
- parameter_keys:
- - address_city_county_or_zip_code
- code: |
- await page.locator("#planContinue").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["available_parameter_keys"] == [
- "plan",
- "provider_first_name",
- "provider_last_name",
- "provider_npi",
- "search_location",
- ]
- assert ctx.workflow_yaml == ""
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_single_synthesized_location_key_rejects_ambiguous_declared_location_alias_without_defaults(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 19,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {parameter_type: workflow, workflow_parameter_type: string, key: search_location}
- - {parameter_type: workflow, workflow_parameter_type: string, key: service_location}
- blocks:
- - block_type: code
- label: open_directory_plan_selection
- parameter_keys:
- - address_city_county_or_zip_code
- code: |
- await page.locator("#planContinue").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["available_parameter_keys"] == ["search_location", "service_location"]
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_single_synthesized_location_key_keeps_authoritative_reject_when_length_alias_ambiguous(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#locInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 19,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- }
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {key: search_location, default_value: "North Carolina, USA"}
- - {key: alternate_location, default_value: "South Carolina, USA"}
- blocks:
- - block_type: code
- label: open_directory_plan_selection
- parameter_keys:
- - address_city_county_or_zip_code
- code: |
- await page.locator("#planContinue").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["block_label"] == "open_directory_plan_selection"
- assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
- assert repair_context["available_parameter_keys"] == ["alternate_location", "search_location"]
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_single_local_string_constant_is_lifted_for_synthesized_key(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_LOCAL_CONSTANT_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
- assert block["parameter_keys"] == ["provider_name"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "provider_name",
- "default_value": "Sample Search",
- }
- ]
-
- @pytest.mark.asyncio
- async def test_single_submitted_string_parameter_is_not_adopted_by_length_for_synthesized_key(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
- ctx.scout_trajectory[0]["accessible_name"] = "Search by doctor name or specialty, hospital, procedure, and more"
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_PARAMETER_YAML}, ctx)
-
- synthesized_key = "search_by_doctor_name_or_specialty_hospital_procedure_and_more"
- assert result["ok"] is False
- assert f"Unable to bind synthesized parameter `{synthesized_key}`" in result["error"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair_context["unresolved_names"] == [synthesized_key]
- assert ctx.workflow_yaml == ""
- assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
-
- @pytest.mark.asyncio
- async def test_mixed_literal_and_computed_fill_rejects_before_persist(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LITERAL_YAML}, ctx)
-
- assert result["ok"] is False
- assert "submitted code mixes direct fills using `provider_name`" in result["error"]
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_promotes_scouted_typed_literal_across_multiple_and_nested_code_blocks(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._typed_default_ctx()
- ctx.scout_trajectory = [self._cafe_search_capture(), {**ctx.scout_trajectory[0], "trajectory_index": 1}]
-
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_TYPED_LITERAL_REWRITE_YAML}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
- assert "fill(str(search))" in blocks["search_catalog"]["code"]
- assert "type(str(search))" in blocks["select_result"]["code"]
- assert "fill(str(search))" in blocks["nested_search"]["code"]
- assert '"example_sku_123"' in blocks["verify_cart"]["code"]
- assert blocks["search_catalog"]["parameter_keys"] == ["existing_filter", "search"]
- assert blocks["select_result"]["parameter_keys"] == ["search"]
- assert blocks["nested_search"]["parameter_keys"] == ["search"]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "existing_filter",
- "default_value": "active",
- },
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "search",
- "default_value": "example_sku_123",
- },
- ]
-
- @pytest.mark.asyncio
- async def test_scouted_typed_default_without_literal_rewrite_does_not_create_orphan_input(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = self._typed_default_ctx()
- ctx.scout_trajectory = [self._cafe_search_capture(), {**ctx.scout_trajectory[0], "trajectory_index": 1}]
- submitted_yaml = _SUBMITTED_TYPED_LITERAL_REWRITE_YAML.replace("example_sku_123", "other_sku_456")
-
- result = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "existing_filter",
- "default_value": "active",
- }
- ]
- for block in workflow_blocks(parsed):
- if str(block.get("block_type") or "").lower() == "code":
- assert "other_sku_456" in str(block.get("code") or "")
- assert "str(search)" not in str(block.get("code") or "")
-
- def _role_name_nav_download_ctx(self) -> CopilotContext:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": "#statement-row",
- "source_url": "https://example.com/billing",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "a",
- "source_url": "https://example.com/billing",
- "role": "link",
- "accessible_name": "View Printable Statement",
- },
- ]
- ctx.reached_download_target = ReachedDownloadTarget(
- selector='a[href="/billing/statement.pdf"]',
- affordance_text="View Printable Statement",
- download_kind="attribute",
- source_step="trajectory_recency",
- already_registered=False,
- )
- return ctx
-
- def test_role_name_bare_nav_click_imposes_with_emitted_get_by_role(self) -> None:
- ctx = self._role_name_nav_download_ctx()
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
-
- assert result.violations == []
- block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
- assert 'await page.get_by_role("link", name="View Printable Statement", exact=True).click()' in block["code"]
- assert "async with page.expect_download()" in block["code"]
- assert "/billing/statement.pdf" in block["code"]
-
- def test_anchorless_bare_nav_click_still_blocks_imposition(self) -> None:
- ctx = self._role_name_nav_download_ctx()
- ctx.scout_trajectory[1].pop("role")
- ctx.scout_trajectory[1].pop("accessible_name")
- workflow_yaml = _yaml(
- """
- title: Statement download
- workflow_definition:
- blocks:
- - block_type: code
- label: download_statement
- code: |
- await page.locator("#statement-row").click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
-
- assert any("ambiguous_bare_selector" in violation for violation in result.violations)
-
- def test_provenance_gate_admits_self_validating_aria_role_name(self) -> None:
- entry = {
- "trajectory_index": 1,
- "selector": "a",
- "emitted_literal": _get_by_role_expr_strict("link", "View Statements"),
- "source": "aria_role_name",
- "role": "link",
- "name": "View Statements",
- }
- assert workflow_update_module._locator_provenance_is_self_validating(entry) is True
-
- def test_provenance_gate_rejects_tampered_aria_role_name(self) -> None:
- tampered_literal = {
- "selector": "a",
- "emitted_literal": 'page.get_by_role("link", name="Spoofed")',
- "source": "aria_role_name",
- "role": "link",
- "name": "View Statements",
- }
- tampered_role = {
- "selector": "a",
- "emitted_literal": _get_by_role_expr_strict("link", "View Statements"),
- "source": "aria_role_name",
- "role": "button",
- "name": "View Statements",
- }
- assert workflow_update_module._locator_provenance_is_self_validating(tampered_literal) is False
- assert workflow_update_module._locator_provenance_is_self_validating(tampered_role) is False
-
- def test_provenance_gate_keeps_selector_byte_equality(self) -> None:
- assert (
- workflow_update_module._locator_provenance_is_self_validating(
- {"selector": "#row", "emitted_literal": "#row", "source": "selector"}
- )
- is True
- )
- assert (
- workflow_update_module._locator_provenance_is_self_validating(
- {"selector": "#row", "emitted_literal": "#other", "source": "selector"}
- )
- is False
- )
- assert (
- workflow_update_module._locator_provenance_is_self_validating(
- {"selector": "#row", "emitted_literal": "#row", "source": "first_fallback"}
- )
- is False
- )
-
- def test_provenance_gate_admits_self_validating_exact_aria_role_name(self) -> None:
- entry = {
- "trajectory_index": 1,
- "selector": "a",
- "emitted_literal": _get_by_role_expr_strict("link", "Download"),
- "source": "aria_role_name",
- "role": "link",
- "name": "Download",
- }
- assert workflow_update_module._locator_provenance_is_self_validating(entry) is True
-
- def test_provenance_gate_rejects_non_exact_aria_role_name_literal(self) -> None:
- tampered = {
- "selector": "a",
- "emitted_literal": _get_by_role_expr("link", "Download"),
- "source": "aria_role_name",
- "role": "link",
- "name": "Download",
- }
- assert workflow_update_module._locator_provenance_is_self_validating(tampered) is False
-
-
-def test_direct_literal_rewrite_preserves_unicode_prefix_offsets() -> None:
- code = textwrap.dedent(
- """
- await page.locator("#café-search-résumé").fill("example_sku_123")
- await page.locator("#naïve-search").type("example_sku_123")
- """
- ).strip()
-
- rewritten, used_keys = workflow_update_module._rewrite_direct_literal_fills(code, {"example_sku_123": "search"})
-
- assert used_keys == ["search"]
- assert (
- rewritten
- == textwrap.dedent(
- """
- await page.locator("#café-search-résumé").fill(str(search))
- await page.locator("#naïve-search").type(str(search))
- """
- ).strip()
- )
-
-
-def test_literal_binding_sees_through_first_last_disambiguator() -> None:
- code = textwrap.dedent(
- """
- await page.locator("input").first.fill("example_sku_123")
- await page.get_by_role("textbox").last.type("example_sku_123")
- """
- ).strip()
-
- assert workflow_update_module._submitted_fill_type_arguments(code) == ["example_sku_123", "example_sku_123"]
-
- rewritten, used_keys = workflow_update_module._rewrite_direct_literal_fills(code, {"example_sku_123": "search"})
- assert used_keys == ["search"]
- assert 'await page.locator("input").first.fill(str(search))' in rewritten
- assert 'await page.get_by_role("textbox").last.type(str(search))' in rewritten
-
-
-def test_direct_parameter_binding_ignores_nested_function_assignment() -> None:
- code = textwrap.dedent(
- """
- def normalize_search() -> str:
- search = "local fallback"
- return search
-
- await page.locator("#search").fill(search)
- """
- ).strip()
-
- assert workflow_update_module._submitted_uses_parameter_in_direct_fill_type(code, "search") is True
-
-
-def test_direct_parameter_binding_rejects_top_level_assignment() -> None:
- code = textwrap.dedent(
- """
- search = "local fallback"
- await page.locator("#search").fill(search)
- """
- ).strip()
-
- assert workflow_update_module._submitted_uses_parameter_in_direct_fill_type(code, "search") is False
-
-
-def test_python_ast_offsets_are_utf8_byte_offsets_for_unicode_source() -> None:
- code = 'await page.locator("#café-search-résumé").fill("example_sku_123")'
- tree = workflow_update_module._wrapped_code_ast(code)
- assert tree is not None
- fill_call = next(
- node
- for node in ast.walk(tree)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Attribute)
- and node.func.attr == "fill"
- and node.args
- )
- literal = fill_call.args[0]
- assert isinstance(literal, ast.Constant)
- assert literal.value == "example_sku_123"
-
- prefix = 'await page.locator("#café-search-résumé").fill('
- assert literal.col_offset == 4 + len(prefix.encode("utf-8"))
- assert literal.col_offset != 4 + len(prefix)
- assert workflow_update_module._AST_COLUMN_OFFSETS_ARE_UTF8_BYTES is True
-
-
-class TestSeamSalvageIntoContext:
- @pytest.mark.asyncio
- async def test_conforming_label_persists_when_sibling_label_rejects(self) -> None:
- workflow_yaml = _yaml(
- """
- title: Registry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: block_one
- code: |
- await page.goto("https://example.com/search")
- records = [{"number": "REC-001"}]
- - block_type: code
- label: block_two
- code: |
- await page.goto("https://example.com/results")
- """
- )
- metadata = [
- _terminal_metadata("block_one", "search the registry"),
- {
- "block_label": "block_two",
- "declared_goal": "expand the result rows",
- "claimed_outcomes": [
- {"id": "claim:rows", "scope": "outcome", "text": "rows expanded", "status": "satisfied"}
- ],
- },
- ]
- ctx = _code_only_ctx()
- result = await _update_workflow({"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata}, ctx)
-
- assert result["ok"] is False
- assert "block_two" in result["error"]
- assert "block_one" not in result["error"]
- assert list(ctx.code_artifact_metadata.keys()) == ["block_one"]
- assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
- assert "contract violation" not in result["user_facing_summary"]
- assert "`" not in result["user_facing_summary"]
-
- @pytest.mark.asyncio
- async def test_stale_entry_dropped_and_draft_metadata_survives_unaccepted_submission(self) -> None:
- draft_yaml = _yaml(
- """
- title: Registry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: block_a
- code: |
- await page.goto("https://example.com/search")
- - block_type: code
- label: block_b
- code: |
- await page.goto("https://example.com/results")
- """
- )
- submitted_yaml = _yaml(
- """
- title: Registry lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: block_a
- code: |
- await page.goto("https://example.com/search")
- """
- )
- ctx = _code_only_ctx()
- ctx.workflow_yaml = draft_yaml
- stored_block_b = _terminal_metadata("block_b", "expand the result rows")
- ctx.code_artifact_metadata = {
- "block_a": _terminal_metadata("block_a", "search the registry"),
- "block_b": stored_block_b,
- }
- ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
- metadata = [
- _terminal_metadata("block_a", "search the registry"),
- _terminal_metadata("ghost", "does not exist"),
- ]
-
- result = await _update_workflow({"workflow_yaml": submitted_yaml, "code_artifact_metadata": metadata}, ctx)
-
- # The stale entry is pruned at the seam; the submission proceeds and
- # only non-metadata gates may reject it, so prior-draft metadata stays.
- error_text = str(result.get("error") or "")
- assert "Artifact metadata" not in error_text
- assert "ghost" not in error_text
- assert sorted(ctx.code_artifact_metadata) == ["block_a", "block_b"]
- assert ctx.code_artifact_metadata["block_b"] == stored_block_b
- assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
-
- @pytest.mark.asyncio
- async def test_minimal_metadata_with_trajectory_produces_no_violation_error(self) -> None:
- ctx = _code_only_ctx()
- metadata = [_terminal_metadata("search_registry", "search the registry")]
- result = await _update_workflow(
- {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
- )
- # The seam may reject later (credential checks need the app); the metadata
- # contract itself must not be the rejection.
- error_text = str(result.get("error") or "")
- assert "Artifact metadata" not in error_text
- assert "contract violation" not in error_text
- assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
-
-
-class TestStaleLabelSeamFlow:
- @pytest.mark.asyncio
- async def test_stale_metadata_label_rekeys_without_any_stale_rejection(self) -> None:
- # Run-2 shape: metadata keyed to a label absent from the submitted
- # YAML. The seam re-keys it to the only uncovered code block, so
- # neither the metadata gate nor the stale-block-metadata validation
- # path can bounce the submission back to the model.
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _SAFE_EXTRACTION_CODE_YAML
- metadata = [_terminal_metadata("search_certificant_stale", "search the registry")]
-
- result = await _update_workflow(
- {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
- )
-
- error_text = str(result.get("error") or "")
- assert "Artifact metadata" not in error_text
- assert "still appears stale" not in error_text
- assert list(ctx.code_artifact_metadata.keys()) == ["search_registry"]
- assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
- # The seam never rewrites YAML labels, so its output cannot trip the
- # stale-block-metadata validation that fires on label/title renames.
- assert _detect_stale_block_metadata(_SAFE_EXTRACTION_CODE_YAML, ctx.workflow_yaml) == []
-
- @pytest.mark.asyncio
- async def test_malformed_per_entry_refs_normalize_without_scout_interactions(self) -> None:
- # Run-3 shape: model-authored observation_refs rows missing the scoped
- # id, authored before any scout interaction was recorded.
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- metadata = [
- {
- **_terminal_metadata("search_registry", "search the registry"),
- "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],
- }
- ]
-
- result = await _update_workflow(
- {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
- )
-
- error_text = str(result.get("error") or "")
- assert "Artifact metadata" not in error_text
- assert "contract violation" not in error_text
- ref = ctx.code_artifact_metadata["search_registry"]["observation_refs"][0]
- assert ref["dependency_id"]
- assert ref["source_tool"]
-
-
-class TestCredentialScoutPersistGate:
- _MULTI_BLOCK_TARGETED_CREDENTIAL_YAML = _yaml(
- """
- title: Saved credential login
- workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: credential_id
- key: login_credential
- default_value: cred_missing
- blocks:
- - block_type: code
- label: enter_username
- parameter_keys: [login_credential]
- code: |
- await page.locator("#email").fill(login_credential.username)
- await page.locator("#continue").click()
- - block_type: code
- label: sign_in_to_business_center
- parameter_keys: [login_credential]
- code: |
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#sign-in").click()
- - block_type: code
- label: open_matching_statement
- code: |
- text = await page.locator("table").inner_text()
- print(text)
- """
- )
- _SUBMIT_CODE_YAML = _credential_code_yaml(
- code="""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(login_credential.totp)
- await page.locator("input[type='submit']").click()
- await page.wait_for_load_state("load")
- """
- )
- _FILL_ONLY_CODE_YAML = _credential_code_yaml(
- code="""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(login_credential.totp)
- """
- )
- _UNSAFE_SUBMIT_CODE_YAML = _credential_code_yaml(
- code="""
- leaked = page.__class__
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("input[type='submit']").click()
- """
- )
- _RUNTIME_OTP_CODE_YAML = _credential_code_yaml(
- code="""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(await login_credential.otp())
- await page.locator("input[type='submit']").click()
- await page.wait_for_load_state("load")
- """
- )
- _DRAFT_DOWNLOAD_CODE_YAML = _credential_code_yaml(
- code="""
- await page.goto("https://example.com/login")
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(await login_credential.otp())
- await page.locator("input[type='submit']").click()
- await page.wait_for_load_state("load")
- async with page.expect_download() as download_info:
- await page.locator("a[href='/invoices/monthly.pdf']").click()
- download = await download_info.value
- print(download.suggested_filename)
- """
- )
- _DOWNLOAD_CODE_YAML = _yaml(
- """
- title: Download invoice
- workflow_definition:
- blocks:
- - block_type: code
- label: download_monthly_invoice_pdf
- code: |
- async with page.expect_download() as download_info:
- await page.locator("a[href='/invoices/monthly.pdf']").click()
- download = await download_info.value
- print(download.suggested_filename)
- """
- )
-
- @pytest.mark.asyncio
- async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
-
- result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- assert "fill_credential_field" in result["error"]
- assert "click the submit control or press Enter" in result["error"]
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
- assert result["user_facing_summary"] == (
- "I need to verify the saved-credential login in the browser before I can save or run this code."
- )
-
- @pytest.mark.asyncio
- async def test_credential_scout_blocker_takes_precedence_over_code_safety(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": self._UNSAFE_SUBMIT_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- assert "fill_credential_field" in result["error"]
- assert "Insecure code detected" not in result["error"]
- assert result["user_facing_summary"] == (
- "I need to verify the saved-credential login in the browser before I can save or run this code."
- )
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- code_safety_diagnostics = result["data"]["diagnostic_code_safety_errors"]
- assert any("private methods or attributes" in error for error in code_safety_diagnostics)
-
- @pytest.mark.asyncio
- async def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- _credential_fill_interaction("username"),
- _credential_fill_interaction("password"),
- _credential_fill_interaction("totp"),
- _submit_interaction(),
- ]
-
- result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- error_text = str(result.get("error") or "")
- assert "was not found in this organization" in error_text
- assert "fill_credential_field" not in error_text
- assert "saved-credential login flow" not in error_text
-
- @pytest.mark.asyncio
- async def test_submit_code_still_requires_later_submit_after_matching_fills(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- _credential_fill_interaction("username"),
- _credential_fill_interaction("password"),
- _credential_fill_interaction("totp"),
- ]
-
- result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- assert "later submit action on the same page" in result["error"]
- assert "click the submit control or press Enter" in result["error"]
-
- @pytest.mark.asyncio
- async def test_fill_only_code_requires_matching_fill_scouts_but_not_submit(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- _credential_fill_interaction("username"),
- _credential_fill_interaction("password"),
- _credential_fill_interaction("totp"),
- ]
-
- result = await _update_workflow({"workflow_yaml": self._FILL_ONLY_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- error_text = str(result.get("error") or "")
- assert "was not found in this organization" in error_text
- assert "click the submit control or press Enter" not in error_text
- assert "saved-credential login flow" not in error_text
-
- @pytest.mark.asyncio
- async def test_runtime_otp_method_does_not_require_impossible_live_otp_fill(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = [
- _credential_fill_interaction("username"),
- _credential_fill_interaction("password"),
- _submit_interaction(),
- ]
-
- result = await _update_workflow({"workflow_yaml": self._RUNTIME_OTP_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- error_text = str(result.get("error") or "")
- assert "was not found in this organization" in error_text
- assert "successful `fill_credential_field` scouting for `totp`" not in error_text
- assert "saved-credential login flow" not in error_text
-
- @pytest.mark.asyncio
- async def test_draft_only_credential_code_download_persists_without_scouts(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _draft_only_credential_ctx()
-
- result = await _update_workflow({"workflow_yaml": self._DRAFT_DOWNLOAD_CODE_YAML}, ctx)
-
- assert result["ok"] is True
- assert "login_with_saved_credential" in ctx.workflow_yaml
- assert "expect_download" in ctx.workflow_yaml
- assert "login_credential.username" in ctx.workflow_yaml
-
- @pytest.mark.asyncio
- async def test_download_scout_gate_still_blocks_normal_code_only_authoring(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": self._DOWNLOAD_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- assert "Scout it first" in result["error"]
- assert "download affordance" in result["error"]
-
- @pytest.mark.asyncio
- async def test_standard_mode_behavior_is_unchanged(self) -> None:
- ctx = _standard_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
-
- assert result["ok"] is False
- error_text = str(result.get("error") or "")
- assert "fill_credential_field" not in error_text
- assert "saved-credential login flow" not in error_text
-
- @pytest.mark.asyncio
- async def test_targeted_run_labels_scope_credential_scout_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- accepted = await _update_workflow(
- {
- "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
- "block_labels": ["open_matching_statement"],
- },
- ctx,
- allow_missing_credentials=True,
- )
-
- assert accepted["ok"] is True
-
- selected_credential_ctx = _code_only_ctx()
- selected_credential_ctx.scout_trajectory = []
- rejected = await _update_workflow(
- {
- "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
- "block_labels": ["sign_in_to_business_center"],
- },
- selected_credential_ctx,
- )
-
- assert rejected["ok"] is False
- assert rejected["data"]["failure_type"] == "missing_credential_or_init"
- assert "sign_in_to_business_center" in rejected["error"]
- assert rejected["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
-
- @pytest.mark.asyncio
- async def test_persisted_parameter_shape_does_not_rescout_unchanged_selected_credential_block(self) -> None:
- sign_in_code = """
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#sign-in").click()
- """
- sign_in_code = textwrap.dedent(sign_in_code).strip()
- login_parameter = {
- "parameter_type": "workflow",
- "workflow_parameter_type": "credential_id",
- "key": "login_credential",
- "default_value": "cred_missing",
- }
- prior_yaml = yaml.safe_dump(
- {
- "title": "Saved credential login",
- "workflow_definition": {
- "parameters": [login_parameter],
- "blocks": [
- {
- "block_type": "code",
- "label": "sign_in_to_business_center",
- "parameters": [login_parameter],
- "code": sign_in_code,
- }
- ],
- },
- },
- sort_keys=False,
- )
- submitted_yaml = yaml.safe_dump(
- {
- "title": "Saved credential login",
- "workflow_definition": {
- "parameters": [
- login_parameter,
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": "account_number",
- "default_value": "100245",
- },
- ],
- "blocks": [
- {
- "block_type": "code",
- "label": "sign_in_to_business_center",
- "parameter_keys": ["login_credential"],
- "code": sign_in_code,
- },
- {
- "block_type": "code",
- "label": "open_matching_statement",
- "parameter_keys": ["account_number"],
- "code": 'await page.get_by_text("View Printable Statement").wait_for(timeout=5000)',
- },
- ],
- },
- },
- sort_keys=False,
- )
- ctx = _code_only_ctx()
- ctx.workflow_yaml = prior_yaml
- ctx.scout_trajectory = []
-
- result = await _update_workflow(
- {
- "workflow_yaml": submitted_yaml,
- "block_labels": ["sign_in_to_business_center", "open_matching_statement"],
- },
- ctx,
- )
-
- assert result["ok"] is False
- error_text = str(result.get("error") or "")
- assert "open_matching_statement" in error_text
- assert "fill_credential_field" not in error_text
-
- changed_yaml = submitted_yaml.replace(
- 'await page.locator("#sign-in").click()',
- 'await page.locator("#sign-in").click()\n await page.locator("#post-login").click()',
- )
- changed_ctx = _code_only_ctx()
- changed_ctx.workflow_yaml = prior_yaml
- changed_ctx.scout_trajectory = []
-
- changed_result = await _update_workflow(
- {
- "workflow_yaml": changed_yaml,
- "block_labels": ["sign_in_to_business_center"],
- },
- changed_ctx,
- )
-
- assert changed_result["ok"] is False
- assert changed_result["data"]["failure_type"] == "missing_credential_or_init"
- assert changed_result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
-
-
-def test_run_id_leak_check_covers_non_numeric_ids() -> None:
- with pytest.raises(ValueError):
- assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")
-
-
-class TestStripRedundantSandboxImports:
- @pytest.mark.parametrize(
- ("code", "expected_module"),
- [
- ("import asyncio\nawait page.goto('https://example.com')", "asyncio"),
- ("import asyncio\nawait asyncio.sleep(1)", "asyncio"),
- ("import json\nvalue = json.dumps({})", "json"),
- ("import json\nvalue = json.loads('{}')", "json"),
- ("import re\nmatch = re.search(r'x', 'x')", "re"),
- ("import html\nvalue = html.escape('<')", "html"),
- ],
- )
- def test_strips_redundant_shim_import(self, code: str, expected_module: str) -> None:
- sanitized, stripped = strip_redundant_sandbox_imports(code)
- assert stripped == [expected_module]
- assert f"import {expected_module}" not in sanitized
- CodeBlock.is_safe_code(sanitized)
-
- @pytest.mark.parametrize(
- "code",
- [
- "import asyncio\nawait asyncio.gather(page.goto('https://example.com'))",
- "import json\ntry:\n json.loads('x')\nexcept json.JSONDecodeError:\n pass",
- "import html\nvalue = html.unescape('&amp;')",
- "import re\nvalue = re.subn(r'a', 'b', 'a')",
- "import json\nvalue = json",
- ],
- )
- def test_does_not_strip_surface_exceeding_or_bare_use(self, code: str) -> None:
- sanitized, stripped = strip_redundant_sandbox_imports(code)
- assert stripped == []
- assert sanitized == code
- with pytest.raises(InsecureCodeDetected):
- CodeBlock.is_safe_code(sanitized)
-
- @pytest.mark.parametrize(
- "code",
- [
- "import os as json\nvalue = json",
- "import json.decoder\nvalue = 1",
- "from re import search\nmatch = search(r'x', 'x')",
- "import json; value = json.dumps({})",
- "import requests\nvalue = requests",
- "import os\nvalue = 1",
- 'import re; import os\nresult = os.environ.get("AWS_SECRET_ACCESS_KEY")',
- 'import os; import re\nresult = os.environ.get("AWS_SECRET_ACCESS_KEY")',
- 'import json; import requests\nresult = requests.get("https://example.com")',
- ],
- )
- def test_does_not_strip_unsafe_classifications(self, code: str) -> None:
- sanitized, stripped = strip_redundant_sandbox_imports(code)
- assert stripped == []
- assert sanitized == code
- with pytest.raises(InsecureCodeDetected):
- CodeBlock.is_safe_code(sanitized)
-
- def test_preserves_surrounding_comments(self) -> None:
- code = "import asyncio # drop me\n# keep this comment\nawait asyncio.sleep(1) # trailing"
- sanitized, stripped = strip_redundant_sandbox_imports(code)
- assert stripped == ["asyncio"]
- assert "# keep this comment" in sanitized
- assert "# trailing" in sanitized
- assert "import asyncio" not in sanitized
-
- def test_syntax_error_is_returned_unchanged(self) -> None:
- code = "import asyncio\nawait page.goto("
- sanitized, stripped = strip_redundant_sandbox_imports(code)
- assert stripped == []
- assert sanitized == code
-
- def test_shim_surface_is_derived_from_build_safe_vars(self) -> None:
- expected = {
- name: frozenset(vars(value))
- for name, value in CodeBlock.build_safe_vars().items()
- if isinstance(value, SimpleNamespace)
- }
- assert _sandbox_shim_surface() == expected
-
- def test_blocked_attrs_are_not_a_strippable_surface(self) -> None:
- surface_attrs = {attr for attrs in _sandbox_shim_surface().values() for attr in attrs}
- assert surface_attrs.isdisjoint(CodeBlock.BLOCKED_ATTRS)
-
-
-class TestStripRedundantSandboxImportsInYaml:
- def test_malformed_yaml_is_returned_unchanged(self) -> None:
- malformed = "title: [unterminated\n"
- sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(malformed)
- assert stripped == []
- assert sanitized == malformed
-
- def test_non_workflow_yaml_is_returned_unchanged(self) -> None:
- non_workflow = "just: a mapping\n"
- sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(non_workflow)
- assert stripped == []
- assert sanitized == non_workflow
-
- def test_multi_block_strips_per_block(self) -> None:
- multi_block = _yaml(
- """
- title: Multi
- workflow_definition:
- blocks:
- - block_type: code
- label: first
- code: |
- import asyncio
- await asyncio.sleep(1)
- - block_type: code
- label: second
- code: |
- await page.goto("https://example.com")
- - block_type: code
- label: third
- code: |
- import json
- value = json.dumps({})
- """
- )
- sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(multi_block)
- assert sorted(stripped) == ["asyncio", "json"]
- assert "import asyncio" not in sanitized
- assert "import json" not in sanitized
-
- def test_no_change_returns_original_text(self) -> None:
- sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(_SAFE_CODE_YAML)
- assert stripped == []
- assert sanitized == _SAFE_CODE_YAML
-
-
-def _distinct_guardrail_yaml(index: int) -> str:
- bodies = [
- f"value = undefined_helper_{index}()",
- f'import os\nawait page.goto("https://example.com/{index}")',
- f'await page.evaluate("{index} + 1")',
- ]
- return _code_yaml(bodies[index % len(bodies)])
-
-
-def _distinct_credential_collision_yaml(index: int) -> str:
- unsafe = [
- f"value = undefined_helper_{index}()",
- "import os",
- f'await page.evaluate("{index} + 1")',
- ]
- return _credential_code_yaml(
- code=f"""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- {unsafe[index % len(unsafe)]}
- """
- )
-
-
-def _page_evaluate_credential_collision_yaml(index: int) -> str:
- return _distinct_credential_collision_yaml((index * 3) + 2)
-
-
-def _safe_credential_collision_yaml(index: int) -> str:
- return _credential_code_yaml(
- code=f"""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- landing_url_{index} = "https://example.com/portal/{index}"
- """
- )
-
-
-def _terminal_challenge_signal() -> CopilotToolBlockerSignal:
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text="A site verification challenge blocked the run.",
- user_facing_reason="The site's verification challenge blocked the run.",
- recovery_hint="report_blocker_to_user",
- internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
- blocked_tool="update_and_run_blocks",
- )
-
-
-class TestCodeAuthoringGuardrailChurnBackstop:
- @pytest.mark.asyncio
- async def test_counter_climbs_on_repeated_recorded_outcome_and_resets_on_accept(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _code_only_ctx()
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
- result = await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- assert result["ok"] is False
- assert ctx.code_authoring_guardrail_reject_count == index + 1
- assert ctx.blocker_signal is None
-
- _stub_successful_update(monkeypatch)
- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
- assert accepted["ok"] is True
- assert ctx.code_authoring_guardrail_reject_count == 0
-
- @pytest.mark.asyncio
- async def test_accepted_persist_at_churn_ceiling_resets_counter_and_clears_held_signals(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _code_only_ctx()
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- held = ctx.blocker_signal
- assert isinstance(held, CopilotToolBlockerSignal)
- assert held.internal_reason_code == "code_authoring_guardrail_churn"
- assert ctx.latest_tool_blocker_signal is held
-
- _stub_successful_update(monkeypatch)
- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
-
- assert accepted["ok"] is True
- assert ctx.code_authoring_guardrail_reject_count == 0
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_counter_climbs_through_credential_scout_branch(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- unsafe_credential_yaml = _credential_code_yaml(
- code="""
- import os
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("input[type='submit']").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": unsafe_credential_yaml}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert ctx.code_authoring_guardrail_reject_count == 1
-
- @pytest.mark.asyncio
- async def test_clean_accept_does_not_climb_counter(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
-
- result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
-
- assert result["ok"] is True
- assert ctx.code_authoring_guardrail_reject_count == 0
-
- @pytest.mark.asyncio
- async def test_output_policy_reject_records_author_time_reject_and_guardrail_count(self) -> None:
- ctx = _code_only_ctx()
-
- result = await _update_workflow(
- {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is False
- assert "raw_secret_leak" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert outcome is not None
- assert outcome.phase == "author_time_reject"
- assert outcome.reason_code == "output_policy_reject"
- assert outcome.is_authoritative is True
- assert outcome.structural_key is not None
- assert "Output policy blocked" in outcome.observed_evidence_summary
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_repeated_output_policy_reject_halts_at_guardrail_backstop(self) -> None:
- ctx = _code_only_ctx()
-
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- result = await _update_workflow(
- {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},
- ctx,
- allow_missing_credentials=True,
- )
- assert result["ok"] is False
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
- assert ctx.latest_tool_blocker_signal is churn
- assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
-
- @pytest.mark.asyncio
- async def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:
- ctx = _code_only_ctx()
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
- assert ctx.latest_tool_blocker_signal is churn
- assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
- assert "safety checks rejected" in churn.user_facing_reason
-
- @pytest.mark.asyncio
- async def test_nth_reject_defers_to_pre_existing_terminal_blocker(self) -> None:
- ctx = _code_only_ctx()
- terminal = _terminal_challenge_signal()
- ctx.blocker_signal = terminal
- ctx.latest_tool_blocker_signal = terminal
-
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- assert ctx.blocker_signal is terminal
- assert ctx.latest_tool_blocker_signal is terminal
-
- def test_churn_stop_defers_while_output_contract_ladder_active_then_fires(self) -> None:
- ctx = _code_only_ctx()
- ctx.output_contract_actuation_count_by_signature["sig_active"] = 1
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- assert ctx.blocker_signal is None
- ctx.output_contract_actuation_count_by_signature.clear()
- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
-
- def test_churn_stop_defers_while_advisory_grant_held(self) -> None:
- ctx = _code_only_ctx()
- ctx.output_contract_actuation_by_signature["sig_granted"] = OutputContractAdvisoryState.GRANTED
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
- assert ctx.blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_mixed_credential_and_unresolved_name_reject_returns_code_repair_progress(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- info_calls: list[tuple[str, dict[str, str | list[str]]]] = []
-
- def capture_info(event: str, **kwargs: str | list[str]) -> None:
- info_calls.append((event, kwargs))
-
- monkeypatch.setattr(workflow_update_module.LOG, "info", capture_info)
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert result["ok"] is False
- assert result["data"]["surface_kind"] == "code_repair_progress"
- assert "failure_type" not in result["data"]
- repair_context = result["data"]["authoring_repair_context"]
- assert repair_context["reason_code"] == SANDBOX_UNRESOLVED_NAME_REASON_CODE
- assert repair_context["block_label"] == "login_with_saved_credential"
- assert repair_context["unresolved_names"] == ["undefined_helper_0"]
- assert result["user_facing_summary"] != CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.last_code_authoring_reject_was_credential_priority is False
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
- assert (
- "copilot code authoring repair context stored",
- {
- "reason_code": SANDBOX_UNRESOLVED_NAME_REASON_CODE,
- "block_label": "login_with_saved_credential",
- "unresolved_names": ["undefined_helper_0"],
- "parameter_keys": [],
- "available_parameter_keys": [],
- "binding_candidates": ["undefined_helper_0"],
- },
- ) in info_calls
-
- @pytest.mark.asyncio
- async def test_single_credential_priority_reject_defers_to_credential_scout_reply(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_mixed_credential_and_non_name_guardrail_uses_credential_priority_path(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.last_code_authoring_repair_context is None
- assert ctx.last_code_authoring_reject_was_credential_priority is True
-
- @pytest.mark.asyncio
- async def test_standard_policy_mixed_credential_and_unresolved_name_omits_repair_context(self) -> None:
- ctx = _standard_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
-
- assert result["ok"] is False
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
-
- @pytest.mark.asyncio
- async def test_credential_priority_reject_defers_below_higher_bound(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):
- await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
- assert _check_enforcement(ctx) is None
-
- @pytest.mark.asyncio
- async def test_credential_priority_churn_stashes_credential_blocker_at_higher_bound(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result: dict[str, object] = {}
- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
- result = await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "credential_priority_authoring_churn"
- assert ctx.latest_tool_blocker_signal is churn
- assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
-
@pytest.mark.asyncio
- async def test_credential_priority_churn_renders_credential_scout_reply_through_enforcement(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
- await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
-
- with pytest.raises(CopilotTurnHalt) as excinfo:
- _check_enforcement(ctx)
-
- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "credential_priority_authoring_churn"
- assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY
-
- @pytest.mark.asyncio
- async def test_single_pure_credential_reject_defers_to_credential_scout_reply(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert "diagnostic_code_safety_errors" not in result["data"]
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_unchanged_persisted_credential_block_does_not_require_new_scout(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_unchanged_prior_code_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
- prior_yaml = _safe_credential_collision_yaml(0)
- ctx = _code_only_ctx()
- ctx.workflow_yaml = prior_yaml
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": prior_yaml}, ctx, allow_missing_credentials=True)
-
- assert result["ok"] is True
- assert result.get("user_facing_summary") != CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.code_authoring_guardrail_reject_count == 0
-
- @pytest.mark.asyncio
- async def test_changed_persisted_credential_block_still_requires_new_scout(self) -> None:
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _safe_credential_collision_yaml(0)
- ctx.scout_trajectory = []
-
- result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(1)}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
-
- @pytest.mark.asyncio
- async def test_pure_credential_priority_churn_climbs_and_halts(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- result: dict[str, object] = {}
- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):
- result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(index)}, ctx)
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert "diagnostic_code_safety_errors" not in result["data"]
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- assert ctx.code_authoring_guardrail_reject_count == index + 1
-
- assert ctx.blocker_signal is None
- assert _check_enforcement(ctx) is None
-
- result = await _update_workflow(
- {"workflow_yaml": _safe_credential_collision_yaml(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1)}, ctx
- )
-
- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "credential_priority_authoring_churn"
- with pytest.raises(CopilotTurnHalt) as excinfo:
- _check_enforcement(ctx)
- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
- assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY
-
- @pytest.mark.asyncio
- async def test_name_safety_churn_still_halts_after_credential_priority_reject(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
- await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
- assert ctx.last_code_authoring_reject_was_credential_priority is True
-
+ ctx = self._provider_search_ctx()
ctx.scout_trajectory = [
{
- "tool_name": "click",
- "selector": "#search-submit",
- "source_url": "https://example.com/search",
+ "tool_name": "type_text",
+ "selector": "input[placeholder='Search']",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 13,
+ "role": "textbox",
+ "accessible_name": "Search",
"trajectory_index": 0,
- }
- ]
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.last_code_authoring_reject_was_credential_priority is False
- assert ctx.blocker_signal is None
- assert _check_enforcement(ctx) is None
-
- @pytest.mark.asyncio
- async def test_non_credential_reject_resets_when_recorded_outcome_changes_after_precharge(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
-
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS + 1):
- await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(index)}, ctx)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS + 1
- assert ctx.last_code_authoring_reject_was_credential_priority is True
- assert ctx.blocker_signal is None
- assert _check_enforcement(ctx) is None
-
- ctx.scout_trajectory = [
+ },
{
"tool_name": "click",
- "selector": "#search-submit",
- "source_url": "https://example.com/search",
- "trajectory_index": 0,
- }
+ "selector": "button.lookup",
+ "source_url": "https://example.com/find-care",
+ "role": "button",
+ "accessible_name": "Lookup",
+ "trajectory_index": 1,
+ },
]
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
-
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.last_code_authoring_reject_was_credential_priority is False
- assert ctx.blocker_signal is None
- assert _check_enforcement(ctx) is None
-
- @pytest.mark.asyncio
- async def test_changed_authoring_recorded_outcome_resets_guardrail_backstop(self) -> None:
- ctx = _code_only_ctx()
-
- await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)
- first_key = ctx.latest_recorded_build_test_outcome.structural_key
- await _update_workflow({"workflow_yaml": _code_yaml("value = missing_second_name()")}, ctx)
-
- assert first_key is not None
- assert ctx.latest_recorded_build_test_outcome.structural_key != first_key
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert ctx.blocker_signal is None
+ ctx.workflow_yaml = _SUBMITTED_LITERAL_YAML
- @pytest.mark.asyncio
- async def test_identical_authoring_recorded_outcome_still_halts_at_backstop(self) -> None:
- ctx = _code_only_ctx()
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ assert result["ok"] is True
+ assert "imposed_substitutions" not in result["data"]
+ assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
@pytest.mark.asyncio
- async def test_accept_after_latched_churn_clears_both_signals(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _code_only_ctx()
- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
- assert ctx.latest_tool_blocker_signal is churn
-
+ async def test_flag_off_code_only_mode_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
-
- assert accepted["ok"] is True
- assert ctx.code_authoring_guardrail_reject_count == 0
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
- assert _check_enforcement(ctx) is None
-
-
-_RESALE_URL = "https://example.com/orders"
-_QUOTE_URL = "https://example.com/quote"
-
-
-def _resale_ctx(*, refiner_selector: str = 'button[data-action="status"]') -> CopilotContext:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#order-id",
- "source_url": _RESALE_URL,
- "typed_length": 6,
- "typed_value": "abc123",
- "role": "textbox",
- "accessible_name": "Order ID",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "button",
- "source_url": _RESALE_URL,
- "trajectory_index": 1,
- },
- {
- "tool_name": "click",
- "selector": refiner_selector,
- "source_url": _RESALE_URL,
- "trajectory_index": 2,
- },
- ]
- return ctx
-
-
-def _resale_submitted_yaml(refiner_selector: str = 'button[data-action="status"]') -> str:
- escaped = refiner_selector.replace('"', '\\"')
- return _yaml(
- f"""
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#order-id").fill(str(order_id))
- await page.locator("{escaped}").click()
- """
- )
-
-
-def _quote_ctx() -> CopilotContext:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#zip",
- "source_url": _QUOTE_URL,
- "typed_length": 5,
- "typed_value": "02110",
- "role": "textbox",
- "accessible_name": "ZIP code",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "#continue",
- "source_url": _QUOTE_URL,
- "trajectory_index": 1,
- },
- {
- "tool_name": "click",
- "selector": "#coverage-next",
- "source_url": "https://example.com/quote/coverage",
- "trajectory_index": 2,
- },
- ]
- return ctx
-
-
-def _author_time_reject_outcome(reason_code: BuildTestOutcomeReasonCode) -> RecordedBuildTestOutcome:
- return recorded_outcome_from_author_time_reject(
- reason_code=reason_code,
- attempted_block_label=workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
- block_labels=[workflow_update_module._SYNTHESIZED_BLOCK_LABEL],
- structural_payload={
- "reason_code": reason_code,
- "block_label": workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
- },
- observed_evidence_summary=reason_code,
- )
-
-
-def _code_blocks(parsed: dict[str, object]) -> dict[str, dict[str, object]]:
- blocks = [block for block in workflow_blocks(parsed) if str(block.get("block_type") or "").lower() == "code"]
- return {str(block.get("label") or ""): block for block in blocks}
-
-
-def _submitted_with_sibling_code(sibling_code: str) -> str:
- indented = textwrap.indent(textwrap.dedent(sibling_code).strip(), " " * 14)
- return _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- - block_type: code
- label: preserved_code
- code: |
-{indented}
- """
- )
-
-
-_METADATA_LESS_DIRECTIVE_EVENT = "copilot_output_contract_spine_structure_directive_emitted"
-
-_RECORDED_METADATA_REJECT_BLOCK_LABEL = "validate_public_start_service_path"
-
-
-def _recorded_metadata_reject_yaml() -> str:
- return _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- parameters:
- - {{parameter_type: output, key: workflow_output}}
- blocks:
- - block_type: code
- label: {_RECORDED_METADATA_REJECT_BLOCK_LABEL}
- code: |
- await page.locator("#start").click()
- """
- )
-
-
-def _metadata_less_output_yaml(selector: str = "#search-submit") -> str:
- return _yaml(
- f"""
- title: Provider lookup
- workflow_definition:
- parameters:
- - {{parameter_type: output, key: workflow_output}}
- blocks:
- - block_type: code
- label: extract_provider
- code: |
- await page.locator("{selector}").click()
- """
- )
-
-
-def _metadata_less_page_read_yaml() -> str:
- return _yaml(
- """
- title: Provider lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: workflow_output}
- blocks:
- - block_type: code
- label: extract_provider
- code: |
- await page.locator("#search-submit").click()
- value = await page.locator("#npi").inner_text()
- return {"npi": value}
- """
- )
-
-
-def _metadata_less_two_block_yaml() -> str:
- return _yaml(
- """
- title: Provider lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: workflow_output}
- blocks:
- - block_type: code
- label: first_block
- description: extract the provider data
- code: |
- await page.locator("#a").click()
- - block_type: code
- label: second_block
- description: extract the license data
- code: |
- await page.locator("#b").click()
- """
- )
+ ctx = self._provider_search_ctx()
+ ctx.impose_synthesized_code_block = False
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
-class TestMetadataLessRejectArming:
- @pytest.mark.asyncio
- async def test_metadata_less_reject_arms_rung_one_directive(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": _metadata_less_output_yaml()}, ctx, allow_missing_credentials=True
- )
- assert result["ok"] is False
- directive = result["data"]["metadata_convergence_directive"]
- assert directive["rung"] == 1
- assert directive["missing_fields_by_label"]["extract_provider"]
- assert "declared_goal" in directive["missing_fields_by_label"]["extract_provider"]
- assert [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len([log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]) == 1
- assert not ctx.output_contract_spine_directive_blockers_by_attempt_key
- assert not ctx.output_contract_spine_directive_stage_count_by_attempt_key
- assert not ctx.output_contract_actuation_by_signature
- assert not ctx.output_contract_declick_attempted_by_signature
- assert not ctx.output_contract_actuation_count_by_signature
+ assert result["ok"] is True
+ assert "imposed_substitutions" not in result["data"]
+ assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
@pytest.mark.asyncio
- async def test_identical_repeat_escalates_to_rung_two_skeleton(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_flag_off_code_only_mode_does_not_promote_scouted_typed_defaults(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- with capture_logs() as logs:
- second = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- directive = second["data"]["metadata_convergence_directive"]
- assert directive["rung"] == 2
- skeleton = directive["metadata_fill_in_skeleton"]["extract_provider"]
- assert skeleton["declared_goal"] == ""
- assert skeleton["claimed_outcomes"] == []
- assert "evidence_refs_or_observation_refs" not in skeleton
- assert skeleton["evidence_refs"] == []
- assert skeleton["observation_refs"] == []
- rung_two_emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(rung_two_emits) == 1 and rung_two_emits[0]["rung"] == 2
+ ctx = self._typed_default_ctx()
+ ctx.impose_synthesized_code_block = False
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_TYPED_LITERAL_REWRITE_YAML}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == _SUBMITTED_TYPED_LITERAL_REWRITE_YAML
@pytest.mark.asyncio
- async def test_third_same_key_is_typed_terminal_without_recording_third_reject(
+ async def test_single_direct_synthesized_parameter_reference_creates_required_input(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- first = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- record_build_test_outcome(
- ctx,
- RecordedBuildTestOutcome(
- phase="scout_evaluate",
- attempted_tool="evaluate",
- verdict="progress_observed",
- reason_code="loaded_result_targets_observed",
- structural_failure_identity="scout:interleaved",
- ),
- )
- second = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- metadata_rejects_before_terminal = sum(
- row.get("reason_code") == "metadata_reject" for row in ctx.recorded_build_test_outcome_history
- )
-
- with capture_logs() as logs:
- third = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
+ ctx = self._provider_search_ctx()
- assert first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert second["data"]["metadata_convergence_directive"]["rung"] == 2
- assert third["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert third["data"]["gate_id"] == "code_artifact_metadata"
- assert (
- third["data"]["missing_fields_by_label"]
- == second["data"]["metadata_convergence_directive"]["missing_fields_by_label"]
- )
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
- assert (
- sum(row.get("reason_code") == "metadata_reject" for row in ctx.recorded_build_test_outcome_history)
- == metadata_rejects_before_terminal
- )
- terminal_logs = [log for log in logs if log["event"] == "copilot_metadata_reject_same_key_terminal_selected"]
- assert len(terminal_logs) == 1
- assert terminal_logs[0]["missing_fields_by_label"] == third["data"]["missing_fields_by_label"]
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_LITERAL_YAML}, ctx)
- @pytest.mark.asyncio
- async def test_update_and_run_blocks_native_preflight_terminals_on_third_same_key(self) -> None:
- ctx = _code_only_ctx()
- tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
- arguments = json.dumps(
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
{
- "workflow_yaml": _metadata_less_output_yaml(),
- "block_labels": ["extract_provider"],
- "code_artifact_metadata": [],
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
}
- )
+ ]
+ expected_selector = ctx.scout_trajectory[0]["selector"]
+ assert f'await page.locator("{expected_selector}").fill(str(provider_name))' in block["code"]
- first = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
- second = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
- third = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+ @pytest.mark.asyncio
+ async def test_repeated_direct_synthesized_parameter_reference_creates_required_input(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
- assert first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert second["data"]["metadata_convergence_directive"]["rung"] == 2
- assert "code_artifact_metadata" in third["error"]
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
- assert ctx.turn_halt.blocker_signal is not None
- assert ctx.turn_halt.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert (
- ctx.turn_halt.blocker_signal.extra["missing_fields_by_label"]
- == second["data"]["metadata_convergence_directive"]["missing_fields_by_label"]
- )
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_REPEATED_COMPUTED_LITERAL_YAML}, ctx)
- @pytest.mark.asyncio
- async def test_update_and_run_blocks_metadata_ladder_defers_run_backed_repair_ceiling(self) -> None:
- ctx = _code_only_ctx()
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
- ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
- tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
- arguments = json.dumps(
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
{
- "workflow_yaml": _metadata_less_output_yaml(),
- "block_labels": ["extract_provider"],
- "code_artifact_metadata": [],
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
}
- )
-
- first = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
- second = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
- third = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
-
- assert first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert second["data"]["metadata_convergence_directive"]["rung"] == 2
- assert third["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
+ ]
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
@pytest.mark.asyncio
- async def test_metadata_ladder_does_not_defer_ceiling_for_unrelated_latest_repair(self) -> None:
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- for rung in (1, 2):
- rejected = await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
- )
- assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
- assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=8) is True
-
- record_build_test_outcome(
- ctx,
- RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="run_blocks",
- verdict="repairable_failure",
- reason_code="runtime_block_failure",
- structural_failure_identity="runtime:unrelated",
- ),
- )
-
- assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=9) is False
+ async def test_mixed_non_locator_fill_does_not_hide_direct_synthesized_parameter_reference(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
- def test_metadata_ladder_does_not_defer_ceiling_for_legacy_context(self) -> None:
- legacy_ctx: Any = SimpleNamespace()
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_FILL_COMPUTED_LITERAL_YAML}, ctx)
- assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(legacy_ctx, count=8) is False
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_name"]
+ expected_selector = ctx.scout_trajectory[0]["selector"]
+ assert f'await page.locator("{expected_selector}").fill(str(provider_name))' in block["code"]
@pytest.mark.asyncio
- async def test_changed_metadata_key_reenables_run_backed_repair_ceiling(self) -> None:
- ctx = _code_only_ctx()
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
- ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
- tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
-
- first = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- tool_ctx,
- json.dumps(
- {
- "workflow_yaml": _metadata_less_output_yaml(),
- "block_labels": ["extract_provider"],
- "code_artifact_metadata": [],
- }
- ),
- )
- )
- changed = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- tool_ctx,
- json.dumps(
- {
- "workflow_yaml": _metadata_less_page_read_yaml(),
- "block_labels": ["extract_provider"],
- "code_artifact_metadata": [],
- }
- ),
- )
+ async def test_selector_join_aliases_synthesized_parameter_to_authored_fill_parameter(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ submitted = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: search_term
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("#provInput").fill(search_term)
+ """
)
- assert first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert changed["data"]["metadata_convergence_directive"]["rung"] == 1
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["search_term"]
+ assert 'await page.locator("#provInput").fill(str(search_term))' in block["code"]
+ assert "provider_name" not in block["code"]
@pytest.mark.asyncio
- async def test_accepted_persist_resets_metadata_ladder_and_generic_churn(
+ async def test_synthesized_internal_parameter_aliases_to_existing_declared_default(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- rejected = await _update_workflow(
- {"workflow_yaml": _metadata_less_output_yaml()},
- ctx,
- allow_missing_credentials=True,
- )
- assert rejected["data"]["metadata_convergence_directive"]["rung"] == 1
- assert ctx.metadata_reject_ladder_state is not None
- assert ctx.code_authoring_guardrail_reject_count == 1
-
- accepted = await _update_workflow(
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
{
- "workflow_yaml": _SAFE_EXTRACTION_CODE_YAML,
- "code_artifact_metadata": [_terminal_metadata("search_registry", "search the registry")],
- },
- ctx,
- allow_missing_credentials=True,
+ "tool_name": "type_text",
+ "selector": "#locInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 17,
+ "typed_value": "Example City, USA",
+ "role": "textbox",
+ "accessible_name": "Address or postal code",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ parameters:
+ - {key: search_location, default_value: "Example City, USA"}
+ - {key: address_or_postal_code, default_value: "Example City, USA"}
+ blocks:
+ - block_type: code
+ label: search_directory
+ parameter_keys: [address_or_postal_code]
+ parameters:
+ - {key: address_or_postal_code, default_value: "Example City, USA"}
+ code: |
+ await page.locator("#location").fill(str(address_or_postal_code))
+ """
)
- assert accepted["ok"] is True
- assert ctx.metadata_reject_ladder_state is None
- assert ctx.code_authoring_guardrail_reject_count == 0
- restarted = await _update_workflow(
- {"workflow_yaml": _metadata_less_output_yaml()},
- ctx,
- allow_missing_credentials=True,
- )
- assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- @pytest.mark.asyncio
- async def test_structural_progress_resets_to_rung_one(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- click_yaml = _metadata_less_output_yaml()
- await _update_workflow({"workflow_yaml": click_yaml}, ctx, allow_missing_credentials=True)
- repeat = await _update_workflow({"workflow_yaml": click_yaml}, ctx, allow_missing_credentials=True)
- assert repeat["data"]["metadata_convergence_directive"]["rung"] == 2
- progressed = await _update_workflow(
- {"workflow_yaml": _metadata_less_page_read_yaml()}, ctx, allow_missing_credentials=True
- )
- assert progressed["ok"] is False
- assert progressed["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["search_location"]
+ assert "parameters" not in block
+ assert "str(search_location)" in block["code"]
+ assert "address_or_postal_code" not in block["code"]
+ parameters = parsed["workflow_definition"]["parameters"]
+ assert [parameter["key"] for parameter in parameters] == ["search_location"]
+ assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
+ "address_or_postal_code": "search_location"
+ }
@pytest.mark.asyncio
- async def test_intervening_normalization_reject_breaks_missing_metadata_streak(
+ async def test_multi_input_synthesized_parameter_aliases_before_ambiguity_guard(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- for rung in (1, 2):
- rejected = await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
- )
- assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#locInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 17,
+ "typed_value": "Example City, USA",
+ "role": "textbox",
+ "accessible_name": "Address or postal code",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "type_text",
+ "selector": "#firstName",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 5,
+ "typed_value": "Given",
+ "role": "textbox",
+ "accessible_name": "Provider First Name",
+ "trajectory_index": 1,
+ },
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ parameters:
+ - {key: search_location, default_value: "Example City, USA"}
+ - {key: provider_first_name, default_value: "Given"}
+ blocks:
+ - block_type: code
+ label: search_directory
+ parameter_keys: [address_or_postal_code, provider_first_name]
+ code: |
+ await page.locator("#location").fill(str(address_or_postal_code))
+ await page.locator("#firstName").fill(str(provider_first_name))
+ """
+ )
- malformed_metadata = {
- "block_label": "extract_provider",
- "artifact_id": "code_artifact:extract_provider",
- "declared_goal": "extract the provider",
- "claimed_outcomes": [
- {
- "id": "claim:provider",
- "scope": "outcome",
- "text": "provider extracted",
- "status": "satisfied",
- "depends_on": ["dependency:page"],
- "covered_criteria": ["criterion:provider"],
- "goal_value_paths": ["provider"],
- }
- ],
- "page_dependencies": [{"id": "dependency:page", "scope": "page", "status": "satisfied"}],
- "completion_criteria": [{"id": "criterion:provider", "text": "provider extracted", "level": "terminal"}],
- "terminal_verifier_expectations": [
- {
- "id": "expectation:provider",
- "text": "provider extracted",
- "criteria_ids": ["criterion:provider"],
- "goal_value_paths": ["provider"],
- }
- ],
- "observation_refs": [
- {
- "observation_ref": "obs:page",
- "dependency_id": "dependency:page",
- "status": "satisfied",
- "source_tool": "scout_interaction",
- }
- ],
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["search_location", "provider_first_name"]
+ assert "str(search_location)" in block["code"]
+ assert "str(provider_first_name)" in block["code"]
+ assert "address_or_postal_code" not in block["code"]
+ assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
+ "address_or_postal_code": "search_location"
}
- intervening = await _update_workflow(
- {"workflow_yaml": yaml_text, "code_artifact_metadata": [malformed_metadata]},
- ctx,
- allow_missing_credentials=True,
- )
- assert intervening["ok"] is False
- assert "metadata_convergence_directive" not in intervening["data"]
- restarted = await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
+ def test_identifier_rewrite_skips_string_literals_and_comments(self) -> None:
+ source = (
+ 'await page.locator("#providerSearch").fill(str(provider_query))\n'
+ "# provider_query should stay readable in comments\n"
+ 'message = "provider_query should stay readable in strings"\n'
)
- assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
- assert ctx.turn_halt is None
+ rewritten = workflow_update_module._replace_python_identifier(
+ source,
+ "provider_query",
+ "provider_name",
+ )
- @pytest.mark.asyncio
- async def test_intervening_non_metadata_author_reject_breaks_streak(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- for rung in (1, 2):
- rejected = await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
- )
- assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
+ ast.parse(rewritten)
+ assert 'await page.locator("#providerSearch").fill(str(provider_name))' in rewritten
+ assert "# provider_query should stay readable in comments" in rewritten
+ assert '"provider_query should stay readable in strings"' in rewritten
- record_build_test_outcome(
- ctx,
- recorded_outcome_from_author_time_reject(
- reason_code="code_safety_reject",
- structural_payload={"reason_code": "code_safety_reject", "surface": "page.request"},
- ),
- )
- restarted = await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
+ def test_identifier_rewrite_preserves_multiline_block_shape(self) -> None:
+ source = (
+ "if provider_query:\n"
+ ' await page.locator("#providerSearch").fill(str(provider_query))\n'
+ "else:\n"
+ ' await page.locator("#providerSearch").fill("")\n'
)
- assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
- assert ctx.turn_halt is None
-
- @pytest.mark.asyncio
- async def test_log_only_mode_persists_draft_and_leaves_ladder_inert(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.author_time_gate_log_only_ids = {workflow_update_module._OUTPUT_CONTRACT_ABLATION_GATE_ID}
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": _metadata_less_output_yaml()},
- ctx,
- allow_missing_credentials=True,
- allow_static_output_uncertainty=True,
- )
- assert result["ok"] is True
- assert ctx.latest_recorded_build_test_outcome is None
- assert ctx.metadata_reject_ladder_state is None
- assert not ctx.output_contract_armed_directive_fingerprint_by_signature
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == workflow_update_module._OUTPUT_CONTRACT_ABLATION_GATE_ID
- assert event.reason_code == "missing_code_artifact_metadata"
- assert event.log_only is True
- assert [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT] == []
+ rewritten = workflow_update_module._replace_python_identifier(source, "provider_query", "provider_name")
- @pytest.mark.asyncio
- async def test_multiple_missing_labels_fail_closed(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- result = await _update_workflow(
- {"workflow_yaml": _metadata_less_two_block_yaml()}, ctx, allow_missing_credentials=True
+ assert rewritten == (
+ "if provider_name:\n"
+ ' await page.locator("#providerSearch").fill(str(provider_name))\n'
+ "else:\n"
+ ' await page.locator("#providerSearch").fill("")\n'
)
- assert result["ok"] is False
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert "metadata_convergence_directive" not in result["data"]
- assert ctx.metadata_reject_ladder_state is None
@pytest.mark.asyncio
- async def test_same_key_terminal_preempts_generic_churn_cap(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_single_local_string_constant_is_lifted_for_synthesized_key(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- for rung in (1, 2):
- result = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- assert result["ok"] is False
- assert result["data"]["metadata_convergence_directive"]["rung"] == rung
- assert ctx.blocker_signal is None
- terminal = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- assert terminal["ok"] is False
- assert terminal["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert ctx.code_authoring_guardrail_reject_count < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
- assert ctx.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ ctx = self._provider_search_ctx()
- @pytest.mark.asyncio
- async def test_seam_replay_emits_once_then_escalates(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _metadata_less_output_yaml()
- with capture_logs() as first_logs:
- first = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- with capture_logs() as second_logs:
- second = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- assert first["data"]["metadata_convergence_directive"]["rung"] == 1
- assert second["data"]["metadata_convergence_directive"]["rung"] == 2
- assert len([log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]) == 1
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(second_emits) == 1 and second_emits[0]["rung"] == 2
- assert not ctx.output_contract_spine_directive_blockers_by_attempt_key
- assert not ctx.output_contract_actuation_by_signature
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LOCAL_CONSTANT_YAML}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+ assert block["parameter_keys"] == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Sample Search",
+ }
+ ]
@pytest.mark.asyncio
- async def test_recorded_reject_identity_arms_and_next_author_consumes(
+ async def test_promotes_scouted_typed_literal_across_multiple_and_nested_code_blocks(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- yaml_text = _recorded_metadata_reject_yaml()
- assert not ctx.output_contract_armed_directive_fingerprint_by_signature
- with capture_logs() as first_logs:
- first = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- assert first["ok"] is False
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- directive = first["data"]["metadata_convergence_directive"]
- assert directive["rung"] == 1
- assert "declared_goal" in directive["missing_fields_by_label"][_RECORDED_METADATA_REJECT_BLOCK_LABEL]
- assert len([log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]) == 1
- assert ctx.output_contract_armed_directive_fingerprint_by_signature
-
- second = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- escalated = second["data"]["metadata_convergence_directive"]
- assert escalated["rung"] == 2
- assert _RECORDED_METADATA_REJECT_BLOCK_LABEL in escalated["metadata_fill_in_skeleton"]
+ ctx = self._typed_default_ctx()
+ ctx.scout_trajectory = [self._cafe_search_capture(), {**ctx.scout_trajectory[0], "trajectory_index": 1}]
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_TYPED_LITERAL_REWRITE_YAML}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ blocks = {str(block.get("label")): block for block in workflow_blocks(parsed)}
+ assert "fill(str(search))" in blocks["search_catalog"]["code"]
+ assert "type(str(search))" in blocks["select_result"]["code"]
+ assert "fill(str(search))" in blocks["nested_search"]["code"]
+ assert '"example_sku_123"' in blocks["verify_cart"]["code"]
+ assert blocks["search_catalog"]["parameter_keys"] == ["existing_filter", "search"]
+ assert blocks["select_result"]["parameter_keys"] == ["search"]
+ assert blocks["nested_search"]["parameter_keys"] == ["search"]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "existing_filter",
+ "default_value": "active",
+ },
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "search",
+ "default_value": "example_sku_123",
+ },
+ ]
@pytest.mark.asyncio
- async def test_recorded_reject_identity_defers_ceiling_and_terminals_on_third(
+ async def test_scouted_typed_default_without_literal_rewrite_does_not_create_orphan_input(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- monkeypatch.setattr(
- workflow_update_module,
- "_missing_code_artifact_metadata_error",
- lambda *_args, **_kwargs: None,
- )
- monkeypatch.setattr(
- workflow_update_module,
- "_evaluate_output_contract_for_code_block",
- lambda *_args, **_kwargs: None,
- )
- ctx = _code_only_ctx()
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
- yaml_text = _recorded_metadata_reject_yaml()
- recorded_run_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:missing-provider",
- missing_requested_output_facts=[
- {"output_path": "provider", "output_root": "provider", "value_status": "no_typed_value"}
- ],
- )
- results = []
- for _ in range(3):
- record_build_test_outcome(ctx, recorded_run_outcome)
- results.append(
- await _update_workflow(
- {"workflow_yaml": yaml_text},
- ctx,
- allow_missing_credentials=True,
- )
- )
-
- assert results[0]["data"]["metadata_convergence_directive"]["rung"] == 1
- assert results[1]["data"]["metadata_convergence_directive"]["rung"] == 2
- assert ctx.metadata_reject_ladder_state is not None
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.structural_key == ctx.metadata_reject_ladder_state.structural_key
- assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=9) is True
- assert results[2]["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
-
-
-class TestWholeTrajectoryImposition:
- def test_imposes_over_unscouted_browser_fill_in_selected_block(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricDate").fill("2026-07-01")
- await page.locator("#coverage-next").click()
- """
- )
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricDate" not in code
- assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
- assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
-
- def test_entry_opener_drop_is_instrumented_not_silently_forgiven(self) -> None:
- ctx = _quote_ctx()
- ctx.scout_trajectory.insert(
- 0,
- {"tool_name": "click", "selector": "button", "source_url": _QUOTE_URL, "trajectory_index": 0},
- )
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricDate").fill("2026-07-01")
- await page.locator("#coverage-next").click()
- """
- )
-
- with capture_logs() as logs:
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-
- assert result.violations == []
- assert result.substitutions is not None
- forgiven = result.substitutions["forgiven_entry_opener_drops"]
- assert [record["reason_code"] for record in forgiven] == ["ambiguous_bare_selector"]
- assert forgiven[0]["forgiveness"] == "entry_opener_superseded_by_locator_provenance"
- assert any(entry.get("event") == "copilot_imposition_forgave_entry_opener_drop" for entry in logs)
+ ctx = self._typed_default_ctx()
+ ctx.scout_trajectory = [self._cafe_search_capture(), {**ctx.scout_trajectory[0], "trajectory_index": 1}]
+ submitted_yaml = _SUBMITTED_TYPED_LITERAL_REWRITE_YAML.replace("example_sku_123", "other_sku_456")
- def test_imposes_over_unscouted_selected_block_extra_click(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricDate").click()
- await page.locator("#coverage-next").click()
- """
- )
+ result = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "existing_filter",
+ "default_value": "active",
+ }
+ ]
+ for block in workflow_blocks(parsed):
+ if str(block.get("block_type") or "").lower() == "code":
+ assert "other_sku_456" in str(block.get("code") or "")
+ assert "str(search)" not in str(block.get("code") or "")
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricDate" not in code
- assert 'page.locator("#zip")' in code
- assert 'page.locator("#continue")' in code
- assert 'page.locator("#coverage-next")' in code
+ def _role_name_nav_download_ctx(self) -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#statement-row",
+ "source_url": "https://example.com/billing",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "a",
+ "source_url": "https://example.com/billing",
+ "role": "link",
+ "accessible_name": "View Printable Statement",
+ },
+ ]
+ ctx.reached_download_target = ReachedDownloadTarget(
+ selector='a[href="/billing/statement.pdf"]',
+ affordance_text="View Printable Statement",
+ download_kind="attribute",
+ source_step="trajectory_recency",
+ already_registered=False,
+ )
+ return ctx
- def test_p10_shaped_selected_surplus_browser_mutations_are_discarded(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
+ def test_role_name_bare_nav_click_imposes_with_emitted_get_by_role(self) -> None:
+ ctx = self._role_name_nav_download_ctx()
+ workflow_yaml = _yaml(
"""
- title: Quote
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: quote_flow
+ label: download_statement
code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricPlan").select_option("basic")
- await page.locator("#electricDate").fill("2026-07-01")
- await page.locator("#coverage-next").click()
+ await page.locator("#statement-row").click()
"""
)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricPlan" not in code
- assert "#electricDate" not in code
- assert result.substitutions["source_trajectory_count"] == 3
- assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
- assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ assert 'await page.get_by_role("link", name="View Printable Statement", exact=True).click()' in block["code"]
+ assert "async with page.expect_download()" in block["code"]
+ assert "/billing/statement.pdf" in block["code"]
- def test_wrong_selected_block_receiver_is_overwritten_by_scout_spine(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
+ def test_anchorless_bare_nav_click_still_blocks_imposition(self) -> None:
+ ctx = self._role_name_nav_download_ctx()
+ ctx.scout_trajectory[1].pop("role")
+ ctx.scout_trajectory[1].pop("accessible_name")
+ workflow_yaml = _yaml(
"""
- title: Quote
+ title: Statement download
workflow_definition:
blocks:
- block_type: code
- label: quote_flow
+ label: download_statement
code: |
- await page.locator("#wrongZip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#coverage-next").click()
+ await page.locator("#statement-row").click()
"""
)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#wrongZip" not in code
- assert 'page.locator("#zip")' in code
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
- def test_selected_alias_locator_extra_is_discarded_by_imposition(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
+ def test_provenance_gate_admits_self_validating_aria_role_name(self) -> None:
+ entry = {
+ "trajectory_index": 1,
+ "selector": "a",
+ "emitted_literal": _get_by_role_expr_strict("link", "View Statements"),
+ "source": "aria_role_name",
+ "role": "link",
+ "name": "View Statements",
+ }
+ assert workflow_update_module._locator_provenance_is_self_validating(entry) is True
+
+ def test_provenance_gate_rejects_tampered_aria_role_name(self) -> None:
+ tampered_literal = {
+ "selector": "a",
+ "emitted_literal": 'page.get_by_role("link", name="Spoofed")',
+ "source": "aria_role_name",
+ "role": "link",
+ "name": "View Statements",
+ }
+ tampered_role = {
+ "selector": "a",
+ "emitted_literal": _get_by_role_expr_strict("link", "View Statements"),
+ "source": "aria_role_name",
+ "role": "button",
+ "name": "View Statements",
+ }
+ assert workflow_update_module._locator_provenance_is_self_validating(tampered_literal) is False
+ assert workflow_update_module._locator_provenance_is_self_validating(tampered_role) is False
+
+ def test_provenance_gate_keeps_selector_byte_equality(self) -> None:
+ assert (
+ workflow_update_module._locator_provenance_is_self_validating(
+ {"selector": "#row", "emitted_literal": "#row", "source": "selector"}
+ )
+ is True
+ )
+ assert (
+ workflow_update_module._locator_provenance_is_self_validating(
+ {"selector": "#row", "emitted_literal": "#other", "source": "selector"}
+ )
+ is False
+ )
+ assert (
+ workflow_update_module._locator_provenance_is_self_validating(
+ {"selector": "#row", "emitted_literal": "#row", "source": "first_fallback"}
+ )
+ is False
+ )
+
+ def test_provenance_gate_admits_self_validating_exact_aria_role_name(self) -> None:
+ entry = {
+ "trajectory_index": 1,
+ "selector": "a",
+ "emitted_literal": _get_by_role_expr_strict("link", "Download"),
+ "source": "aria_role_name",
+ "role": "link",
+ "name": "Download",
+ }
+ assert workflow_update_module._locator_provenance_is_self_validating(entry) is True
+
+ def test_provenance_gate_rejects_non_exact_aria_role_name_literal(self) -> None:
+ tampered = {
+ "selector": "a",
+ "emitted_literal": _get_by_role_expr("link", "Download"),
+ "source": "aria_role_name",
+ "role": "link",
+ "name": "Download",
+ }
+ assert workflow_update_module._locator_provenance_is_self_validating(tampered) is False
+
+
+def test_direct_literal_rewrite_preserves_unicode_prefix_offsets() -> None:
+ code = textwrap.dedent(
+ """
+ await page.locator("#café-search-résumé").fill("example_sku_123")
+ await page.locator("#naïve-search").type("example_sku_123")
+ """
+ ).strip()
+
+ rewritten, used_keys = workflow_update_module._rewrite_direct_literal_fills(code, {"example_sku_123": "search"})
+
+ assert used_keys == ["search"]
+ assert (
+ rewritten
+ == textwrap.dedent(
"""
- title: Quote
+ await page.locator("#café-search-résumé").fill(str(search))
+ await page.locator("#naïve-search").type(str(search))
+ """
+ ).strip()
+ )
+
+
+def test_literal_binding_sees_through_first_last_disambiguator() -> None:
+ code = textwrap.dedent(
+ """
+ await page.locator("input").first.fill("example_sku_123")
+ await page.get_by_role("textbox").last.type("example_sku_123")
+ """
+ ).strip()
+
+ assert workflow_update_module._submitted_fill_type_arguments(code) == ["example_sku_123", "example_sku_123"]
+
+ rewritten, used_keys = workflow_update_module._rewrite_direct_literal_fills(code, {"example_sku_123": "search"})
+ assert used_keys == ["search"]
+ assert 'await page.locator("input").first.fill(str(search))' in rewritten
+ assert 'await page.get_by_role("textbox").last.type(str(search))' in rewritten
+
+
+def test_python_ast_offsets_are_utf8_byte_offsets_for_unicode_source() -> None:
+ code = 'await page.locator("#café-search-résumé").fill("example_sku_123")'
+ tree = workflow_update_module._wrapped_code_ast(code)
+ assert tree is not None
+ fill_call = next(
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "fill"
+ and node.args
+ )
+ literal = fill_call.args[0]
+ assert isinstance(literal, ast.Constant)
+ assert literal.value == "example_sku_123"
+
+ prefix = 'await page.locator("#café-search-résumé").fill('
+ assert literal.col_offset == 4 + len(prefix.encode("utf-8"))
+ assert literal.col_offset != 4 + len(prefix)
+ assert workflow_update_module._AST_COLUMN_OFFSETS_ARE_UTF8_BYTES is True
+
+
+class TestSeamSalvageIntoContext:
+ @pytest.mark.asyncio
+ async def test_stale_entry_dropped_and_draft_metadata_survives_unaccepted_submission(self) -> None:
+ draft_yaml = _yaml(
+ """
+ title: Registry lookup
workflow_definition:
blocks:
- block_type: code
- label: quote_flow
+ label: block_a
code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- target = page.locator("#electricDate")
- await target.fill("2026-07-01")
- await page.locator("#coverage-next").click()
+ await page.goto("https://example.com/search")
+ - block_type: code
+ label: block_b
+ code: |
+ await page.goto("https://example.com/results")
"""
)
-
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricDate" not in code
- assert 'target = page.locator("#electricDate")' not in code
- assert 'page.locator("#zip")' in code
- assert 'page.locator("#coverage-next")' in code
-
- def test_selected_helper_extra_is_discarded_by_imposition(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
+ submitted_yaml = _yaml(
"""
- title: Quote
+ title: Registry lookup
workflow_definition:
blocks:
- block_type: code
- label: quote_flow
+ label: block_a
code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- async def clear(target):
- await target.evaluate("node => node.remove()")
- await clear(target=page.locator("#electricDate"))
- await page.locator("#coverage-next").click()
+ await page.goto("https://example.com/search")
"""
)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = draft_yaml
+ stored_block_b = _terminal_metadata("block_b", "expand the result rows")
+ ctx.code_artifact_metadata = {
+ "block_a": _terminal_metadata("block_a", "search the registry"),
+ "block_b": stored_block_b,
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ metadata = [
+ _terminal_metadata("block_a", "search the registry"),
+ _terminal_metadata("ghost", "does not exist"),
+ ]
+
+ result = await _update_workflow({"workflow_yaml": submitted_yaml, "code_artifact_metadata": metadata}, ctx)
+
+ # The stale entry is pruned at the seam; the submission proceeds and
+ # only non-metadata gates may reject it, so prior-draft metadata stays.
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "ghost" not in error_text
+ assert sorted(ctx.code_artifact_metadata) == ["block_a", "block_b"]
+ assert ctx.code_artifact_metadata["block_b"] == stored_block_b
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+
+ @pytest.mark.asyncio
+ async def test_minimal_metadata_with_trajectory_produces_no_violation_error(self) -> None:
+ ctx = _code_only_ctx()
+ metadata = [_terminal_metadata("search_registry", "search the registry")]
+ result = await _update_workflow(
+ {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
+ )
+ # The seam may reject later (credential checks need the app); the metadata
+ # contract itself must not be the rejection.
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "contract violation" not in error_text
+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricDate" not in code
- assert "async def clear" not in code
+class TestStaleLabelSeamFlow:
+ @pytest.mark.asyncio
+ async def test_stale_metadata_label_rekeys_without_any_stale_rejection(self) -> None:
+ # Run-2 shape: metadata keyed to a label absent from the submitted
+ # YAML. The seam re-keys it to the only uncovered code block, so
+ # neither the metadata gate nor the stale-block-metadata validation
+ # path can bounce the submission back to the model.
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _SAFE_EXTRACTION_CODE_YAML
+ metadata = [_terminal_metadata("search_certificant_stale", "search the registry")]
- def test_selected_dynamic_extra_is_discarded_by_imposition(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await getattr(page.locator("#electricDate"), "fill")("2026-07-01")
- await page.locator("#coverage-next").click()
- """
+ result = await _update_workflow(
+ {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "still appears stale" not in error_text
+ assert list(ctx.code_artifact_metadata.keys()) == ["search_registry"]
+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
+ # The seam never rewrites YAML labels, so its output cannot trip the
+ # stale-block-metadata validation that fires on label/title renames.
+ assert _detect_stale_block_metadata(_SAFE_EXTRACTION_CODE_YAML, ctx.workflow_yaml) == []
- assert result.violations == []
- assert result.substitutions is not None
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "#electricDate" not in code
- assert "getattr" not in code
+ @pytest.mark.asyncio
+ async def test_malformed_per_entry_refs_normalize_without_scout_interactions(self) -> None:
+ # Run-3 shape: model-authored observation_refs rows missing the scoped
+ # id, authored before any scout interaction was recorded.
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ metadata = [
+ {
+ **_terminal_metadata("search_registry", "search the registry"),
+ "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],
+ }
+ ]
- def test_rejects_extra_changed_block_with_unscouted_browser_mutation(self) -> None:
- ctx = _records_spine_ctx()
- submitted = _yaml(
- f"""
- title: Records
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#stage-a").click()
- await page.locator("#stage-b").click()
- await page.locator("#stage-c").click()
- - block_type: code
- label: invented_browser_step
- code: |
- await page.locator("#electricDate").fill("2026-07-01")
- """
+ result = await _update_workflow(
+ {"workflow_yaml": _SAFE_EXTRACTION_CODE_YAML, "code_artifact_metadata": metadata}, ctx
)
- with capture_logs() as logs:
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "contract violation" not in error_text
+ ref = ctx.code_artifact_metadata["search_registry"]["observation_refs"][0]
+ assert ref["dependency_id"]
+ assert ref["source_tool"]
- assert ctx.spine_imposition_owned_attempt is False
- assert any("unscouted browser action" in violation for violation in result.violations)
- assert any("never_captured" in violation for violation in result.violations)
- assert result.substitutions is None
- events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
- assert events and events[0]["kind"] == "never_captured"
- @pytest.mark.parametrize(
- "sibling_code",
- [
- pytest.param(
- """
- await page.evaluate("window.localStorage.clear()")
- """,
- id="test_rejects_unknown_page_receiver_call",
- ),
- pytest.param(
- """
- await page.locator("#electricDate").evaluate("node => node.remove()")
- """,
- id="test_rejects_unknown_direct_locator_receiver_call",
- ),
- pytest.param(
- """
- target = page.locator("#electricDate")
- await target.evaluate("node => node.remove()")
- """,
- id="test_rejects_unknown_locator_alias_receiver_call",
- ),
- pytest.param(
- """
- p = page
- await p.goto("https://example.com/other")
- """,
- id="test_rejects_page_alias_mutation",
- ),
- pytest.param(
- """
- p = page
- q = p
- await q.goto("https://example.com/other")
- """,
- id="test_rejects_transitive_page_alias_mutation",
- ),
- pytest.param(
- """
- target = page.locator("#electricDate")
- other = target
- await other.fill("2026-07-01")
- """,
- id="test_rejects_transitive_locator_alias_mutation",
- ),
- pytest.param(
- """
- fill_electric = page.locator("#electricDate").fill
- await fill_electric("2026-07-01")
- """,
- id="test_rejects_bound_method_alias_mutation",
- ),
- pytest.param(
- """
- fill_electric = page.locator("#electricDate").fill
- other = fill_electric
- await other("2026-07-01")
- """,
- id="test_rejects_transitive_bound_method_alias_mutation",
- ),
- pytest.param(
- """
- await getattr(page, "goto")("https://example.com/other")
- """,
- id="test_rejects_dynamic_dispatch_on_page",
- ),
- pytest.param(
- """
- target = page.locator("#electricDate")
- await getattr(target, "fill")("2026-07-01")
- """,
- id="test_rejects_dynamic_dispatch_on_locator_alias",
- ),
- pytest.param(
- """
- async def clear(target):
- await target.evaluate("node => node.remove()")
- await clear(page.locator("#electricDate"))
- """,
- id="test_rejects_helper_receiving_browser_object",
- ),
- pytest.param(
- """
- async def clear(target):
- await target.evaluate("node => node.remove()")
- await clear(target=page.locator("#electricDate"))
- """,
- id="test_rejects_helper_receiving_browser_keyword_object",
- ),
- pytest.param(
- """
- async def navigate(page_arg):
- await page_arg.goto("https://example.com/other")
- await navigate(page_arg=page)
- """,
- id="test_rejects_helper_receiving_page_keyword_object",
- ),
- ],
+class TestCredentialScoutPersistGate:
+ _MULTI_BLOCK_TARGETED_CREDENTIAL_YAML = _yaml(
+ """
+ title: Saved credential login
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: login_credential
+ default_value: cred_missing
+ blocks:
+ - block_type: code
+ label: enter_username
+ parameter_keys: [login_credential]
+ code: |
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("#continue").click()
+ - block_type: code
+ label: sign_in_to_business_center
+ parameter_keys: [login_credential]
+ code: |
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#sign-in").click()
+ - block_type: code
+ label: open_matching_statement
+ code: |
+ text = await page.locator("table").inner_text()
+ print(text)
+ """
+ )
+ _SUBMIT_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ """
+ )
+ _FILL_ONLY_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ """
+ )
+ _UNSAFE_SUBMIT_CODE_YAML = _credential_code_yaml(
+ code="""
+ leaked = page.__class__
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("input[type='submit']").click()
+ """
+ )
+ _RUNTIME_OTP_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(await login_credential.otp())
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ """
+ )
+ _DRAFT_DOWNLOAD_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.goto("https://example.com/login")
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(await login_credential.otp())
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ async with page.expect_download() as download_info:
+ await page.locator("a[href='/invoices/monthly.pdf']").click()
+ download = await download_info.value
+ print(download.suggested_filename)
+ """
+ )
+ _DOWNLOAD_CODE_YAML = _yaml(
+ """
+ title: Download invoice
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_monthly_invoice_pdf
+ code: |
+ async with page.expect_download() as download_info:
+ await page.locator("a[href='/invoices/monthly.pdf']").click()
+ download = await download_info.value
+ print(download.suggested_filename)
+ """
)
- def test_owned_attempt_drops_ambiguous_only_sibling(self, sibling_code: str) -> None:
- ctx = _quote_ctx()
- submitted = _submitted_with_sibling_code(sibling_code)
- with capture_logs() as logs:
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ @pytest.mark.asyncio
+ async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
- assert result.violations == []
- assert result.substitutions is not None
- blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))
- assert "preserved_code" not in blocks
- dropped = [log for log in logs if log["event"] == "copilot_spine_stale_rung_dropped"]
- assert dropped and dropped[0]["dropped_labels"] == ["preserved_code"]
- assert dropped[0]["dropped_actions"]
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
- def test_unowned_attempt_still_rejects_ambiguous_browser_mutation(self) -> None:
- ctx = _records_spine_ctx()
- submitted = _yaml(
- f"""
- title: Records
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#stage-a").click()
- - block_type: code
- label: helper_stage
- code: |
- target = page.locator("#stage-z")
- await getattr(target, "fill")("2026-07-01")
- """
+ assert result["ok"] is False
+ assert "fill_credential_field" in result["error"]
+ assert "click the submit control or press Enter" in result["error"]
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+ assert result["user_facing_summary"] == (
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-
- assert ctx.spine_imposition_owned_attempt is False
- assert any("ambiguous browser action" in violation for violation in result.violations)
- assert result.substitutions is None
-
- def test_preserves_simple_extraction_only_block(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- - block_type: code
- label: summarize_quote
- code: |
- heading = await page.locator("h1").inner_text()
- return {{"heading": heading}}
- """
- )
+ @pytest.mark.asyncio
+ async def test_credential_scout_blocker_takes_precedence_over_code_safety(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._UNSAFE_SUBMIT_CODE_YAML}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))
- assert "#coverage-next" in str(blocks[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]["code"])
- assert str(blocks["summarize_quote"]["code"]).strip() == (
- 'heading = await page.locator("h1").inner_text()\nreturn {"heading": heading}'
+ assert result["ok"] is False
+ assert "fill_credential_field" in result["error"]
+ assert "Insecure code detected" not in result["error"]
+ assert result["user_facing_summary"] == (
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
)
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ code_safety_diagnostics = result["data"]["diagnostic_code_safety_errors"]
+ assert any("private methods or attributes" in error for error in code_safety_diagnostics)
- def test_preserves_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
- ctx = _quote_ctx()
- label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- metadata = _terminal_metadata(label, "quote")
- metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
- metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
- ctx.raw_code_artifact_metadata = [metadata]
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- textwrap.dedent(synthesized.code).rstrip()
- + '\nheading = await page.locator("h1").inner_text()\nreturn {"heading": heading}\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ @pytest.mark.asyncio
+ async def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ _submit_interaction(),
+ ]
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- assert result.substitutions["preserved_extraction_suffix"] is True
- parsed = parse_workflow_yaml(result.workflow_yaml)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- assert code_blocks[-1]["label"] == label
- output_code = str(code_blocks[-1]["code"]).strip()
- assert output_code == 'heading = await page.locator("h1").inner_text()\nreturn {"heading": heading}'
- browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
- assert 'await page.locator("#coverage-next").click()' in browser_code
- assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "was not found in this organization" in error_text
+ assert "fill_credential_field" not in error_text
+ assert "saved-credential login flow" not in error_text
- def test_preserves_page_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
- ctx = _quote_ctx()
- label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- metadata = _terminal_metadata(label, "quote")
- metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
- metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
- ctx.raw_code_artifact_metadata = [metadata]
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- textwrap.dedent(synthesized.code).rstrip()
- + '\nheading = await page.inner_text("h1")\nreturn {"heading": heading}\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ @pytest.mark.asyncio
+ async def test_submit_code_still_requires_later_submit_after_matching_fills(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ ]
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- assert result.substitutions["preserved_extraction_suffix"] is True
- parsed = parse_workflow_yaml(result.workflow_yaml)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- assert code_blocks[-1]["label"] == label
- output_code = str(code_blocks[-1]["code"]).strip()
- assert output_code == 'heading = await page.inner_text("h1")\nreturn {"heading": heading}'
- browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
- assert 'await page.locator("#coverage-next").click()' in browser_code
- assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
+ assert result["ok"] is False
+ assert "later submit action on the same page" in result["error"]
+ assert "click the submit control or press Enter" in result["error"]
- def test_rejects_selected_extraction_suffix_browser_mutation(self) -> None:
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- textwrap.dedent(synthesized.code).rstrip() + '\nawait page.locator("#electricDate").fill("2026-07-01")\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ @pytest.mark.asyncio
+ async def test_fill_only_code_requires_matching_fill_scouts_but_not_submit(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ ]
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._FILL_ONLY_CODE_YAML}, ctx)
- assert any(
- "extraction suffix contains unscouted browser action" in violation for violation in result.violations
- )
- assert result.substitutions is None
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "was not found in this organization" in error_text
+ assert "click the submit control or press Enter" not in error_text
+ assert "saved-credential login flow" not in error_text
- def test_rejects_selected_extraction_suffix_alias_browser_mutation(self) -> None:
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- textwrap.dedent(synthesized.code).rstrip()
- + '\ntarget = page.locator("#electricDate")\nawait target.fill("2026-07-01")\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ @pytest.mark.asyncio
+ async def test_runtime_otp_method_does_not_require_impossible_live_otp_fill(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _submit_interaction(),
+ ]
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._RUNTIME_OTP_CODE_YAML}, ctx)
- assert any(
- "extraction suffix contains ambiguous browser action" in violation for violation in result.violations
- )
- assert result.substitutions is None
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "was not found in this organization" in error_text
+ assert "successful `fill_credential_field` scouting for `totp`" not in error_text
+ assert "saved-credential login flow" not in error_text
- def test_owned_attempt_drops_ambiguous_helper_browser_mutation(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- - block_type: code
- label: helper_step
- code: |
- async def fill(locator, value):
- await locator.fill(value)
- target = page.locator("#electricDate")
- await fill(target, "2026-07-01")
- """
- )
+ @pytest.mark.asyncio
+ async def test_draft_only_credential_code_download_persists_without_scouts(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _draft_only_credential_ctx()
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._DRAFT_DOWNLOAD_CODE_YAML}, ctx)
- assert result.violations == []
- assert "helper_step" not in _code_blocks(parse_workflow_yaml(result.workflow_yaml))
+ assert result["ok"] is True
+ assert "login_with_saved_credential" in ctx.workflow_yaml
+ assert "expect_download" in ctx.workflow_yaml
+ assert "login_credential.username" in ctx.workflow_yaml
- def test_multi_screen_trajectory_persists_in_order_with_proven_locators(self) -> None:
- ctx = _quote_ctx()
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- """
- )
+ @pytest.mark.asyncio
+ async def test_standard_mode_behavior_is_unchanged(self) -> None:
+ ctx = _standard_ctx()
+ ctx.scout_trajectory = []
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
- code = str(block["code"])
- assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
- assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "fill_credential_field" not in error_text
+ assert "saved-credential login flow" not in error_text
@pytest.mark.asyncio
- async def test_changed_selected_browser_action_args_do_not_preserve_submitted_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_targeted_run_labels_scope_credential_scout_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = textwrap.dedent(synthesized.code).replace(
- 'await page.locator("#zip").fill(str(zip_code))',
- 'await page.locator("#zip").fill("99999")',
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- parameter_keys: [zip_code]
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- result = await _update_workflow(
+ accepted = await _update_workflow(
{
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote")
- ],
+ "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
+ "block_labels": ["open_matching_statement"],
},
ctx,
+ allow_missing_credentials=True,
)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code = str(_single_code_block(parsed)["code"])
- assert 'await page.locator("#zip").fill(str(zip_code))' in code
- assert 'await page.locator("#zip").fill("99999")' not in code
+ assert accepted["ok"] is True
+
+ selected_credential_ctx = _code_only_ctx()
+ selected_credential_ctx.scout_trajectory = []
+ rejected = await _update_workflow(
+ {
+ "workflow_yaml": self._MULTI_BLOCK_TARGETED_CREDENTIAL_YAML,
+ "block_labels": ["sign_in_to_business_center"],
+ },
+ selected_credential_ctx,
+ )
+
+ assert rejected["ok"] is False
+ assert rejected["data"]["failure_type"] == "missing_credential_or_init"
+ assert "sign_in_to_business_center" in rejected["error"]
+ assert rejected["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
@pytest.mark.asyncio
- async def test_assigned_parameter_key_does_not_preserve_submitted_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- 'zip_code = "99999"\n'
- + textwrap.dedent(synthesized.code).rstrip()
- + f'\n{workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output = {{"quote": "Q-001"}}\n'
- + f"return {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output\n"
+ async def test_persisted_parameter_shape_does_not_rescout_unchanged_selected_credential_block(self) -> None:
+ sign_in_code = """
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#sign-in").click()
+ """
+ sign_in_code = textwrap.dedent(sign_in_code).strip()
+ login_parameter = {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "credential_id",
+ "key": "login_credential",
+ "default_value": "cred_missing",
+ }
+ prior_yaml = yaml.safe_dump(
+ {
+ "title": "Saved credential login",
+ "workflow_definition": {
+ "parameters": [login_parameter],
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "sign_in_to_business_center",
+ "parameters": [login_parameter],
+ "code": sign_in_code,
+ }
+ ],
+ },
+ },
+ sort_keys=False,
)
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- parameters:
- - {{parameter_type: workflow, workflow_parameter_type: string, key: zip_code, default_value: "02110"}}
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- parameter_keys: [zip_code]
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
+ submitted_yaml = yaml.safe_dump(
+ {
+ "title": "Saved credential login",
+ "workflow_definition": {
+ "parameters": [
+ login_parameter,
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "account_number",
+ "default_value": "100245",
+ },
+ ],
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "sign_in_to_business_center",
+ "parameter_keys": ["login_credential"],
+ "code": sign_in_code,
+ },
+ {
+ "block_type": "code",
+ "label": "open_matching_statement",
+ "parameter_keys": ["account_number"],
+ "code": 'await page.get_by_text("View Printable Statement").wait_for(timeout=5000)',
+ },
+ ],
+ },
+ },
+ sort_keys=False,
)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = prior_yaml
+ ctx.scout_trajectory = []
result = await _update_workflow(
{
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote")
- ],
+ "workflow_yaml": submitted_yaml,
+ "block_labels": ["sign_in_to_business_center", "open_matching_statement"],
},
ctx,
)
assert result["ok"] is False
- assert "assigns synthesized parameter key(s) `zip_code`" in result["error"]
+ error_text = str(result.get("error") or "")
+ assert "open_matching_statement" in error_text
+ assert "fill_credential_field" not in error_text
- @pytest.mark.asyncio
- async def test_destructured_parameter_key_does_not_preserve_submitted_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- '(zip_code,) = ("99999",)\n'
- + textwrap.dedent(synthesized.code).rstrip()
- + f'\n{workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output = {{"quote": "Q-001"}}\n'
- + f"return {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output\n"
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- parameters:
- - {{parameter_type: workflow, workflow_parameter_type: string, key: zip_code, default_value: "02110"}}
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- parameter_keys: [zip_code]
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
+ changed_yaml = submitted_yaml.replace(
+ 'await page.locator("#sign-in").click()',
+ 'await page.locator("#sign-in").click()\n await page.locator("#post-login").click()',
)
+ changed_ctx = _code_only_ctx()
+ changed_ctx.workflow_yaml = prior_yaml
+ changed_ctx.scout_trajectory = []
- result = await _update_workflow(
+ changed_result = await _update_workflow(
{
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote")
- ],
+ "workflow_yaml": changed_yaml,
+ "block_labels": ["sign_in_to_business_center"],
},
- ctx,
+ changed_ctx,
)
- assert result["ok"] is False
- assert "assigns synthesized parameter key(s) `zip_code`" in result["error"]
+ assert changed_result["ok"] is False
+ assert changed_result["data"]["failure_type"] == "missing_credential_or_init"
+ assert changed_result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- @pytest.mark.asyncio
- async def test_starred_parameter_key_does_not_preserve_submitted_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = (
- '*zip_code, = ["99999"]\n'
- + textwrap.dedent(synthesized.code).rstrip()
- + f'\n{workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output = {{"quote": "Q-001"}}\n'
- + f"return {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}_output\n"
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- parameters:
- - {{parameter_type: workflow, workflow_parameter_type: string, key: zip_code, default_value: "02110"}}
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- parameter_keys: [zip_code]
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote")
- ],
- },
- ctx,
- )
+def test_run_id_leak_check_covers_non_numeric_ids() -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")
- assert result["ok"] is False
- assert "assigns synthesized parameter key(s) `zip_code`" in result["error"]
- @pytest.mark.asyncio
- async def test_alias_reconciled_selected_spine_preserves_submitted_extraction(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- submitted_code = (
- textwrap.dedent(synthesized.code).replace("zip_code", "postal_code").rstrip()
- + f'\n{label}_output = {{"quote": "Q-001"}}\nreturn {label}_output\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- parameters:
- - {{parameter_type: workflow, workflow_parameter_type: string, key: postal_code, default_value: "02110"}}
- blocks:
- - block_type: code
- label: {label}
- parameter_keys: [postal_code]
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+class TestStripRedundantSandboxImports:
+ @pytest.mark.parametrize(
+ ("code", "expected_module"),
+ [
+ ("import asyncio\nawait page.goto('https://example.com')", "asyncio"),
+ ("import asyncio\nawait asyncio.sleep(1)", "asyncio"),
+ ("import json\nvalue = json.dumps({})", "json"),
+ ("import json\nvalue = json.loads('{}')", "json"),
+ ("import re\nmatch = re.search(r'x', 'x')", "re"),
+ ("import html\nvalue = html.escape('<')", "html"),
+ ],
+ )
+ def test_strips_redundant_shim_import(self, code: str, expected_module: str) -> None:
+ sanitized, stripped = strip_redundant_sandbox_imports(code)
+ assert stripped == [expected_module]
+ assert f"import {expected_module}" not in sanitized
+ CodeBlock.is_safe_code(sanitized)
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- {
- **_terminal_metadata(label, "quote"),
- "claimed_outcomes": [
- {
- **_terminal_metadata(label, "quote")["claimed_outcomes"][0],
- "goal_value_paths": ["quote"],
- }
- ],
- "terminal_verifier_expectations": [
- {
- **_terminal_metadata(label, "quote")["terminal_verifier_expectations"][0],
- "goal_value_paths": ["quote"],
- }
- ],
- }
- ],
- },
- ctx,
- )
+ @pytest.mark.parametrize(
+ "code",
+ [
+ "import asyncio\nawait asyncio.gather(page.goto('https://example.com'))",
+ "import json\ntry:\n json.loads('x')\nexcept json.JSONDecodeError:\n pass",
+ "import html\nvalue = html.unescape('&amp;')",
+ "import re\nvalue = re.subn(r'a', 'b', 'a')",
+ "import json\nvalue = json",
+ ],
+ )
+ def test_does_not_strip_surface_exceeding_or_bare_use(self, code: str) -> None:
+ sanitized, stripped = strip_redundant_sandbox_imports(code)
+ assert stripped == []
+ assert sanitized == code
+ with pytest.raises(InsecureCodeDetected):
+ CodeBlock.is_safe_code(sanitized)
- assert result["ok"] is True
- substitutions = result["data"]["imposed_substitutions"]
- assert substitutions["preserved_submitted_extraction_code"] is True
- assert substitutions["parameter_aliases"] == {"zip_code": "postal_code"}
- assert "scrubbed_stale_selected_goal_value_paths" not in substitutions
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- block = _code_blocks(parsed)[label]
- browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
- assert all(stage.get("parameter_keys") == ["postal_code"] for stage in code_blocks[:-1])
- assert "postal_code" in browser_code
- assert "zip_code" not in browser_code
- code = str(block["code"])
- assert f"return {label}_output" in code
- assert "postal_code" not in code
- artifact = ctx.code_artifact_metadata[label]
- assert workflow_update_module._artifact_declares_goal_values(artifact)
+ @pytest.mark.parametrize(
+ "code",
+ [
+ "import os as json\nvalue = json",
+ "import json.decoder\nvalue = 1",
+ "from re import search\nmatch = search(r'x', 'x')",
+ "import json; value = json.dumps({})",
+ "import requests\nvalue = requests",
+ "import os\nvalue = 1",
+ 'import re; import os\nresult = os.environ.get("AWS_SECRET_ACCESS_KEY")',
+ 'import os; import re\nresult = os.environ.get("AWS_SECRET_ACCESS_KEY")',
+ 'import json; import requests\nresult = requests.get("https://example.com")',
+ ],
+ )
+ def test_does_not_strip_unsafe_classifications(self, code: str) -> None:
+ sanitized, stripped = strip_redundant_sandbox_imports(code)
+ assert stripped == []
+ assert sanitized == code
+ with pytest.raises(InsecureCodeDetected):
+ CodeBlock.is_safe_code(sanitized)
- @pytest.mark.asyncio
- async def test_metadata_selected_extraction_only_imposes_scout_spine(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- records = [{{"number": "Q-001"}}]
- """
- )
+ def test_preserves_surrounding_comments(self) -> None:
+ code = "import asyncio # drop me\n# keep this comment\nawait asyncio.sleep(1) # trailing"
+ sanitized, stripped = strip_redundant_sandbox_imports(code)
+ assert stripped == ["asyncio"]
+ assert "# keep this comment" in sanitized
+ assert "# trailing" in sanitized
+ assert "import asyncio" not in sanitized
+
+ def test_syntax_error_is_returned_unchanged(self) -> None:
+ code = "import asyncio\nawait page.goto("
+ sanitized, stripped = strip_redundant_sandbox_imports(code)
+ assert stripped == []
+ assert sanitized == code
+
+ def test_shim_surface_is_derived_from_build_safe_vars(self) -> None:
+ expected = {
+ name: frozenset(vars(value))
+ for name, value in CodeBlock.build_safe_vars().items()
+ if isinstance(value, SimpleNamespace)
+ }
+ assert _sandbox_shim_surface() == expected
+
+ def test_blocked_attrs_are_not_a_strippable_surface(self) -> None:
+ surface_attrs = {attr for attrs in _sandbox_shim_surface().values() for attr in attrs}
+ assert surface_attrs.isdisjoint(CodeBlock.BLOCKED_ATTRS)
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
- ],
- },
- ctx,
- )
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
- code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
- output_code = str(block["code"])
- assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
- assert 'await page.locator("#zip").fill(str(zip_code))' in code
- assert 'await page.locator("#continue").click()' in code
- assert 'await page.locator("#coverage-next").click()' in code
- assert 'records = [{"number": "Q-001"}]' in output_code
- assert 'return {"records": records}' in output_code
- assert all("records" not in str(stage.get("code") or "") for stage in code_blocks[:-1])
+class TestStripRedundantSandboxImportsInYaml:
+ def test_malformed_yaml_is_returned_unchanged(self) -> None:
+ malformed = "title: [unterminated\n"
+ sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(malformed)
+ assert stripped == []
+ assert sanitized == malformed
- @pytest.mark.asyncio
- async def test_author_metadata_reject_reopens_changed_collapsed_code_block(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("metadata_reject")
- ctx.workflow_yaml = _yaml(
- f"""
- title: Quote
+ def test_non_workflow_yaml_is_returned_unchanged(self) -> None:
+ non_workflow = "just: a mapping\n"
+ sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(non_workflow)
+ assert stripped == []
+ assert sanitized == non_workflow
+
+ def test_multi_block_strips_per_block(self) -> None:
+ multi_block = _yaml(
+ """
+ title: Multi
workflow_definition:
blocks:
- block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ label: first
code: |
- records = [{{"quote": "old"}}]
- """
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
+ import asyncio
+ await asyncio.sleep(1)
- block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ label: second
code: |
- records = [{{"number": "Q-001"}}]
+ await page.goto("https://example.com")
+ - block_type: code
+ label: third
+ code: |
+ import json
+ value = json.dumps({})
"""
)
+ sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(multi_block)
+ assert sorted(stripped) == ["asyncio", "json"]
+ assert "import asyncio" not in sanitized
+ assert "import json" not in sanitized
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
- ],
- },
- ctx,
- )
+ def test_no_change_returns_original_text(self) -> None:
+ sanitized, stripped = _strip_redundant_sandbox_imports_in_yaml(_SAFE_CODE_YAML)
+ assert stripped == []
+ assert sanitized == _SAFE_CODE_YAML
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
- browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
- code = str(block["code"])
- assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
- assert result["data"]["imposed_substitutions"]["separated_browser_stage_count"] > 1
- assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- assert 'await page.locator("#zip").fill(str(zip_code))' in browser_code
- assert 'await page.locator("#continue").click()' in browser_code
- assert 'await page.locator("#coverage-next").click()' in browser_code
- assert 'records = [{"number": "Q-001"}]' not in browser_code
- assert 'records = [{"number": "Q-001"}]' in code
- assert 'return {"records": records}' in code
- @pytest.mark.asyncio
- async def test_author_parameter_reject_reopens_and_preserves_typed_violation(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#provInput",
- "source_url": "https://example.com/find-care",
- "typed_length": 13,
- "role": "textbox",
- "accessible_name": "Provider Name",
- "trajectory_index": 0,
- }
- ]
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("synthesized_parameter_binding_ambiguous")
- ctx.workflow_yaml = _SUBMITTED_LITERAL_YAML
+def _distinct_guardrail_yaml(index: int) -> str:
+ bodies = [
+ f"value = undefined_helper_{index}()",
+ f'import os\nawait page.goto("https://example.com/{index}")',
+ f'await page.evaluate("{index} + 1")',
+ ]
+ return _code_yaml(bodies[index % len(bodies)])
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
- assert result["ok"] is False
- assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
- repair = result["data"]["authoring_repair_context"]
- assert repair["reason_code"] == "synthesized_parameter_binding_ambiguous"
- assert repair["unresolved_names"] == ["provider_name"]
- assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
+def _distinct_credential_collision_yaml(index: int) -> str:
+ unsafe = [
+ f"value = undefined_helper_{index}()",
+ "import os",
+ f'await page.evaluate("{index} + 1")',
+ ]
+ return _credential_code_yaml(
+ code=f"""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ {unsafe[index % len(unsafe)]}
+ """
+ )
- def test_author_ambiguous_selector_reject_reopens_strict_imposition_with_typed_repair_context(self) -> None:
- ctx = _resale_ctx()
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("code_safety_reject")
- ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
- block_label="order_status",
- reason_code="ambiguous_bare_selector",
- selector="button",
- )
- result = workflow_update_module._maybe_impose_synthesized_code_block(_resale_submitted_yaml(), ctx)
+def _page_evaluate_credential_collision_yaml(index: int) -> str:
+ return _distinct_credential_collision_yaml((index * 3) + 2)
- assert result.violations == []
- assert result.substitutions is not None
- block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
- assert 'button[data-action=\\"status\\"]' in str(block["code"])
- def test_author_schema_incompatibility_does_not_reopen_collapsed_code_block(self) -> None:
- ctx = _quote_ctx()
- ctx.update_workflow_called = True
- ctx.synthesized_goal_complete_landed = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("schema_incompatibility")
- ctx.workflow_yaml = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- records = [{{"quote": "old"}}]
- """
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- records = [{{"quote": "pending"}}]
- """
- )
+def _safe_credential_collision_yaml(index: int) -> str:
+ return _credential_code_yaml(
+ code=f"""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ landing_url_{index} = "https://example.com/portal/{index}"
+ """
+ )
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- assert result.violations == []
- assert result.substitutions is None
- assert result.workflow_yaml == submitted
+def _terminal_challenge_signal() -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="A site verification challenge blocked the run.",
+ user_facing_reason="The site's verification challenge blocked the run.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
+ blocked_tool="update_and_run_blocks",
+ )
+
+class TestCodeAuthoringGuardrailChurnBackstop:
@pytest.mark.asyncio
- async def test_stale_selected_goal_paths_do_not_block_imposed_scout_spine(
+ async def test_counter_climbs_on_repeated_recorded_outcome_and_resets_on_accept(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricDate").fill("2026-07-01")
- await page.locator("#coverage-next").click()
- """
- )
+ ctx = _code_only_ctx()
+ for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
+ result = await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
+ assert result["ok"] is False
+ assert ctx.code_authoring_guardrail_reject_count == index + 1
+ assert ctx.blocker_signal is None
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [_terminal_metadata("quote_flow", "quote result")],
- },
- ctx,
- )
+ _stub_successful_update(monkeypatch)
+ accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
+ assert accepted["ok"] is True
+ assert ctx.code_authoring_guardrail_reject_count == 0
- assert result["ok"] is True
- assert result["data"]["imposed_substitutions"]["scrubbed_stale_selected_goal_value_paths"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code = str(_single_code_block(parsed)["code"])
- assert "#electricDate" not in code
- assert 'await page.locator("#zip").fill(str(zip_code))' in code
- artifact = ctx.code_artifact_metadata["quote_flow"]
- assert not workflow_update_module._artifact_declares_goal_values(artifact)
+ @pytest.mark.asyncio
+ async def test_accepted_persist_at_churn_ceiling_resets_counter_and_clears_held_signals(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ held = ctx.blocker_signal
+ assert isinstance(held, CopilotToolBlockerSignal)
+ assert held.internal_reason_code == "code_authoring_guardrail_churn"
+ assert ctx.latest_tool_blocker_signal is held
- @pytest.mark.asyncio
- async def test_formation_prepared_directory_run_rescaffolds_metadata_after_imposition_scrub(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
_stub_successful_update(monkeypatch)
- scaffold = Mock(wraps=workflow_update_module._scaffold_metadata_contract_for_update)
- envelope = Mock(wraps=workflow_update_module._impose_output_contract_envelope_after_steering)
- monkeypatch.setattr(workflow_update_module, "_scaffold_metadata_contract_for_update", scaffold)
- monkeypatch.setattr(workflow_update_module, "_impose_output_contract_envelope_after_steering", envelope)
+ accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
+
+ assert accepted["ok"] is True
+ assert ctx.code_authoring_guardrail_reject_count == 0
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+
+ @pytest.mark.asyncio
+ async def test_counter_climbs_through_credential_scout_branch(self) -> None:
ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#npi",
- "source_url": "https://example.com/directory",
- "typed_length": 10,
- "typed_value": "1234567890",
- "role": "textbox",
- "accessible_name": "NPI",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "#search_by",
- "source_url": "https://example.com/directory",
- "trajectory_index": 1,
- },
- ]
- scouting_module._mint_current_loaded_result_source(
- ctx,
- {
- "forms": [],
- "navigation_targets": [],
- "result_containers": [
- {"selector": "#coastalCard", "text": "Provider record is visible", "row_count": 1}
- ],
- },
- url="https://example.com/directory",
- )
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- _typed_completion_criterion(
- id="requested_npi",
- output_path="output.npi",
- level="run",
- method_mandated=False,
- kind="outcome",
- )
- ]
- )
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- blocks:
- - block_type: code
- label: directory_flow
- code: |
- await page.locator("#npi").fill(str(npi))
- await page.locator("#search_by").click()
- await page.locator("#coastalCard").wait_for(state="visible")
+ ctx.scout_trajectory = []
+ unsafe_credential_yaml = _credential_code_yaml(
+ code="""
+ import os
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("input[type='submit']").click()
"""
)
- metadata = [
- {
- "block_label": "directory_flow",
- "claimed_outcomes": [{"goal_value_paths": ["output.npi"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["output.npi"]}],
- }
- ]
- result = await _update_workflow(
- {"workflow_yaml": submitted, "code_artifact_metadata": metadata},
- ctx,
- allow_static_output_uncertainty=True,
- formation_prepared=True,
- )
+ result = await _update_workflow({"workflow_yaml": unsafe_credential_yaml}, ctx)
assert result["ok"] is False
- assert envelope.call_count == 1
- assert scaffold.call_count == 1
- artifact = workflow_update_module._metadata_item_for_block_label(
- ctx.raw_code_artifact_metadata, "directory_flow"
- )
- assert artifact is not None
- assert not workflow_update_module._artifact_declares_goal_values(artifact)
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert ctx.code_authoring_guardrail_reject_count == 1
@pytest.mark.asyncio
- async def test_metadata_selected_page_goto_extra_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_clean_accept_does_not_climb_counter(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.goto("https://example.com/other")
- records = [{"number": "Q-001"}]
- """
- )
+ ctx = _code_only_ctx()
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [_terminal_metadata("quote_flow", "quote result")],
- },
- ctx,
- )
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code = str(_single_code_block(parsed)["code"])
- assert 'page.goto("https://example.com/other")' not in code
- assert 'records = [{"number": "Q-001"}]' not in code
- assert 'await page.locator("#zip").fill(str(zip_code))' in code
- assert result["data"]["imposed_substitutions"]["scrubbed_stale_selected_goal_value_paths"] is True
+ assert ctx.code_authoring_guardrail_reject_count == 0
@pytest.mark.asyncio
- async def test_valid_selected_extraction_suffix_keeps_goal_path_metadata(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = textwrap.dedent(synthesized.code).rstrip() + '\nreturn {"records": [{"number": "Q-001"}]}\n'
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ async def test_output_policy_reject_records_author_time_reject_and_guardrail_count(self) -> None:
+ ctx = _code_only_ctx()
result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
- ],
- },
+ {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},
ctx,
+ allow_missing_credentials=True,
)
- assert result["ok"] is True
- assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
- artifact = ctx.code_artifact_metadata[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
- assert workflow_update_module._artifact_declares_goal_values(artifact)
+ assert result["ok"] is False
+ assert "raw_secret_leak" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.phase == "author_time_reject"
+ assert outcome.reason_code == "output_policy_reject"
+ assert outcome.is_authoritative is True
+ assert outcome.structural_key is not None
+ assert "Output policy blocked" in outcome.observed_evidence_summary
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert ctx.blocker_signal is None
@pytest.mark.asyncio
- async def test_p9_opaque_self_authored_extraction_metadata_is_preserved(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
- submitted_code = (
- textwrap.dedent(synthesized.code).rstrip()
- + f'\n{label}_output = {{"premium": "$123", "eligible": True}}\nreturn {label}_output\n'
- )
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {label}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
- schema = (
- '{"type":"object","properties":{"premium":{"type":"string"},'
- '"eligible":{"type":"boolean"}},"required":["premium","eligible"]}'
- )
- metadata = _terminal_metadata(label, "quote result")
- metadata["claimed_outcomes"][0]["goal_value_paths"] = ["premium", "eligible"]
- metadata["claimed_outcomes"][0]["extraction_schema"] = schema
- metadata["claimed_outcomes"][0]["extraction_schema_provenance"] = "self_authored"
- metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["premium", "eligible"]
- metadata["terminal_verifier_expectations"][0]["extraction_schema"] = schema
- metadata["terminal_verifier_expectations"][0]["extraction_schema_provenance"] = "self_authored"
-
- result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": [metadata]}, ctx)
+ async def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:
+ ctx = _code_only_ctx()
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- assert result["ok"] is True
- assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
- assert "scrubbed_stale_selected_goal_value_paths" not in result["data"]["imposed_substitutions"]
- artifact = ctx.code_artifact_metadata[label]
- assert artifact["claimed_outcomes"][0]["goal_value_paths"] == ["premium", "eligible"]
- assert artifact["terminal_verifier_expectations"][0]["goal_value_paths"] == ["premium", "eligible"]
- assert artifact["claimed_outcomes"][0]["extraction_schema"] == schema
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
- assert len(code_blocks) > 1
- code = str(_code_blocks(parsed)[label]["code"])
- assert f"return {label}_output" in code
- assert all(label + "_output" not in str(block.get("code") or "") for block in code_blocks[:-1])
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ assert ctx.latest_tool_blocker_signal is churn
+ assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
+ assert "safety checks rejected" in churn.user_facing_reason
@pytest.mark.asyncio
- async def test_invalid_selected_extraction_suffix_keeps_goal_path_rejection(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = textwrap.dedent(synthesized.code).rstrip() + "\nheading = 'Review'\n"
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ async def test_nth_reject_defers_to_pre_existing_terminal_blocker(self) -> None:
+ ctx = _code_only_ctx()
+ terminal = _terminal_challenge_signal()
+ ctx.blocker_signal = terminal
+ ctx.latest_tool_blocker_signal = terminal
+
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
+
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ assert ctx.blocker_signal is terminal
+ assert ctx.latest_tool_blocker_signal is terminal
+
+ def test_churn_stop_defers_while_output_contract_ladder_active_then_fires(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.output_contract_actuation_count_by_signature["sig_active"] = 1
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ assert ctx.blocker_signal is None
+ ctx.output_contract_actuation_count_by_signature.clear()
+ workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+
+ def test_churn_stop_defers_while_advisory_grant_held(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.output_contract_actuation_by_signature["sig_granted"] = OutputContractAdvisoryState.GRANTED
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
+ assert ctx.blocker_signal is None
+
+ @pytest.mark.asyncio
+ async def test_mixed_credential_and_unresolved_name_reject_returns_code_repair_progress(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ info_calls: list[tuple[str, dict[str, str | list[str]]]] = []
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [
- _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
- ],
- },
- ctx,
- )
+ def capture_info(event: str, **kwargs: str | list[str]) -> None:
+ info_calls.append((event, kwargs))
+ monkeypatch.setattr(workflow_update_module.LOG, "info", capture_info)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
+
+ assert ctx.code_authoring_guardrail_reject_count == 1
assert result["ok"] is False
- assert "does not return a keyed structure" in result["error"]
- assert "records" in result["error"]
+ assert result["data"]["surface_kind"] == "code_repair_progress"
+ assert "failure_type" not in result["data"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == SANDBOX_UNRESOLVED_NAME_REASON_CODE
+ assert repair_context["block_label"] == "login_with_saved_credential"
+ assert repair_context["unresolved_names"] == ["undefined_helper_0"]
+ assert result["user_facing_summary"] != CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.last_code_authoring_reject_was_credential_priority is False
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+ assert (
+ "copilot code authoring repair context stored",
+ {
+ "reason_code": SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ "block_label": "login_with_saved_credential",
+ "unresolved_names": ["undefined_helper_0"],
+ "parameter_keys": [],
+ "available_parameter_keys": [],
+ "binding_candidates": ["undefined_helper_0"],
+ },
+ ) in info_calls
- def test_rejects_selected_extraction_suffix_page_mutation(self) -> None:
- ctx = _quote_ctx()
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
- submitted_code = textwrap.dedent(synthesized.code).rstrip() + '\nawait page.goto("https://example.com/other")\n'
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
-{textwrap.indent(submitted_code, " " * 18)}
- """
- )
+ @pytest.mark.asyncio
+ async def test_single_credential_priority_reject_defers_to_credential_scout_reply(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
- assert any(
- "extraction suffix contains unscouted browser action" in violation for violation in result.violations
- )
- assert result.substitutions is None
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
@pytest.mark.asyncio
- async def test_sibling_invalid_goal_paths_still_reject(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- submitted = _yaml(
- f"""
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- - block_type: code
- label: summarize_quote
- code: |
- return await page.locator("h1").inner_text()
- """
- )
+ async def test_mixed_credential_and_non_name_guardrail_uses_credential_priority_path(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- result = await _update_workflow(
- {"workflow_yaml": submitted, "code_artifact_metadata": [_terminal_metadata("summarize_quote", "summary")]},
- ctx,
- )
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(1)}, ctx)
assert result["ok"] is False
- assert "summarize_quote" in result["error"]
- assert "flat text blob" in result["error"]
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.last_code_authoring_repair_context is None
+ assert ctx.last_code_authoring_reject_was_credential_priority is True
@pytest.mark.asyncio
- async def test_p10_shaped_stale_metadata_imposes_scout_spine(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _quote_ctx()
- submitted = _yaml(
- """
- title: Quote
- workflow_definition:
- blocks:
- - block_type: code
- label: quote_flow
- code: |
- await page.locator("#zip").fill(str(zip_code))
- await page.locator("#continue").click()
- await page.locator("#electricDate").fill("2026-07-01")
- await page.locator("#electricPlan").select_option("basic")
- await page.locator("#coverage-next").click()
- """
- )
+ async def test_standard_policy_mixed_credential_and_unresolved_name_omits_repair_context(self) -> None:
+ ctx = _standard_ctx()
+ ctx.scout_trajectory = []
- result = await _update_workflow(
- {
- "workflow_yaml": submitted,
- "code_artifact_metadata": [_terminal_metadata("quote_flow", "quote result")],
- },
- ctx,
- )
+ result = await _update_workflow({"workflow_yaml": _distinct_credential_collision_yaml(0)}, ctx)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- code = str(_single_code_block(parsed)["code"])
- assert "#electricDate" not in code
- assert "#electricPlan" not in code
- assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
- assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
+ assert result["ok"] is False
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+ @pytest.mark.asyncio
+ async def test_credential_priority_reject_defers_below_higher_bound(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
-class TestBareDropSupersession:
- def test_selector_refines_css_accepts_identity_qualifiers(self) -> None:
- for candidate in (
- 'button[data-action="status"]',
- "button#submit",
- "button.primary",
- 'button[aria-label="Close ]"]', # a literal ] inside the attribute value must not read as a combinator
- ):
- assert workflow_update_module._selector_refines("button", candidate) is True
+ for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):
+ await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
- def test_selector_refines_role_accepts_named_same_role(self) -> None:
- assert workflow_update_module._selector_refines("role=button", 'role=button[name="Next"]') is True
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+ assert _check_enforcement(ctx) is None
- def test_selector_refines_rejects_positional_structural_and_cross_shape(self) -> None:
- bare = "button"
- for candidate in (
- "button:nth-child(2)",
- "button:nth-of-type(2)",
- "button >> nth=1",
- "button.primary span",
- "button#x + button",
- "button[data-x] > svg",
- "button:visible",
- "button:enabled",
- "button:not(.foo)",
- 'button:has-text("Next")',
- "a[href]",
- "buttonx[id=1]",
- "button",
- ):
- assert workflow_update_module._selector_refines(bare, candidate) is False
- assert workflow_update_module._selector_refines("role=button", 'role=link[name="Next"]') is False
- assert workflow_update_module._selector_refines("role=button", 'button[data-action="x"]') is False
- assert workflow_update_module._selector_refines("button", 'role=button[name="x"]') is False
- assert workflow_update_module._selector_refines("role=button", 'role=button[name="N"] >> nth=1') is False
+ @pytest.mark.asyncio
+ async def test_credential_priority_churn_stashes_credential_blocker_at_higher_bound(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- def test_stable_bare_click_refiner_accepts_only_same_kind_text_and_role_anchors(self) -> None:
- assert (
- _stable_same_kind_bare_click_refiner("button", "xpath=//button[normalize-space()='Check Order Status']")
- is True
- )
- assert _stable_same_kind_bare_click_refiner("button", 'role=button[name="Next"]') is True
- for candidate in (
- "xpath=//a[normalize-space()='Check Order Status']",
- 'role=link[name="Next"]',
- "xpath=(//button[normalize-space()='Check Order Status'])[2]",
- "xpath=//button[contains(normalize-space(), 'Check')]",
- ):
- assert _stable_same_kind_bare_click_refiner("button", candidate) is False
+ result: dict[str, object] = {}
+ for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
+ result = await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
- def test_supersession_true_returns_pairing_record(self) -> None:
- dropped = {
- "reason_code": "ambiguous_bare_selector",
- "tool_name": "click",
- "selector": "button",
- "trajectory_index": 1,
- }
- ctx = _resale_ctx()
- claimed: set[int] = set()
- forgiven, record = workflow_update_module._bare_drop_superseded_on_screen(
- dropped, ctx.scout_trajectory, claimed_refiner_indices=claimed
- )
- assert forgiven is True
- assert record == {
- "dropped_index": 1,
- "dropped_selector": "button",
- "refiner_index": 2,
- "refiner_selector": 'button[data-action="status"]',
- "source_url": _RESALE_URL,
- }
- assert claimed == {2}
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "credential_priority_authoring_churn"
+ assert ctx.latest_tool_blocker_signal is churn
+ assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
- def test_supersession_false_across_different_source_url(self) -> None:
- ctx = _resale_ctx()
- ctx.scout_trajectory[2]["source_url"] = "https://example.com/other"
- dropped = {
- "reason_code": "ambiguous_bare_selector",
- "tool_name": "click",
- "selector": "button",
- "trajectory_index": 1,
- }
- forgiven, record = workflow_update_module._bare_drop_superseded_on_screen(
- dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
- )
- assert forgiven is False
- assert record is None
+ @pytest.mark.asyncio
+ async def test_credential_priority_churn_renders_credential_scout_reply_through_enforcement(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):
+ await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)
- def test_supersession_false_without_later_refiner(self) -> None:
- ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
- dropped = {
- "reason_code": "ambiguous_bare_selector",
- "tool_name": "click",
- "selector": "button",
- "trajectory_index": 1,
- }
- forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
- dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
- )
- assert forgiven is False
+ with pytest.raises(CopilotTurnHalt) as excinfo:
+ _check_enforcement(ctx)
- def test_supersession_false_on_empty_source_url(self) -> None:
- ctx = _resale_ctx()
- ctx.scout_trajectory[1]["source_url"] = ""
- dropped = {
- "reason_code": "ambiguous_bare_selector",
- "tool_name": "click",
- "selector": "button",
- "trajectory_index": 1,
- }
- forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
- dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
- )
- assert forgiven is False
+ assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "credential_priority_authoring_churn"
+ assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY
- def test_supersession_false_on_out_of_bounds_index(self) -> None:
- ctx = _resale_ctx()
- for bad_index in (-1, 99, "1", None):
- dropped = {
- "reason_code": "ambiguous_bare_selector",
- "tool_name": "click",
- "selector": "button",
- "trajectory_index": bad_index,
- }
- forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
- dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
- )
- assert forgiven is False
+ @pytest.mark.asyncio
+ async def test_single_pure_credential_reject_defers_to_credential_scout_reply(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- def test_imposition_forgives_mid_trajectory_bare_drop_and_records_substitution(self) -> None:
- ctx = _resale_ctx()
- result = workflow_update_module._maybe_impose_synthesized_code_block(_resale_submitted_yaml(), ctx)
+ result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(0)}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- forgiven = result.substitutions["forgiven_superseded_bare_drops"]
- assert forgiven == [
- {
- "dropped_index": 1,
- "dropped_selector": "button",
- "refiner_index": 2,
- "refiner_selector": 'button[data-action="status"]',
- "source_url": _RESALE_URL,
- }
- ]
- block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
- assert 'button[data-action=\\"status\\"]' in str(block["code"])
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert "diagnostic_code_safety_errors" not in result["data"]
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
- def test_imposition_keeps_bare_drop_fatal_when_sibling_is_positional(self) -> None:
- ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
- result = workflow_update_module._maybe_impose_synthesized_code_block(
- _resale_submitted_yaml("button:nth-of-type(2)"), ctx
- )
+ @pytest.mark.asyncio
+ async def test_unchanged_persisted_credential_block_does_not_require_new_scout(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ prior_yaml = _safe_credential_collision_yaml(0)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = prior_yaml
+ ctx.scout_trajectory = []
- assert any("ambiguous_bare_selector" in violation for violation in result.violations)
- assert result.substitutions is None
+ result = await _update_workflow({"workflow_yaml": prior_yaml}, ctx, allow_missing_credentials=True)
- def test_imposition_one_refiner_does_not_forgive_two_bare_drops(self) -> None:
- ctx = _resale_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
- {
- "tool_name": "click",
- "selector": 'button[data-action="status"]',
- "source_url": _RESALE_URL,
- "trajectory_index": 3,
- },
- ]
- submitted = _yaml(
- """
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#start").click()
- await page.locator("button[data-action=\\"status\\"]").click()
- """
- )
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert result["ok"] is True
+ assert result.get("user_facing_summary") != CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.code_authoring_guardrail_reject_count == 0
- assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+ @pytest.mark.asyncio
+ async def test_changed_persisted_credential_block_still_requires_new_scout(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _safe_credential_collision_yaml(0)
+ ctx.scout_trajectory = []
- def test_imposition_two_refiners_forgive_two_bare_drops(self) -> None:
- ctx = _resale_ctx()
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
- {
- "tool_name": "click",
- "selector": 'button[data-action="open"]',
- "source_url": _RESALE_URL,
- "trajectory_index": 3,
- },
- {
- "tool_name": "click",
- "selector": 'button[data-action="status"]',
- "source_url": _RESALE_URL,
- "trajectory_index": 4,
- },
- ]
- submitted = _yaml(
- """
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#start").click()
- await page.locator("button[data-action=\\"open\\"]").click()
- await page.locator("button[data-action=\\"status\\"]").click()
- """
- )
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(1)}, ctx)
- assert result.violations == []
- assert result.substitutions is not None
- forgiven = result.substitutions["forgiven_superseded_bare_drops"]
- assert {record["dropped_index"] for record in forgiven} == {1, 2}
- assert {record["refiner_index"] for record in forgiven} == {3, 4}
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
- def test_imposition_two_exact_text_xpath_button_refiners_forgive_two_bare_drops(self) -> None:
- ctx = _resale_ctx()
- first_refiner = "xpath=//button[normalize-space()='Check Order Status']"
- second_refiner = "xpath=//button[normalize-space()='View / Download']"
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#order-id",
- "source_url": _RESALE_URL,
- "typed_length": 6,
- "typed_value": "abc123",
- "trajectory_index": 0,
- },
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
- {"tool_name": "click", "selector": first_refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
- {"tool_name": "click", "selector": second_refiner, "source_url": _RESALE_URL, "trajectory_index": 4},
- ]
- submitted = _yaml(
- """
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#order-id").fill(str(order_id))
- await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
- await page.locator("xpath=//button[normalize-space()='View / Download']").click()
- """
- )
+ @pytest.mark.asyncio
+ async def test_pure_credential_priority_churn_climbs_and_halts(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ result: dict[str, object] = {}
+ for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):
+ result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(index)}, ctx)
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert "diagnostic_code_safety_errors" not in result["data"]
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ assert ctx.code_authoring_guardrail_reject_count == index + 1
- assert result.violations == []
- assert result.substitutions is not None
- forgiven = result.substitutions["forgiven_superseded_bare_drops"]
- assert {record["dropped_index"] for record in forgiven} == {1, 2}
- assert {record["refiner_index"] for record in forgiven} == {3, 4}
- assert {record["refiner_selector"] for record in forgiven} == {first_refiner, second_refiner}
- block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
- assert first_refiner in str(block["code"])
- assert second_refiner in str(block["code"])
+ assert ctx.blocker_signal is None
+ assert _check_enforcement(ctx) is None
- def test_imposition_one_exact_text_xpath_refiner_does_not_forgive_two_bare_drops(self) -> None:
- ctx = _resale_ctx()
- refiner = "xpath=//button[normalize-space()='Check Order Status']"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
- {"tool_name": "click", "selector": refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
- ]
- submitted = _yaml(
- """
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#start").click()
- await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
- """
+ result = await _update_workflow(
+ {"workflow_yaml": _safe_credential_collision_yaml(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1)}, ctx
)
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
+ assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "credential_priority_authoring_churn"
+ with pytest.raises(CopilotTurnHalt) as excinfo:
+ _check_enforcement(ctx)
+ assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
+ assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY
+
+ @pytest.mark.asyncio
+ async def test_identical_authoring_recorded_outcome_still_halts_at_backstop(self) -> None:
+ ctx = _code_only_ctx()
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)
- assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"
- def test_imposition_positional_xpath_refiner_keeps_bare_drop_fatal(self) -> None:
- ctx = _resale_ctx()
- positional_refiner = "xpath=(//button[normalize-space()='Check Order Status'])[2]"
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
- {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
- {"tool_name": "click", "selector": positional_refiner, "source_url": _RESALE_URL, "trajectory_index": 2},
- ]
- submitted = _yaml(
- """
- title: Order status
- workflow_definition:
- blocks:
- - block_type: code
- label: order_status
- code: |
- await page.locator("#start").click()
- await page.locator("xpath=(//button[normalize-space()='Check Order Status'])[2]").click()
- """
- )
+ @pytest.mark.asyncio
+ async def test_accept_after_latched_churn_clears_both_signals(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _code_only_ctx()
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ assert ctx.latest_tool_blocker_signal is churn
- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ _stub_successful_update(monkeypatch)
+ accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
- assert any("ambiguous_bare_selector" in violation for violation in result.violations)
- assert result.substitutions is None
+ assert accepted["ok"] is True
+ assert ctx.code_authoring_guardrail_reject_count == 0
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+ assert _check_enforcement(ctx) is None
- @pytest.mark.asyncio
- async def test_auto_act_non_navigating_reads_role_name(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _auto_act_scout_ctx()
- captured: dict[str, object] = {}
- async def _fake_resolve(
- _ctx: AgentContext, selector: str | None, *, allow_browser_read: bool
- ) -> tuple[str, str]:
- captured["allow_browser_read"] = allow_browser_read
- return "button", "Continue"
+_RESALE_URL = "https://example.com/orders"
+_QUOTE_URL = "https://example.com/quote"
- monkeypatch.setattr(scouting_module, "_resolve_scout_role_name", _fake_resolve)
- async def _same_url(_ctx: AgentContext) -> str:
- return "https://example.com/orders"
+def _resale_ctx(*, refiner_selector: str = 'button[data-action="status"]') -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#order-id",
+ "source_url": _RESALE_URL,
+ "typed_length": 6,
+ "typed_value": "abc123",
+ "role": "textbox",
+ "accessible_name": "Order ID",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "button",
+ "source_url": _RESALE_URL,
+ "trajectory_index": 1,
+ },
+ {
+ "tool_name": "click",
+ "selector": refiner_selector,
+ "source_url": _RESALE_URL,
+ "trajectory_index": 2,
+ },
+ ]
+ return ctx
- monkeypatch.setattr(scouting_module, "_live_working_page_url", _same_url)
- async def _evidence(_ctx: AgentContext, *, url: str) -> dict[str, object] | None:
- return None
+def _resale_submitted_yaml(refiner_selector: str = 'button[data-action="status"]') -> str:
+ escaped = refiner_selector.replace('"', '\\"')
+ return _yaml(
+ f"""
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#order-id").fill(str(order_id))
+ await page.locator("{escaped}").click()
+ """
+ )
- monkeypatch.setattr(scouting_module, "_scout_act_observe_page_evidence", _evidence)
- acted = await scouting_module._auto_act_on_repeat(
- ctx,
- {"data": {}},
- url="https://example.com/orders",
- target={"selector": "#continue", "text": "Continue"},
- )
+def _quote_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#zip",
+ "source_url": _QUOTE_URL,
+ "typed_length": 5,
+ "typed_value": "02110",
+ "role": "textbox",
+ "accessible_name": "ZIP code",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#continue",
+ "source_url": _QUOTE_URL,
+ "trajectory_index": 1,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#coverage-next",
+ "source_url": "https://example.com/quote/coverage",
+ "trajectory_index": 2,
+ },
+ ]
+ return ctx
- assert acted is True
- assert captured["allow_browser_read"] is True
- last = ctx.scout_trajectory[-1]
- assert last["role"] == "button"
- assert last["accessible_name"] == "Continue"
- @pytest.mark.asyncio
- async def test_auto_act_navigating_skips_browser_read(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _auto_act_scout_ctx()
- captured: dict[str, object] = {}
+def _author_time_reject_outcome(reason_code: BuildTestOutcomeReasonCode) -> RecordedBuildTestOutcome:
+ return recorded_outcome_from_author_time_reject(
+ reason_code=reason_code,
+ attempted_block_label=workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
+ block_labels=[workflow_update_module._SYNTHESIZED_BLOCK_LABEL],
+ structural_payload={
+ "reason_code": reason_code,
+ "block_label": workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
+ },
+ observed_evidence_summary=reason_code,
+ )
- async def _fake_resolve(
- _ctx: AgentContext, selector: str | None, *, allow_browser_read: bool
- ) -> tuple[str, str]:
- captured["allow_browser_read"] = allow_browser_read
- return "", ""
- monkeypatch.setattr(scouting_module, "_resolve_scout_role_name", _fake_resolve)
+def _code_blocks(parsed: dict[str, object]) -> dict[str, dict[str, object]]:
+ blocks = [block for block in workflow_blocks(parsed) if str(block.get("block_type") or "").lower() == "code"]
+ return {str(block.get("label") or ""): block for block in blocks}
- urls = iter(["https://example.com/orders", "https://example.com/status"])
- async def _moving_url(_ctx: AgentContext) -> str:
- return next(urls)
+def _submitted_with_sibling_code(sibling_code: str) -> str:
+ indented = textwrap.indent(textwrap.dedent(sibling_code).strip(), " " * 14)
+ return _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ - block_type: code
+ label: preserved_code
+ code: |
+{indented}
+ """
+ )
- monkeypatch.setattr(scouting_module, "_live_working_page_url", _moving_url)
- async def _evidence(_ctx: AgentContext, *, url: str) -> dict[str, object] | None:
- return None
+_METADATA_LESS_DIRECTIVE_EVENT = "copilot_output_contract_spine_structure_directive_emitted"
- monkeypatch.setattr(scouting_module, "_scout_act_observe_page_evidence", _evidence)
+_RECORDED_METADATA_REJECT_BLOCK_LABEL = "validate_public_start_service_path"
- acted = await scouting_module._auto_act_on_repeat(
- ctx,
- {"data": {}},
- url="https://example.com/orders",
- target={"selector": "#continue", "text": "Continue"},
- )
- assert acted is True
- assert captured["allow_browser_read"] is False
+def _recorded_metadata_reject_yaml() -> str:
+ return _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - {{parameter_type: output, key: workflow_output}}
+ blocks:
+ - block_type: code
+ label: {_RECORDED_METADATA_REJECT_BLOCK_LABEL}
+ code: |
+ await page.locator("#start").click()
+ """
+ )
+
+def _metadata_less_output_yaml(selector: str = "#search-submit") -> str:
+ return _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - {{parameter_type: output, key: workflow_output}}
+ blocks:
+ - block_type: code
+ label: extract_provider
+ code: |
+ await page.locator("{selector}").click()
+ """
+ )
+
+
+def _metadata_less_page_read_yaml() -> str:
+ return _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: output, key: workflow_output}
+ blocks:
+ - block_type: code
+ label: extract_provider
+ code: |
+ await page.locator("#search-submit").click()
+ value = await page.locator("#npi").inner_text()
+ return {"npi": value}
+ """
+ )
-def _auto_act_scout_ctx() -> AgentContext:
- ctx = AgentContext.__new__(AgentContext)
- ctx.browser_session_id = None
- ctx.scouted_interactions = []
- ctx.scout_trajectory = []
- ctx.scout_observed_terminal_criterion_ids = set()
- ctx.completion_criteria_turn_state = None
- ctx.discovery_mcp_server = _AutoActClickServer()
- return ctx
+def _metadata_less_two_block_yaml() -> str:
+ return _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: output, key: workflow_output}
+ blocks:
+ - block_type: code
+ label: first_block
+ description: extract the provider data
+ code: |
+ await page.locator("#a").click()
+ - block_type: code
+ label: second_block
+ description: extract the license data
+ code: |
+ await page.locator("#b").click()
+ """
+ )
-class _AutoActClickServer:
- async def call_internal_tool(self, tool_name: str, args: dict[str, object]) -> dict[str, object]:
- return {"ok": True, "data": {"selector": args.get("selector")}}
+class TestMetadataLessRejectArming:
+ def test_metadata_ladder_does_not_defer_ceiling_for_legacy_context(self) -> None:
+ legacy_ctx: Any = SimpleNamespace()
-def _declaration_stamp_ctx() -> CopilotContext:
- ctx = _code_only_ctx()
- ctx.turn_id = "t-decl"
- ctx.scout_trajectory = []
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="c_record",
- outcome="The returned record includes record id.",
- output_path="output.record_id",
- ),
- CompletionCriterion(
- id="c_blocker",
- outcome="A blocker is reported when the site blocks submission.",
- contingent_on="the site blocks submission",
- contingent_antecedent_output_path="output.blocker",
- ),
- ]
- )
- return ctx
+ assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(legacy_ctx, count=8) is False
-def _degraded_declaration_stamp_ctx() -> CopilotContext:
- ctx = _code_only_ctx()
- ctx.turn_id = "t-degraded-decl"
- ctx.scout_trajectory = []
- requested_paths = (
- "output.confirmation_number",
- "output.account_number",
- "output.start_date",
- "output.deposit_amount",
- "output.next_owner",
- )
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- *[
- CompletionCriterion(
- id=f"slot_{index}",
- outcome=f"The returned record includes {path}.",
- request_slot_id=f"slot_{index}",
- pinability="unpinnable",
- mint_disposition="degraded",
- mint_degrade="undecidable_judgment",
- requested_output_floor_rekeyed=True,
- floor_rekeyed_from_path=path,
- )
- for index, path in enumerate(requested_paths)
- ],
- CompletionCriterion(
- id="c_blocker",
- outcome="A blocker is reported when the site blocks submission.",
- contingent_on="the site blocks submission",
- contingent_antecedent_output_path="output.blocker",
- ),
- ]
- )
- return ctx
+class TestWholeTrajectoryImposition:
+ def test_imposes_over_unscouted_browser_fill_in_selected_block(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#electricDate").fill("2026-07-01")
+ await page.locator("#coverage-next").click()
+ """
+ )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-class TestDeclarationContractStamp:
- @pytest.mark.parametrize("empty_value", ["None", "''", "' '", "[]", "{}", "[None]", "[{}]", "{'a': None}"])
- def test_static_empty_observation_value_is_non_run_eligible(self, empty_value: str) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {empty_value}, "blocker": None}}}}')
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.record_id"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths={"output.blocker"},
- )
- ]
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricDate" not in code
+ assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
+ assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- metadata,
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
+ def test_entry_opener_drop_is_instrumented_not_silently_forgiven(self) -> None:
+ ctx = _quote_ctx()
+ ctx.scout_trajectory.insert(
+ 0,
+ {"tool_name": "click", "selector": "button", "source_url": _QUOTE_URL, "trajectory_index": 0},
+ )
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#electricDate").fill("2026-07-01")
+ await page.locator("#coverage-next").click()
+ """
)
- assert evaluation is not None
- assert evaluation.missing_metadata_paths == []
- assert evaluation.missing_schema_paths == []
- assert evaluation.missing_return_paths == []
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
- assert evaluation.can_attempt_run is False
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- @pytest.mark.parametrize("value", ["0", "False", "'record-123'", "[None, 'x']", "{'a': 'x'}"])
- def test_static_value_bearing_observation_remains_run_eligible(self, value: str) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {value}, "blocker": None}}}}')
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.record_id"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths={"output.blocker"},
- )
- ]
+ assert result.violations == []
+ assert result.substitutions is not None
+ forgiven = result.substitutions["forgiven_entry_opener_drops"]
+ assert [record["reason_code"] for record in forgiven] == ["ambiguous_bare_selector"]
+ assert forgiven[0]["forgiveness"] == "entry_opener_superseded_by_locator_provenance"
+ assert any(entry.get("event") == "copilot_imposition_forgave_entry_opener_drop" for entry in logs)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ def test_imposes_over_unscouted_selected_block_extra_click(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#electricDate").click()
+ await page.locator("#coverage-next").click()
+ """
)
- assert evaluation is not None
- assert evaluation.has_deficiencies is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_full_metadata_advisory_rejects_declaration_only_return(self) -> None:
- ctx = _declaration_stamp_ctx()
- required_paths = {"output.record_id", "output.blocker"}
- signature = workflow_update_module._stable_output_contract_key(
- workflow_update_module._output_contract_scope_key(ctx), required_paths
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricDate" not in code
+ assert 'page.locator("#zip")' in code
+ assert 'page.locator("#continue")' in code
+ assert 'page.locator("#coverage-next")' in code
+
+ def test_p10_shaped_selected_surplus_browser_mutations_are_discarded(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#electricPlan").select_option("basic")
+ await page.locator("#electricDate").fill("2026-07-01")
+ await page.locator("#coverage-next").click()
+ """
)
- workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
- workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.record_id"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths={"output.blocker"},
- )
- ]
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- metadata,
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricPlan" not in code
+ assert "#electricDate" not in code
+ assert result.substitutions["source_trajectory_count"] == 3
+ assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
+ assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
+
+ def test_wrong_selected_block_receiver_is_overwritten_by_scout_spine(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#wrongZip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#coverage-next").click()
+ """
)
- assert evaluation is not None
- assert evaluation.missing_metadata_paths == []
- assert evaluation.missing_schema_paths == []
- assert evaluation.missing_return_paths == ["output.record_id"]
- assert evaluation.payload["static_return_advisory_paths"] == []
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
- assert evaluation.can_attempt_run is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- @pytest.mark.asyncio
- async def test_full_metadata_declaration_only_contract_rejects_at_persist_seam(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.record_id"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths={"output.blocker"},
- )
- ]
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#wrongZip" not in code
+ assert 'page.locator("#zip")' in code
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
+ def test_selected_alias_locator_extra_is_discarded_by_imposition(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ target = page.locator("#electricDate")
+ await target.fill("2026-07-01")
+ await page.locator("#coverage-next").click()
+ """
)
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["contract_liveness"] == "value_required"
- assert result["data"]["missing_goal_value_paths"] == []
- assert result["data"]["missing_extraction_schema_paths"] == []
- assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_grant_preflight_rejects_declaration_only_return(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricDate" not in code
+ assert 'target = page.locator("#electricDate")' not in code
+ assert 'page.locator("#zip")' in code
+ assert 'page.locator("#coverage-next")' in code
+
+ def test_selected_helper_extra_is_discarded_by_imposition(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ async def clear(target):
+ await target.evaluate("node => node.remove()")
+ await clear(target=page.locator("#electricDate"))
+ await page.locator("#coverage-next").click()
+ """
)
- assert evaluation is not None
- workflow_update_module._grant_output_contract_advisory_run(ctx, evaluation.canonical_signature)
- preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
- assert evaluation.can_attempt_run is False
- assert preflight is not None
- assert preflight["data"]["reason_code"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in preflight["data"]
- assert (
- workflow_update_module._output_contract_advisory_state(ctx, evaluation.canonical_signature)
- == OutputContractAdvisoryState.UNUSED
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricDate" not in code
+ assert "async def clear" not in code
+
+ def test_selected_dynamic_extra_is_discarded_by_imposition(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await getattr(page.locator("#electricDate"), "fill")("2026-07-01")
+ await page.locator("#coverage-next").click()
+ """
)
- assert evaluation.canonical_signature not in ctx.output_contract_pending_run_evidence
- def test_degraded_requested_slots_make_declaration_only_contract_non_run_eligible(self) -> None:
- ctx = _degraded_declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- formed_yaml,
- metadata,
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
+ assert result.violations == []
+ assert result.substitutions is not None
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "#electricDate" not in code
+ assert "getattr" not in code
- assert applied is False
- assert formed_yaml == workflow_yaml
- assert evaluation is not None
- signature = evaluation.canonical_signature
- assert ctx.output_contract_armed_directive_fingerprint_by_signature.get(signature) == (
- workflow_update_module._VALUE_BEARING_PREARM_FINGERPRINT_PREFIX
- + workflow_update_module._output_contract_structural_fingerprint(workflow_yaml, signature)
+ def test_rejects_extra_changed_block_with_unscouted_browser_mutation(self) -> None:
+ ctx = _records_spine_ctx()
+ submitted = _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#stage-a").click()
+ await page.locator("#stage-b").click()
+ await page.locator("#stage-c").click()
+ - block_type: code
+ label: invented_browser_step
+ code: |
+ await page.locator("#electricDate").fill("2026-07-01")
+ """
)
- assert evaluation.payload["contract_liveness"] == "degraded_empty"
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
- assert evaluation.can_attempt_run is False
- assert "canonical_evaluation_paths" not in evaluation.payload
- assert [row["request_slot_id"] for row in evaluation.payload["degraded_request_slots"]] == [
- "slot_0",
- "slot_1",
- "slot_2",
- "slot_3",
- "slot_4",
- ]
- assert [row["floor_rekeyed_from_path"] for row in evaluation.payload["degraded_request_slots"]] == [
- "output.confirmation_number",
- "output.account_number",
- "output.start_date",
- "output.deposit_amount",
- "output.next_owner",
- ]
- def test_degraded_declaration_only_directive_arm_is_idempotent(self) -> None:
- ctx = _degraded_declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- first_yaml, _, first_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- second_yaml, _, second_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, first_yaml, []
- )
+ assert ctx.spine_imposition_owned_attempt is False
+ assert any("unscouted browser action" in violation for violation in result.violations)
+ assert any("never_captured" in violation for violation in result.violations)
+ assert result.substitutions is None
+ events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
+ assert events and events[0]["kind"] == "never_captured"
- assert first_applied is False
- assert second_applied is False
- assert first_yaml == workflow_yaml
- assert second_yaml == workflow_yaml
- assert len(ctx.output_contract_armed_directive_fingerprint_by_signature) == 1
+ @pytest.mark.parametrize(
+ "sibling_code",
+ [
+ pytest.param(
+ """
+ await page.evaluate("window.localStorage.clear()")
+ """,
+ id="test_rejects_unknown_page_receiver_call",
+ ),
+ pytest.param(
+ """
+ await page.locator("#electricDate").evaluate("node => node.remove()")
+ """,
+ id="test_rejects_unknown_direct_locator_receiver_call",
+ ),
+ pytest.param(
+ """
+ target = page.locator("#electricDate")
+ await target.evaluate("node => node.remove()")
+ """,
+ id="test_rejects_unknown_locator_alias_receiver_call",
+ ),
+ pytest.param(
+ """
+ p = page
+ await p.goto("https://example.com/other")
+ """,
+ id="test_rejects_page_alias_mutation",
+ ),
+ pytest.param(
+ """
+ p = page
+ q = p
+ await q.goto("https://example.com/other")
+ """,
+ id="test_rejects_transitive_page_alias_mutation",
+ ),
+ pytest.param(
+ """
+ target = page.locator("#electricDate")
+ other = target
+ await other.fill("2026-07-01")
+ """,
+ id="test_rejects_transitive_locator_alias_mutation",
+ ),
+ pytest.param(
+ """
+ fill_electric = page.locator("#electricDate").fill
+ await fill_electric("2026-07-01")
+ """,
+ id="test_rejects_bound_method_alias_mutation",
+ ),
+ pytest.param(
+ """
+ fill_electric = page.locator("#electricDate").fill
+ other = fill_electric
+ await other("2026-07-01")
+ """,
+ id="test_rejects_transitive_bound_method_alias_mutation",
+ ),
+ pytest.param(
+ """
+ await getattr(page, "goto")("https://example.com/other")
+ """,
+ id="test_rejects_dynamic_dispatch_on_page",
+ ),
+ pytest.param(
+ """
+ target = page.locator("#electricDate")
+ await getattr(target, "fill")("2026-07-01")
+ """,
+ id="test_rejects_dynamic_dispatch_on_locator_alias",
+ ),
+ pytest.param(
+ """
+ async def clear(target):
+ await target.evaluate("node => node.remove()")
+ await clear(page.locator("#electricDate"))
+ """,
+ id="test_rejects_helper_receiving_browser_object",
+ ),
+ pytest.param(
+ """
+ async def clear(target):
+ await target.evaluate("node => node.remove()")
+ await clear(target=page.locator("#electricDate"))
+ """,
+ id="test_rejects_helper_receiving_browser_keyword_object",
+ ),
+ pytest.param(
+ """
+ async def navigate(page_arg):
+ await page_arg.goto("https://example.com/other")
+ await navigate(page_arg=page)
+ """,
+ id="test_rejects_helper_receiving_page_keyword_object",
+ ),
+ ],
+ )
+ def test_owned_attempt_drops_ambiguous_only_sibling(self, sibling_code: str) -> None:
+ ctx = _quote_ctx()
+ submitted = _submitted_with_sibling_code(sibling_code)
- def test_second_run_request_slot_producer_failure_rejects_blocker_only_return(self) -> None:
- requested_paths = (
- "output.confirmation_number",
- "output.account_number",
- "output.start_date",
- "output.deposit_amount",
- "output.next_owner",
- )
- ctx = _code_only_ctx()
- ctx.turn_id = "t-request-slot-producer-failure"
- ctx.scout_trajectory = []
- ctx.request_policy = request_policy_module._classification_from_raw(
- {
- "completion_criteria": [
- {
- "outcome": f"The returned record includes {path}.",
- "output_path": path,
- "request_slot_source_id": "latest_request",
- "request_slot_source_quote": path,
- }
- for path in requested_paths
- ]
- },
- request_slot_failure_kind="invalid_output",
- )
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ assert result.violations == []
+ assert result.substitutions is not None
+ blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))
+ assert "preserved_code" not in blocks
+ dropped = [log for log in logs if log["event"] == "copilot_spine_stale_rung_dropped"]
+ assert dropped and dropped[0]["dropped_labels"] == ["preserved_code"]
+ assert dropped[0]["dropped_actions"]
- assert ctx.request_policy.request_slot_failure_kind == "invalid_output"
- assert all(criterion.output_path is None for criterion in ctx.request_policy.completion_criteria)
- assert all(criterion.mint_disposition == "degraded" for criterion in ctx.request_policy.completion_criteria)
- assert evaluation is not None
- assert evaluation.payload["contract_liveness"] == "degraded_empty"
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
- assert evaluation.can_attempt_run is False
- assert "canonical_evaluation_paths" not in evaluation.payload
- assert [row["floor_rekeyed_from_path"] for row in evaluation.payload["degraded_request_slots"]] == sorted(
- requested_paths
+ def test_unowned_attempt_still_rejects_ambiguous_browser_mutation(self) -> None:
+ ctx = _records_spine_ctx()
+ submitted = _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#stage-a").click()
+ - block_type: code
+ label: helper_stage
+ code: |
+ target = page.locator("#stage-z")
+ await getattr(target, "fill")("2026-07-01")
+ """
)
- assert {row["request_slot_failure_kind"] for row in evaluation.payload["degraded_request_slots"]} == {
- "invalid_output"
- }
- assert preflight is not None
- assert preflight["data"]["reason_code"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in preflight["data"]
-
- @pytest.mark.asyncio
- async def test_degraded_declaration_only_contract_never_persists_or_dispatches(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _degraded_declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
- persisted = False
- dispatched = False
- async def _unexpected_update(*_args: object, **_kwargs: object) -> dict[str, object]:
- nonlocal persisted
- persisted = True
- return {"ok": True}
-
- async def _unexpected_dispatch(*_args: object, **_kwargs: object) -> dict[str, object]:
- nonlocal dispatched
- dispatched = True
- return {"ok": True}
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(tools_module, "_update_workflow", _unexpected_update)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _unexpected_dispatch)
+ assert ctx.spine_imposition_owned_attempt is False
+ assert any("ambiguous browser action" in violation for violation in result.violations)
+ assert result.substitutions is None
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["extract_entry_output"]}),
+ def test_preserves_simple_extraction_only_block(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ - block_type: code
+ label: summarize_quote
+ code: |
+ heading = await page.locator("h1").inner_text()
+ return {{"heading": heading}}
+ """
)
- parsed = json.loads(result)
- assert parsed["ok"] is False
- assert parsed["data"]["reason_code"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in parsed["data"]
- assert parsed["data"]["contract_liveness"] == "degraded_empty"
- assert persisted is False
- assert dispatched is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_mixed_value_and_declaration_return_remains_valid(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- 'record_id = await page.locator("#record-id").text_content()\nreturn {"output": {"record_id": record_id}}'
+ assert result.violations == []
+ assert result.substitutions is not None
+ blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))
+ assert "#coverage-next" in str(blocks[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]["code"])
+ assert str(blocks["summarize_quote"]["code"]).strip() == (
+ 'heading = await page.locator("h1").inner_text()\nreturn {"heading": heading}'
)
- formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
+ def test_preserves_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
+ ctx = _quote_ctx()
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ metadata = _terminal_metadata(label, "quote")
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
+ ctx.raw_code_artifact_metadata = [metadata]
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip()
+ + '\nheading = await page.locator("h1").inner_text()\nreturn {"heading": heading}\n'
)
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, formed_yaml, metadata)
-
- assert applied is True
- assert evaluation is not None
- assert evaluation.has_deficiencies is False
- code = str(workflow_blocks(parse_workflow_yaml(formed_yaml))[0].get("code") or "")
- assert '"record_id": record_id' in code
- assert '"blocker": None' in code
-
- def test_formation_does_not_create_declaration_only_scaffold(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {}')
-
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
)
- assert applied is True
- assert new_yaml == workflow_yaml
- code = str(workflow_blocks(parse_workflow_yaml(new_yaml))[0].get("code") or "")
- assert "blocker" not in code
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, new_yaml, _metadata)
- assert evaluation is not None
- assert evaluation.can_attempt_run is False
-
- def test_stamp_is_idempotent_across_calls(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")')
-
- first_yaml, _metadata, _applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
- )
- again_yaml, _metadata2, _applied2 = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, first_yaml, []
- )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- assert again_yaml == first_yaml
+ assert result.violations == []
+ assert result.substitutions is not None
+ assert result.substitutions["preserved_extraction_suffix"] is True
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ assert code_blocks[-1]["label"] == label
+ output_code = str(code_blocks[-1]["code"]).strip()
+ assert output_code == 'heading = await page.locator("h1").inner_text()\nreturn {"heading": heading}'
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
- def test_stamp_skips_without_single_owner_block(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _yaml(
- "title: Two blocks\n"
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: first\n"
- " code: |\n"
- ' await page.click("#a")\n'
- " - block_type: code\n"
- " label: second\n"
- " code: |\n"
- ' await page.click("#b")\n'
+ def test_preserves_page_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
+ ctx = _quote_ctx()
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ metadata = _terminal_metadata(label, "quote")
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
+ ctx.raw_code_artifact_metadata = [metadata]
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip()
+ + '\nheading = await page.inner_text("h1")\nreturn {"heading": heading}\n'
)
-
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
- ctx, workflow_yaml, []
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
)
- assert applied is False
- assert new_yaml == workflow_yaml
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_stamp_refused_for_statically_empty_observation_code(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"record_id": None}}')
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ assert result.violations == []
+ assert result.substitutions is not None
+ assert result.substitutions["preserved_extraction_suffix"] is True
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ assert code_blocks[-1]["label"] == label
+ output_code = str(code_blocks[-1]["code"]).strip()
+ assert output_code == 'heading = await page.inner_text("h1")\nreturn {"heading": heading}'
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
- stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
- ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
+ def test_rejects_selected_extraction_suffix_browser_mutation(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip() + '\nawait page.locator("#electricDate").fill("2026-07-01")\n'
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
)
- assert applied is False
- assert stamped_yaml == workflow_yaml
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_stamp_applies_for_runtime_populated_observation_code(self) -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- 'record_id = await page.inner_text("#record")\nreturn {"output": {"record_id": record_id}}'
+ assert any(
+ "extraction suffix contains unscouted browser action" in violation for violation in result.violations
)
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ assert result.substitutions is None
- stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
- ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
+ def test_rejects_selected_extraction_suffix_alias_browser_mutation(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip()
+ + '\ntarget = page.locator("#electricDate")\nawait target.fill("2026-07-01")\n'
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
)
- assert applied is True
- code = str(workflow_blocks(parse_workflow_yaml(stamped_yaml))[0].get("code") or "")
- assert '"blocker": None' in code
-
-
-def test_static_return_synthesis_refuses_statically_none_observation_local() -> None:
- code = "record_id = None"
-
- keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
- code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
- )
-
- assert violations
- assert keyed == code
-
-
-def test_static_return_synthesis_accepts_runtime_populated_observation_local() -> None:
- code = 'record_id = await page.inner_text("#record")'
-
- keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
- code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
- )
-
- assert violations == []
- assert {"output.record_id", "output.blocker"} <= workflow_update_module._code_block_produced_output_paths(keyed)
-
-
-def test_static_return_synthesis_refuses_covered_statically_empty_return() -> None:
- code = 'return {"output": {"record_id": None, "blocker": None}}'
-
- _, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
- code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
- )
-
- assert violations
-
-
-def test_value_bearing_lattice_unpack_after_empty_literal_is_fail_open() -> None:
- code = 'extra = await collect()\nreturn {"output": {"record_id": None, **extra}}'
-
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
-
-
-def test_value_bearing_lattice_top_level_unpack_after_output_key_is_fail_open() -> None:
- code = 'extra = await collect()\nreturn {"output": {"record_id": None}, **extra}'
-
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert any(
+ "extraction suffix contains ambiguous browser action" in violation for violation in result.violations
+ )
+ assert result.substitutions is None
-def test_value_bearing_lattice_empty_literal_after_unpack_stays_refused() -> None:
- code = 'extra = await collect()\nreturn {"output": {**extra, "record_id": None}}'
+ def test_owned_attempt_drops_ambiguous_helper_browser_mutation(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ - block_type: code
+ label: helper_step
+ code: |
+ async def fill(locator, value):
+ await locator.fill(value)
+ target = page.locator("#electricDate")
+ await fill(target, "2026-07-01")
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert result.violations == []
+ assert "helper_step" not in _code_blocks(parse_workflow_yaml(result.workflow_yaml))
- "mutation",
- [
- "payload, extra = await split()",
- "*rest, payload = await split()",
- 'with open("audit.log") as payload:\n pass',
- "try:\n pass\nexcept ValueError as payload:\n pass",
- "if (payload := await build()):\n pass",
- 'payload["output"]["record_id"] = await fetch()',
- 'payload["output"].update(await fetch())',
- 'for item in await rows():\n payload["output"]["items"].append(item)',
- "populate(payload)",
- "payload.output = await fetch()",
- 'alias = payload\nalias["output"]["record_id"] = await fetch()',
- 'alias = payload\nalias["output"].update(await fetch())',
- 'inner = payload["output"]\ninner["record_id"] = await fetch()',
- "alias = payload\npopulate(alias)",
- "first = payload\nsecond = first\nsecond.output = await fetch()",
- ],
-)
-def test_value_bearing_lattice_mutated_literal_is_fail_open(mutation: str) -> None:
- code = f'payload = {{"output": {{"record_id": None}}}}\n{mutation}\nreturn payload'
+ def test_multi_screen_trajectory_persists_in_order_with_proven_locators(self) -> None:
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
- assert workflow_update_module._statically_valueless_return_envelope(code) is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+ assert result.violations == []
+ assert result.substitutions is not None
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ code = str(block["code"])
+ assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
+ assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
-def test_value_bearing_lattice_unmutated_literal_stays_refused() -> None:
- code = 'payload = {"output": {"record_id": None}}\nother = await fetch()\nreturn payload'
+ @pytest.mark.asyncio
+ async def test_changed_selected_browser_action_args_do_not_preserve_submitted_extraction(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = textwrap.dedent(synthesized.code).replace(
+ 'await page.locator("#zip").fill(str(zip_code))',
+ 'await page.locator("#zip").fill("99999")',
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ parameter_keys: [zip_code]
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
- assert workflow_update_module._statically_valueless_return_envelope(code) is True
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote")
+ ],
+ },
+ ctx,
+ )
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert 'await page.locator("#zip").fill(str(zip_code))' in code
+ assert 'await page.locator("#zip").fill("99999")' not in code
-def test_value_bearing_lattice_unmutated_alias_stays_refused() -> None:
- code = 'payload = {"output": {"record_id": None}}\nalias = payload\nreturn payload'
+ @pytest.mark.asyncio
+ async def test_alias_reconciled_selected_spine_preserves_submitted_extraction(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ submitted_code = (
+ textwrap.dedent(synthesized.code).replace("zip_code", "postal_code").rstrip()
+ + f'\n{label}_output = {{"quote": "Q-001"}}\nreturn {label}_output\n'
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ parameters:
+ - {{parameter_type: workflow, workflow_parameter_type: string, key: postal_code, default_value: "02110"}}
+ blocks:
+ - block_type: code
+ label: {label}
+ parameter_keys: [postal_code]
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
- assert workflow_update_module._statically_valueless_return_envelope(code) is True
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ {
+ **_terminal_metadata(label, "quote"),
+ "claimed_outcomes": [
+ {
+ **_terminal_metadata(label, "quote")["claimed_outcomes"][0],
+ "goal_value_paths": ["quote"],
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ **_terminal_metadata(label, "quote")["terminal_verifier_expectations"][0],
+ "goal_value_paths": ["quote"],
+ }
+ ],
+ }
+ ],
+ },
+ ctx,
+ )
+ assert result["ok"] is True
+ substitutions = result["data"]["imposed_substitutions"]
+ assert substitutions["preserved_submitted_extraction_code"] is True
+ assert substitutions["parameter_aliases"] == {"zip_code": "postal_code"}
+ assert "scrubbed_stale_selected_goal_value_paths" not in substitutions
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ block = _code_blocks(parsed)[label]
+ browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ assert all(stage.get("parameter_keys") == ["postal_code"] for stage in code_blocks[:-1])
+ assert "postal_code" in browser_code
+ assert "zip_code" not in browser_code
+ code = str(block["code"])
+ assert f"return {label}_output" in code
+ assert "postal_code" not in code
+ artifact = ctx.code_artifact_metadata[label]
+ assert workflow_update_module._artifact_declares_goal_values(artifact)
- "aliasing",
- [
- 'holder = {}\nholder["slot"] = payload',
- 'holder = {}\nholder["slot"] = payload\nholder["slot"]["output"]["record_id"] = await fetch()',
- 'holder = {}\nholder["slot"] = payload\nholder["slot"]["output"].update(await fetch())',
- 'holder = obj()\nholder.slot = payload\nholder.slot["output"]["record_id"] = await fetch()',
- ],
-)
-def test_value_bearing_lattice_alias_into_container_target_is_fail_open(aliasing: str) -> None:
- # Assigning payload into a subscript/attribute target aliases it, so a mutation through the
- # container could make the return value-bearing; the value name must taint (fail-open), not
- # resolve to its pre-alias empty literal.
- code = f'payload = {{"output": {{"record_id": None}}}}\n{aliasing}\nreturn payload'
+ @pytest.mark.asyncio
+ async def test_metadata_selected_extraction_only_imposes_scout_spine(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"number": "Q-001"}}]
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
- assert workflow_update_module._statically_valueless_return_envelope(code) is False
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
+ ],
+ },
+ ctx,
+ )
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
+ code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ output_code = str(block["code"])
+ assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
+ assert 'await page.locator("#zip").fill(str(zip_code))' in code
+ assert 'await page.locator("#continue").click()' in code
+ assert 'await page.locator("#coverage-next").click()' in code
+ assert 'records = [{"number": "Q-001"}]' in output_code
+ assert 'return {"records": records}' in output_code
+ assert all("records" not in str(stage.get("code") or "") for stage in code_blocks[:-1])
-def test_value_bearing_lattice_nul_byte_source_is_fail_open() -> None:
- code = 'return {"output": {"record_id": None}}\x00'
+ @pytest.mark.asyncio
+ async def test_author_metadata_reject_reopens_changed_collapsed_code_block(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ ctx.update_workflow_called = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("metadata_reject")
+ ctx.workflow_yaml = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "old"}}]
+ """
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"number": "Q-001"}}]
+ """
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
- assert workflow_update_module._statically_valueless_return_envelope(code) is False
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
+ ],
+ },
+ ctx,
+ )
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
+ browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ code = str(block["code"])
+ assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
+ assert result["data"]["imposed_substitutions"]["separated_browser_stage_count"] > 1
+ assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ assert 'await page.locator("#zip").fill(str(zip_code))' in browser_code
+ assert 'await page.locator("#continue").click()' in browser_code
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert 'records = [{"number": "Q-001"}]' not in browser_code
+ assert 'records = [{"number": "Q-001"}]' in code
+ assert 'return {"records": records}' in code
-def test_value_bearing_lattice_deep_alias_chain_is_fail_open() -> None:
- lines = ['v0 = {"output": {"record_id": None}}']
- lines.extend(f"v{index} = v{index - 1}" for index in range(1, 1500))
- lines.append("return v1499")
- code = "\n".join(lines)
+ def test_author_ambiguous_selector_reject_reopens_strict_imposition_with_typed_repair_context(self) -> None:
+ ctx = _resale_ctx()
+ ctx.update_workflow_called = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("code_safety_reject")
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="order_status",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ )
- assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
- assert workflow_update_module._statically_valueless_return_envelope(code) is False
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_resale_submitted_yaml(), ctx)
+ assert result.violations == []
+ assert result.substitutions is not None
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ assert 'button[data-action=\\"status\\"]' in str(block["code"])
- ("code", "expected"),
- [
- ('runtime_value = await collect()\nreturn {"output": runtime_value}', "proven"),
- (
- 'runtime_value = await collect()\npayload = {"output": runtime_value}\nalias = payload\nreturn alias',
- "proven",
- ),
- ("runtime_value = await collect()\nreturn runtime_value", "unknown"),
- ('runtime_value = await collect()\nreturn {"request_id": runtime_value}', "absent"),
- ('runtime_value = await collect()\nreturn {"output.request_id": runtime_value}', "absent"),
- ('runtime_value = await collect()\noutput_key = "output"\nreturn {output_key: runtime_value}', "unknown"),
- (
- 'runtime_value = await collect()\nextra = await collect_extra()\nreturn {"output": runtime_value, **extra}',
- "unknown",
- ),
- (
- 'if should_return_output:\n return {"output": runtime_value}\nreturn {"request_id": runtime_value}',
- "absent",
- ),
- ('return payload\npayload = {"output": runtime_value}', "unknown"),
- ("await page.click('#submit')", "absent"),
- ],
-)
-def test_root_output_envelope_state_requires_literal_root_on_every_return(code: str, expected: str) -> None:
- assert workflow_update_module._root_output_envelope_state(code) == expected
+ def test_author_schema_incompatibility_does_not_reopen_collapsed_code_block(self) -> None:
+ ctx = _quote_ctx()
+ ctx.update_workflow_called = True
+ ctx.synthesized_goal_complete_landed = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("schema_incompatibility")
+ ctx.workflow_yaml = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "old"}}]
+ """
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "pending"}}]
+ """
+ )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
-def test_canonical_output_contract_signature_is_single_keyed_under_degraded_slots() -> None:
- ctx = _degraded_declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- assert evaluation is not None
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- directive_paths = workflow_update_module._value_bearing_directive_paths(contract)
- signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=directive_paths)
-
- assert directive_paths == {"output"}
- assert evaluation.canonical_signature == signature
- assert evaluation.payload["canonical_output_contract_signature"] == signature
- assert workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, directive_paths) == signature
- structural = workflow_update_module._output_contract_author_time_structural_payload(ctx, directive_paths)
- assert structural is not None
- assert structural["canonical_output_contract_signature"] == signature
- workflow_update_module._record_output_contract_family_reject(
- ctx, directive_paths, reject_family="value_bearing_output_required"
- )
- assert ctx.output_contract_reject_count_by_signature == {signature: 1}
- workflow_update_module._record_output_contract_deferral(ctx, directive_paths)
- assert ctx.output_contract_deferral_count_by_signature == {signature: 1}
- identity_free = workflow_update_module._stable_output_contract_key(
- workflow_update_module._output_contract_scope_key(ctx), directive_paths
- )
- assert signature != identity_free
+ assert result.violations == []
+ assert result.substitutions is None
+ assert result.workflow_yaml == submitted
+ @pytest.mark.asyncio
+ async def test_metadata_selected_page_goto_extra_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.goto("https://example.com/other")
+ records = [{"number": "Q-001"}]
+ """
+ )
-class TestValueBearingDefinitionRejectComposition:
- @staticmethod
- def _ctx() -> CopilotContext:
- ctx = _live_root_value_bearing_ctx()
- ctx.request_policy.completion_criteria.append(
- CompletionCriterion(
- id="reusable_inputs",
- outcome="The workflow uses the service address as a reusable input.",
- level="definition",
- output_path="workflow.parameters",
- )
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [_terminal_metadata("quote_flow", "quote result")],
+ },
+ ctx,
)
- return ctx
- @staticmethod
- def _workflow_yaml() -> str:
- return _yaml(
- """
- title: Submit reusable service request
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert 'page.goto("https://example.com/other")' not in code
+ assert 'records = [{"number": "Q-001"}]' not in code
+ assert 'await page.locator("#zip").fill(str(zip_code))' in code
+ assert result["data"]["imposed_substitutions"]["scrubbed_stale_selected_goal_value_paths"] is True
+
+ @pytest.mark.asyncio
+ async def test_valid_selected_extraction_suffix_keeps_goal_path_metadata(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = textwrap.dedent(synthesized.code).rstrip() + '\nreturn {"records": [{"number": "Q-001"}]}\n'
+ submitted = _yaml(
+ f"""
+ title: Quote
workflow_definition:
- parameters:
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
blocks:
- block_type: code
- label: extract_record
- parameter_keys: []
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
code: |
- return {"output": None}
+{textwrap.indent(submitted_code, " " * 18)}
"""
)
- @staticmethod
- def _assert_composed_reject(result: dict[str, object], *, rung: int) -> None:
- data = result["data"]
- assert isinstance(data, dict)
- assert data["reason_code"] == "definition_contract_unsatisfied"
- assert data["unreferenced_parameter_keys"] == ["service_address"]
- assert data["canonical_required_child_paths"] == ["output"]
- assert data["shape_violations"] == ["value_bearing_output_required"]
- assert data["satisfying_templates"]
- assert data["output_contract_actuation"] == "structure_directive"
- assert data["value_bearing_convergence_directive"] == {
- "rung": rung,
- "required_output_paths": ["output"],
- "shape_violations": ["value_bearing_output_required"],
- "satisfying_templates": data["satisfying_templates"],
- }
-
- def test_live_request_policy_definition_reject_composes_root_directive_and_escalates(self) -> None:
- ctx = self._ctx()
- workflow_yaml = self._workflow_yaml()
-
- with capture_logs() as first_logs:
- first = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
- ctx.output_contract_bail_actuated_this_call = False
- with capture_logs() as second_logs:
- second = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
-
- assert first is not None
- assert second is not None
- self._assert_composed_reject(first, rung=1)
- self._assert_composed_reject(second, rung=2)
- first_emits = [log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(first_emits) == 1
- assert len(second_emits) == 1
- assert first_emits[0]["canonical_required_child_paths"] == ["output"]
- assert second_emits[0]["canonical_required_child_paths"] == ["output"]
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.latest_recorded_build_test_outcome.reason_code == "definition_contract_unsatisfied"
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
-
- @pytest.mark.asyncio
- async def test_public_p5_tool_replay_emits_rungs_then_preserves_definition_route(self) -> None:
- ctx = self._ctx()
- workflow_yaml = self._workflow_yaml()
- metadata: list[dict[str, object]] = []
- submitted_metadata = json.dumps(metadata, sort_keys=True)
- tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
- arguments = json.dumps(
+ result = await _update_workflow(
{
- "workflow_yaml": workflow_yaml,
- "block_labels": ["extract_record"],
- "code_artifact_metadata": metadata,
- }
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
+ ],
+ },
+ ctx,
)
- with capture_logs() as logs:
- first = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
- second = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
-
- self._assert_composed_reject(first, rung=1)
- self._assert_composed_reject(second, rung=2)
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert [event["rung"] for event in emits] == [1, 2]
- assert all(event["canonical_required_child_paths"] == ["output"] for event in emits)
- assert emits[0]["guidance_fingerprint"] == emits[1]["guidance_fingerprint"]
- assert json.dumps(metadata, sort_keys=True) == submitted_metadata
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.latest_recorded_build_test_outcome.reason_code == "definition_contract_unsatisfied"
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.phase == "author_time_reject"
- assert requirement.reason_code == "definition_contract_unsatisfied"
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.workflow_yaml == ""
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+ assert result["ok"] is True
+ assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
+ artifact = ctx.code_artifact_metadata[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
+ assert workflow_update_module._artifact_declares_goal_values(artifact)
@pytest.mark.asyncio
- async def test_update_workflow_definition_reject_returns_same_composed_root_directive(self) -> None:
- ctx = self._ctx()
+ async def test_p9_opaque_self_authored_extraction_metadata_is_preserved(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip()
+ + f'\n{label}_output = {{"premium": "$123", "eligible": True}}\nreturn {label}_output\n'
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
+ schema = (
+ '{"type":"object","properties":{"premium":{"type":"string"},'
+ '"eligible":{"type":"boolean"}},"required":["premium","eligible"]}'
+ )
+ metadata = _terminal_metadata(label, "quote result")
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = ["premium", "eligible"]
+ metadata["claimed_outcomes"][0]["extraction_schema"] = schema
+ metadata["claimed_outcomes"][0]["extraction_schema_provenance"] = "self_authored"
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["premium", "eligible"]
+ metadata["terminal_verifier_expectations"][0]["extraction_schema"] = schema
+ metadata["terminal_verifier_expectations"][0]["extraction_schema_provenance"] = "self_authored"
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": self._workflow_yaml(), "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- )
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": [metadata]}, ctx)
- self._assert_composed_reject(result, rung=1)
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert emits[0]["canonical_required_child_paths"] == ["output"]
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.latest_recorded_build_test_outcome.reason_code == "definition_contract_unsatisfied"
- assert ctx.workflow_yaml == ""
+ assert result["ok"] is True
+ assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
+ assert "scrubbed_stale_selected_goal_value_paths" not in result["data"]["imposed_substitutions"]
+ artifact = ctx.code_artifact_metadata[label]
+ assert artifact["claimed_outcomes"][0]["goal_value_paths"] == ["premium", "eligible"]
+ assert artifact["terminal_verifier_expectations"][0]["goal_value_paths"] == ["premium", "eligible"]
+ assert artifact["claimed_outcomes"][0]["extraction_schema"] == schema
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ code = str(_code_blocks(parsed)[label]["code"])
+ assert f"return {label}_output" in code
+ assert all(label + "_output" not in str(block.get("code") or "") for block in code_blocks[:-1])
+ def test_rejects_selected_extraction_suffix_page_mutation(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = textwrap.dedent(synthesized.code).rstrip() + '\nawait page.goto("https://example.com/other")\n'
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
-class TestValueBearingConvergenceDirective:
- @staticmethod
- def _directive(
- data: dict[str, object],
- *,
- paths: list[str],
- rung: int,
- ) -> dict[str, object]:
- return {
- "rung": rung,
- "required_output_paths": paths,
- "shape_violations": ["value_bearing_output_required"],
- "satisfying_templates": data["satisfying_templates"],
- }
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- @staticmethod
- def _ctx() -> CopilotContext:
- ctx = _declaration_stamp_ctx()
- ctx.request_policy.completion_criteria.append(
- CompletionCriterion(
- id="c_account",
- outcome="The returned record includes account number.",
- output_path="output.account_number",
- )
+ assert any(
+ "extraction suffix contains unscouted browser action" in violation for violation in result.violations
)
- return ctx
+ assert result.substitutions is None
- @staticmethod
- def _workflow_and_metadata() -> tuple[str, list[dict[str, object]]]:
- workflow_yaml = _collapsed_spine_yaml(
- 'return {"output": {"record_id": None, "account_number": None, "blocker": None}}'
+ @pytest.mark.asyncio
+ async def test_p10_shaped_stale_metadata_imposes_scout_spine(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ submitted = _yaml(
+ """
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: quote_flow
+ code: |
+ await page.locator("#zip").fill(str(zip_code))
+ await page.locator("#continue").click()
+ await page.locator("#electricDate").fill("2026-07-01")
+ await page.locator("#electricPlan").select_option("basic")
+ await page.locator("#coverage-next").click()
+ """
)
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.record_id", "output.account_number"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths={"output.blocker"},
- )
- ]
- return workflow_yaml, metadata
- @staticmethod
- def _degraded_ctx(*floor_paths: str) -> CopilotContext:
- ctx = _code_only_ctx()
- ctx.turn_id = "t-degraded-value-bearing-directive"
- ctx.scout_trajectory = []
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id=f"slot_{index}",
- outcome=f"The returned record includes slot {index}.",
- output_path="output",
- request_slot_id=f"slot_{index}",
- pinability="unpinnable",
- mint_disposition="degraded",
- mint_degrade="undecidable_judgment",
- requested_output_floor_rekeyed=True,
- floor_rekeyed_from_path=path,
- )
- for index, path in enumerate(floor_paths)
- ]
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [_terminal_metadata("quote_flow", "quote result")],
+ },
+ ctx,
)
- return ctx
- @staticmethod
- def _declaration_only_ctx() -> CopilotContext:
- ctx = _code_only_ctx()
- ctx.turn_id = "t-declaration-only-value-bearing-directive"
- ctx.scout_trajectory = []
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(
- id="c_blocker",
- outcome="A blocker is returned when submission is unavailable.",
- output_path="output.blocker",
- contingent_on="submission is unavailable",
- contingent_antecedent_output_path="output.blocker",
- antecedent_family="blocker",
- )
- ]
- )
- return ctx
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert "#electricDate" not in code
+ assert "#electricPlan" not in code
+ assert code.index('page.locator("#zip")') < code.index('page.locator("#continue")')
+ assert code.index('page.locator("#continue")') < code.index('page.locator("#coverage-next")')
- @staticmethod
- def _live_root_request_policy_ctx() -> CopilotContext:
- return _live_root_value_bearing_ctx()
- @staticmethod
- def _live_root_evaluation(
- ctx: CopilotContext,
- workflow_yaml: str,
- metadata: list[dict[str, object]],
- ) -> workflow_update_module._OutputContractEvaluation:
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- metadata,
- enforce_value_bearing_liveness=True,
- )
- assert evaluation is not None
- assert evaluation.required_paths == {"output"}
- assert evaluation.payload["canonical_required_child_paths"] == ["output"]
- assert evaluation.canonical_signature == workflow_update_module._output_contract_signature(
- ctx=ctx, required_paths={"output"}
- )
- return evaluation
+class TestBareDropSupersession:
+ def test_selector_refines_css_accepts_identity_qualifiers(self) -> None:
+ for candidate in (
+ 'button[data-action="status"]',
+ "button#submit",
+ "button.primary",
+ 'button[aria-label="Close ]"]', # a literal ] inside the attribute value must not read as a combinator
+ ):
+ assert workflow_update_module._selector_refines("button", candidate) is True
- def test_declaration_only_run_preflight_returns_root_family_guidance_and_accepts_exact_reauthor(self) -> None:
- ctx = self._declaration_only_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ def test_selector_refines_role_accepts_named_same_role(self) -> None:
+ assert workflow_update_module._selector_refines("role=button", 'role=button[name="Next"]') is True
- first = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ def test_selector_refines_rejects_positional_structural_and_cross_shape(self) -> None:
+ bare = "button"
+ for candidate in (
+ "button:nth-child(2)",
+ "button:nth-of-type(2)",
+ "button >> nth=1",
+ "button.primary span",
+ "button#x + button",
+ "button[data-x] > svg",
+ "button:visible",
+ "button:enabled",
+ "button:not(.foo)",
+ 'button:has-text("Next")',
+ "a[href]",
+ "buttonx[id=1]",
+ "button",
+ ):
+ assert workflow_update_module._selector_refines(bare, candidate) is False
+ assert workflow_update_module._selector_refines("role=button", 'role=link[name="Next"]') is False
+ assert workflow_update_module._selector_refines("role=button", 'button[data-action="x"]') is False
+ assert workflow_update_module._selector_refines("button", 'role=button[name="x"]') is False
+ assert workflow_update_module._selector_refines("role=button", 'role=button[name="N"] >> nth=1') is False
- assert first is not None and first["ok"] is False
- assert first["data"]["contract_liveness"] == "absent"
- assert first["data"]["canonical_required_child_paths"] == ["output"]
- assert first["data"]["declaration_only_child_paths"] == ["output.blocker"]
- assert first["data"]["reason_code"] == "value_bearing_output_required"
- assert first["data"]["reject_reason"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in first["data"]
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=1
+ def test_stable_bare_click_refiner_accepts_only_same_kind_text_and_role_anchors(self) -> None:
+ assert (
+ _stable_same_kind_bare_click_refiner("button", "xpath=//button[normalize-space()='Check Order Status']")
+ is True
)
- assert first["data"]["satisfying_templates"]["return_skeleton"] == (
- 'return {"output": {"blocker": None, **output_value}}'
+ assert _stable_same_kind_bare_click_refiner("button", 'role=button[name="Next"]') is True
+ for candidate in (
+ "xpath=//a[normalize-space()='Check Order Status']",
+ 'role=link[name="Next"]',
+ "xpath=(//button[normalize-space()='Check Order Status'])[2]",
+ "xpath=//button[contains(normalize-space(), 'Check')]",
+ ):
+ assert _stable_same_kind_bare_click_refiner("button", candidate) is False
+
+ def test_supersession_true_returns_pairing_record(self) -> None:
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "tool_name": "click",
+ "selector": "button",
+ "trajectory_index": 1,
+ }
+ ctx = _resale_ctx()
+ claimed: set[int] = set()
+ forgiven, record = workflow_update_module._bare_drop_superseded_on_screen(
+ dropped, ctx.scout_trajectory, claimed_refiner_indices=claimed
)
- assert first["data"]["satisfying_templates"]["extraction_schema"]["properties"]["output"]["properties"][
- "blocker"
- ] == {"type": ["string", "null"]}
- assert "code_artifact_metadata" in first["error"]
- assert "return skeleton" in first["error"]
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
+ assert forgiven is True
+ assert record == {
+ "dropped_index": 1,
+ "dropped_selector": "button",
+ "refiner_index": 2,
+ "refiner_selector": 'button[data-action="status"]',
+ "source_url": _RESALE_URL,
+ }
+ assert claimed == {2}
- templates = first["data"]["satisfying_templates"]
- reauthored_code = (
- 'output_value = {"status": await page.locator("#status").text_content()}\n' + templates["return_skeleton"]
+ def test_supersession_false_across_different_source_url(self) -> None:
+ ctx = _resale_ctx()
+ ctx.scout_trajectory[2]["source_url"] = "https://example.com/other"
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "tool_name": "click",
+ "selector": "button",
+ "trajectory_index": 1,
+ }
+ forgiven, record = workflow_update_module._bare_drop_superseded_on_screen(
+ dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
)
- reauthored_yaml = _collapsed_spine_yaml(reauthored_code)
- ctx.output_contract_bail_actuated_this_call = False
+ assert forgiven is False
+ assert record is None
- assert (
- workflow_update_module._metadata_contract_run_preflight_reject(
- ctx,
- reauthored_yaml,
- [templates["code_artifact_metadata"]],
- )
- is None
- )
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert (
- workflow_update_module.output_contract_value_bearing_run_reject(
- ctx,
- {"extract_record": reauthored_code},
- )
- is None
+ def test_supersession_false_without_later_refiner(self) -> None:
+ ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "tool_name": "click",
+ "selector": "button",
+ "trajectory_index": 1,
+ }
+ forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
+ dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
)
+ assert forgiven is False
- def test_degraded_empty_run_preflight_returns_root_family_guidance(self) -> None:
- ctx = self._degraded_ctx("output.confirmation_number", "output.account_number")
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
-
- result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
-
- assert result is not None and result["ok"] is False
- assert result["data"]["contract_liveness"] == "degraded_empty"
- assert result["data"]["canonical_required_child_paths"] == ["output"]
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["reject_reason"] == "value_bearing_output_required"
- assert "output_contract_reason_code" not in result["data"]
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"], paths=["output"], rung=1
+ def test_supersession_false_on_empty_source_url(self) -> None:
+ ctx = _resale_ctx()
+ ctx.scout_trajectory[1]["source_url"] = ""
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "tool_name": "click",
+ "selector": "button",
+ "trajectory_index": 1,
+ }
+ forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
+ dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
)
- assert "code_artifact_metadata" in result["error"]
- assert "return skeleton" in result["error"]
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+ assert forgiven is False
- @pytest.mark.parametrize(
- "code",
- [
- 'runtime_value = await collect()\nreturn {"output": runtime_value}',
- 'runtime_value = await collect()\npayload = {"output": runtime_value}\nalias = payload\nreturn alias',
- ],
- )
- def test_root_only_runtime_envelopes_exit_value_bearing_liveness(self, code: str) -> None:
- ctx = self._degraded_ctx("output.confirmation_number")
- workflow_yaml = _collapsed_spine_yaml(code)
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths=set(),
+ def test_supersession_false_on_out_of_bounds_index(self) -> None:
+ ctx = _resale_ctx()
+ for bad_index in (-1, 99, "1", None):
+ dropped = {
+ "reason_code": "ambiguous_bare_selector",
+ "tool_name": "click",
+ "selector": "button",
+ "trajectory_index": bad_index,
+ }
+ forgiven, _ = workflow_update_module._bare_drop_superseded_on_screen(
+ dropped, ctx.scout_trajectory, claimed_refiner_indices=set()
)
+ assert forgiven is False
+
+ def test_imposition_forgives_mid_trajectory_bare_drop_and_records_substitution(self) -> None:
+ ctx = _resale_ctx()
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_resale_submitted_yaml(), ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ forgiven = result.substitutions["forgiven_superseded_bare_drops"]
+ assert forgiven == [
+ {
+ "dropped_index": 1,
+ "dropped_selector": "button",
+ "refiner_index": 2,
+ "refiner_selector": 'button[data-action="status"]',
+ "source_url": _RESALE_URL,
+ }
]
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ assert 'button[data-action=\\"status\\"]' in str(block["code"])
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- metadata,
- enforce_value_bearing_liveness=True,
+ def test_imposition_keeps_bare_drop_fatal_when_sibling_is_positional(self) -> None:
+ ctx = _resale_ctx(refiner_selector="button:nth-of-type(2)")
+ result = workflow_update_module._maybe_impose_synthesized_code_block(
+ _resale_submitted_yaml("button:nth-of-type(2)"), ctx
)
- assert evaluation is not None
- assert "value_bearing_output_required" not in evaluation.shape_violations
- assert evaluation.payload["reason_code"] == "output_contract_required"
- assert workflow_update_module.output_contract_value_bearing_run_reject(ctx, {"extract_record": code}) is None
-
- @pytest.mark.asyncio
- async def test_native_reject_emits_full_sorted_guidance_on_both_rungs(self) -> None:
- ctx = self._ctx()
- workflow_yaml, metadata = self._workflow_and_metadata()
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+ assert result.substitutions is None
- with capture_logs() as first_logs:
- first = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
- with capture_logs() as second_logs:
- second = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ def test_imposition_one_refiner_does_not_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {
+ "tool_name": "click",
+ "selector": 'button[data-action="status"]',
+ "source_url": _RESALE_URL,
+ "trajectory_index": 3,
+ },
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("button[data-action=\\"status\\"]").click()
+ """
+ )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- assert first["ok"] is False
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"],
- paths=["output.account_number", "output.blocker", "output.record_id"],
- rung=1,
- )
- assert "metadata_convergence_directive" not in first["data"]
- assert second["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"],
- paths=["output.account_number", "output.blocker", "output.record_id"],
- rung=2,
- )
- first_emits = [log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(first_emits) == 1
- assert first_emits[0]["directive_family"] == "value_bearing_output_required"
- assert first_emits[0]["block_label"] == "extract_record"
- assert first_emits[0]["canonical_output_contract_signature"]
- assert first_emits[0]["rung"] == 1
- assert len(second_emits) == 1
- assert second_emits[0]["directive_family"] == "value_bearing_output_required"
- assert second_emits[0]["rung"] == 2
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.output_contract_actuation_count_by_signature == {}
- assert ctx.output_contract_declick_attempted_by_signature == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert ctx.workflow_yaml == ""
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
- @pytest.mark.asyncio
- async def test_identical_guidance_escalates_across_candidate_changes_and_logs_both_fingerprints(self) -> None:
- ctx = self._ctx()
- first_workflow, metadata = self._workflow_and_metadata()
- second_workflow = _collapsed_spine_yaml(
- 'return {"output": {"record_id": None, "account_number": None, "unused": None, "blocker": None}}'
+ def test_imposition_two_refiners_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {
+ "tool_name": "click",
+ "selector": 'button[data-action="open"]',
+ "source_url": _RESALE_URL,
+ "trajectory_index": 3,
+ },
+ {
+ "tool_name": "click",
+ "selector": 'button[data-action="status"]',
+ "source_url": _RESALE_URL,
+ "trajectory_index": 4,
+ },
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("button[data-action=\\"open\\"]").click()
+ await page.locator("button[data-action=\\"status\\"]").click()
+ """
)
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- with capture_logs() as first_logs:
- first = await _update_workflow(
- {"workflow_yaml": first_workflow, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
- with capture_logs() as second_logs:
- second = await _update_workflow(
- {"workflow_yaml": second_workflow, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ assert result.violations == []
+ assert result.substitutions is not None
+ forgiven = result.substitutions["forgiven_superseded_bare_drops"]
+ assert {record["dropped_index"] for record in forgiven} == {1, 2}
+ assert {record["refiner_index"] for record in forgiven} == {3, 4}
- first_directive = first["data"]["value_bearing_convergence_directive"]
- second_directive = second["data"]["value_bearing_convergence_directive"]
- assert first_directive == self._directive(
- first["data"], paths=["output.account_number", "output.blocker", "output.record_id"], rung=1
- )
- assert second_directive == self._directive(
- second["data"], paths=["output.account_number", "output.blocker", "output.record_id"], rung=2
+ def test_imposition_two_exact_text_xpath_button_refiners_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ first_refiner = "xpath=//button[normalize-space()='Check Order Status']"
+ second_refiner = "xpath=//button[normalize-space()='View / Download']"
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#order-id",
+ "source_url": _RESALE_URL,
+ "typed_length": 6,
+ "typed_value": "abc123",
+ "trajectory_index": 0,
+ },
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {"tool_name": "click", "selector": first_refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
+ {"tool_name": "click", "selector": second_refiner, "source_url": _RESALE_URL, "trajectory_index": 4},
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#order-id").fill(str(order_id))
+ await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
+ await page.locator("xpath=//button[normalize-space()='View / Download']").click()
+ """
)
- first_emit = next(log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT)
- second_emit = next(log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT)
- assert first_emit["guidance_fingerprint"] == second_emit["guidance_fingerprint"]
- assert first_emit["candidate_fingerprint"] != second_emit["candidate_fingerprint"]
- @pytest.mark.asyncio
- async def test_emitted_directive_clears_stale_author_time_rendering_but_preserves_history(self) -> None:
- ctx = self._ctx()
- workflow_yaml, metadata = self._workflow_and_metadata()
- record_build_test_outcome(ctx, _author_time_reject_outcome("definition_contract_unsatisfied"))
- history = list(ctx.recorded_build_test_outcome_history)
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ assert result.violations == []
+ assert result.substitutions is not None
+ forgiven = result.substitutions["forgiven_superseded_bare_drops"]
+ assert {record["dropped_index"] for record in forgiven} == {1, 2}
+ assert {record["refiner_index"] for record in forgiven} == {3, 4}
+ assert {record["refiner_selector"] for record in forgiven} == {first_refiner, second_refiner}
+ block = _single_code_block(parse_workflow_yaml(result.workflow_yaml))
+ assert first_refiner in str(block["code"])
+ assert second_refiner in str(block["code"])
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"], paths=["output.account_number", "output.blocker", "output.record_id"], rung=1
+ def test_imposition_one_exact_text_xpath_refiner_does_not_forgive_two_bare_drops(self) -> None:
+ ctx = _resale_ctx()
+ refiner = "xpath=//button[normalize-space()='Check Order Status']"
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 2},
+ {"tool_name": "click", "selector": refiner, "source_url": _RESALE_URL, "trajectory_index": 3},
+ ]
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("xpath=//button[normalize-space()='Check Order Status']").click()
+ """
)
- assert ctx.latest_recorded_build_test_outcome is None
- assert ctx.recorded_build_test_outcome_history == history
- @pytest.mark.asyncio
- async def test_degraded_request_slots_emit_only_canonical_root_guidance(self) -> None:
- ctx = self._degraded_ctx("output.confirmation_number", "output.account_number")
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- with capture_logs() as first_logs:
- first = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
- with capture_logs() as second_logs:
- second = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
- assert first["ok"] is False
- assert first["data"]["canonical_required_child_paths"] == ["output"]
- assert "canonical_evaluation_paths" not in first["data"]
- assert [row["floor_rekeyed_from_path"] for row in first["data"]["degraded_request_slots"]] == [
- "output.confirmation_number",
- "output.account_number",
+ def test_imposition_positional_xpath_refiner_keeps_bare_drop_fatal(self) -> None:
+ ctx = _resale_ctx()
+ positional_refiner = "xpath=(//button[normalize-space()='Check Order Status'])[2]"
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#start", "source_url": _RESALE_URL, "trajectory_index": 0},
+ {"tool_name": "click", "selector": "button", "source_url": _RESALE_URL, "trajectory_index": 1},
+ {"tool_name": "click", "selector": positional_refiner, "source_url": _RESALE_URL, "trajectory_index": 2},
]
- assert first["data"]["contract_liveness"] == "degraded_empty"
- assert first["data"]["output_contract_reject_count"] == 1
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=1
- )
- assert first["data"]["output_contract_actuation"] == "structure_directive"
- assert second["data"]["output_contract_reject_count"] == 2
- assert second["data"]["value_bearing_convergence_directive"] == self._directive(
- second["data"], paths=["output"], rung=2
- )
- assert second["data"]["output_contract_actuation"] == "structure_directive"
- first_emits = [log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(first_emits) == 1
- assert len(second_emits) == 1
- assert first_emits[0]["canonical_required_child_paths"] == ["output"]
- assert second_emits[0]["canonical_required_child_paths"] == ["output"]
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.output_contract_actuation_count_by_signature == {}
- assert ctx.output_contract_declick_attempted_by_signature == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert ctx.workflow_yaml == ""
-
- @pytest.mark.asyncio
- async def test_root_only_directives_reach_existing_budget_terminal(self) -> None:
- ctx = self._degraded_ctx("output.confirmation_number", "output.account_number")
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- contract = workflow_update_module._output_contract_required_paths_source(ctx)
- assert contract.union == set()
- sentinel_signature = workflow_update_module._output_contract_signature(
- ctx=ctx, required_paths={workflow_update_module._OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH}
- )
- ctx.output_contract_reject_count_by_signature[sentinel_signature] = (
- workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS - 1
- )
- ctx.output_contract_deferral_count_by_signature[sentinel_signature] = (
- workflow_update_module._MAX_OUTPUT_CONTRACT_DEFERRALS - 1
+ submitted = _yaml(
+ """
+ title: Order status
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: order_status
+ code: |
+ await page.locator("#start").click()
+ await page.locator("xpath=(//button[normalize-space()='Check Order Status'])[2]").click()
+ """
)
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
-
- assert result["data"]["reason_code"] == "output_contract_reject_budget_exhausted"
- assert result["data"]["reject_reason"] == "output_contract_reject_budget_exhausted"
- assert "value_bearing_convergence_directive" not in result["data"]
- assert "output_contract_actuation" not in result["data"]
- assert [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT] == []
- assert ctx.turn_halt is not None
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == "output_contract_reject_budget_exhausted"
- assert (
- ctx.output_contract_reject_count_by_signature[sentinel_signature]
- == workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS
- )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
- def test_live_blocker_budget_routes_to_root_guidance(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- evaluation = self._live_root_evaluation(ctx, workflow_yaml, [])
- blocker_signature = workflow_update_module._output_contract_signature(
- ctx=ctx, required_paths={"output.blocker"}
- )
- ctx.output_contract_reject_count_by_signature[blocker_signature] = (
- workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS
- )
- ctx.output_contract_deferral_count_by_signature[blocker_signature] = (
- workflow_update_module._MAX_OUTPUT_CONTRACT_DEFERRALS
- )
+ assert any("ambiguous_bare_selector" in violation for violation in result.violations)
+ assert result.substitutions is None
- payload = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=[],
- )
-
- root_signature = evaluation.canonical_signature
- assert root_signature != blocker_signature
- assert payload["canonical_required_child_paths"] == ["output"]
- assert payload["declaration_only_child_paths"] == ["output.blocker"]
- assert payload["value_bearing_convergence_directive"] == self._directive(payload, paths=["output"], rung=1)
- assert payload["output_contract_reject_count"] == 1
- assert payload["reason_code"] == "value_bearing_output_required"
- assert ctx.output_contract_reject_count_by_signature[blocker_signature] == (
- workflow_update_module._MAX_OUTPUT_CONTRACT_REJECTS
- )
- assert ctx.output_contract_reject_count_by_signature[root_signature] == 1
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+ @pytest.mark.asyncio
+ async def test_auto_act_non_navigating_reads_role_name(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _auto_act_scout_ctx()
+ captured: dict[str, object] = {}
- def test_emitted_directive_does_not_arm_recorded_outcome_grounding(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- evaluation = self._live_root_evaluation(ctx, workflow_yaml, [])
+ async def _fake_resolve(
+ _ctx: AgentContext, selector: str | None, *, allow_browser_read: bool
+ ) -> tuple[str, str]:
+ captured["allow_browser_read"] = allow_browser_read
+ return "button", "Continue"
- first = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=[],
- )
+ monkeypatch.setattr(scouting_module, "_resolve_scout_role_name", _fake_resolve)
- assert first["value_bearing_convergence_directive"] == self._directive(first, paths=["output"], rung=1)
- assert ctx.latest_recorded_build_test_outcome is None
+ async def _same_url(_ctx: AgentContext) -> str:
+ return "https://example.com/orders"
- ctx.output_contract_bail_actuated_this_call = False
- second = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=[],
- )
+ monkeypatch.setattr(scouting_module, "_live_working_page_url", _same_url)
- assert second["value_bearing_convergence_directive"] == self._directive(first, paths=["output"], rung=2)
- assert second["reason_code"] == "value_bearing_output_required"
- assert ctx.latest_recorded_build_test_outcome is None
+ async def _evidence(_ctx: AgentContext, *, url: str) -> dict[str, object] | None:
+ return None
- @pytest.mark.asyncio
- async def test_mixed_degraded_diagnostics_do_not_invent_child_guidance(self) -> None:
- ctx = self._degraded_ctx("output.account_number", "output")
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
+ monkeypatch.setattr(scouting_module, "_scout_act_observe_page_evidence", _evidence)
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": []},
+ acted = await scouting_module._auto_act_on_repeat(
ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
+ {"data": {}},
+ url="https://example.com/orders",
+ target={"selector": "#continue", "text": "Continue"},
)
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"], paths=["output"], rung=1
- )
- assert "output.account_number" not in result["data"]["canonical_required_child_paths"]
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+ assert acted is True
+ assert captured["allow_browser_read"] is True
+ last = ctx.scout_trajectory[-1]
+ assert last["role"] == "button"
+ assert last["accessible_name"] == "Continue"
@pytest.mark.asyncio
- async def test_observation_paths_take_precedence_over_incomplete_degraded_diagnostics(self) -> None:
- ctx = self._ctx()
- ctx.request_policy.completion_criteria.append(
- CompletionCriterion(
- id="slot_incomplete",
- outcome="The returned record includes an additional slot.",
- output_path="output",
- request_slot_id="slot_incomplete",
- pinability="unpinnable",
- mint_disposition="degraded",
- mint_degrade="undecidable_judgment",
- requested_output_floor_rekeyed=True,
- floor_rekeyed_from_path="output",
- )
- )
- workflow_yaml, metadata = self._workflow_and_metadata()
+ async def test_auto_act_navigating_skips_browser_read(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _auto_act_scout_ctx()
+ captured: dict[str, object] = {}
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ async def _fake_resolve(
+ _ctx: AgentContext, selector: str | None, *, allow_browser_read: bool
+ ) -> tuple[str, str]:
+ captured["allow_browser_read"] = allow_browser_read
+ return "", ""
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"],
- paths=["output.account_number", "output.blocker", "output.record_id"],
- rung=1,
- )
+ monkeypatch.setattr(scouting_module, "_resolve_scout_role_name", _fake_resolve)
- @pytest.mark.asyncio
- @pytest.mark.parametrize("invalid_path", ["", "output"])
- async def test_degraded_diagnostics_are_not_directive_path_authority(self, invalid_path: str) -> None:
- ctx = self._degraded_ctx("output.account_number", invalid_path)
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
+ urls = iter(["https://example.com/orders", "https://example.com/status"])
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- formation_prepared=True,
- )
+ async def _moving_url(_ctx: AgentContext) -> str:
+ return next(urls)
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["output_contract_reject_count"] == 1
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"], paths=["output"], rung=1
- )
- assert result["data"]["output_contract_actuation"] == "structure_directive"
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert emits[0]["canonical_required_child_paths"] == ["output"]
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert ctx.workflow_yaml == ""
+ monkeypatch.setattr(scouting_module, "_live_working_page_url", _moving_url)
- @pytest.mark.asyncio
- async def test_live_request_policy_evaluation_emits_root_guidance_without_invented_children(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- evaluation = self._live_root_evaluation(ctx, workflow_yaml, [])
+ async def _evidence(_ctx: AgentContext, *, url: str) -> dict[str, object] | None:
+ return None
- with capture_logs() as logs:
- result = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=[],
- )
+ monkeypatch.setattr(scouting_module, "_scout_act_observe_page_evidence", _evidence)
- assert result["canonical_required_child_paths"] == ["output"]
- assert "canonical_evaluation_paths" not in result
- assert result["reason_code"] == "value_bearing_output_required"
- assert result["value_bearing_convergence_directive"] == self._directive(result, paths=["output"], rung=1)
- assert result["output_contract_actuation"] == "structure_directive"
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert emits[0]["canonical_required_child_paths"] == ["output"]
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- assert ctx.workflow_yaml == ""
+ acted = await scouting_module._auto_act_on_repeat(
+ ctx,
+ {"data": {}},
+ url="https://example.com/orders",
+ target={"selector": "#continue", "text": "Continue"},
+ )
- @pytest.mark.asyncio
- async def test_live_request_policy_native_preflight_emits_evaluator_root_guidance(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- submitted_metadata = [
- {
- "artifact_id": "code_artifact:extract_record",
- "block_label": "extract_record",
- "claimed_outcomes": [
- {
- "goal_value_paths": [
- "confirmation_number",
- "account_number",
- "selected_start_date",
- "deposit_amount",
- "next_owner",
- ],
- "extraction_schema": json.dumps(
- {
- "type": "object",
- "properties": {
- "confirmation_number": {"type": "string"},
- "account_number": {"type": "string"},
- },
- }
- ),
- }
- ],
- }
- ]
+ assert acted is True
+ assert captured["allow_browser_read"] is False
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": workflow_yaml, "code_artifact_metadata": submitted_metadata},
- ctx,
- allow_missing_credentials=True,
- )
- assert result["data"]["canonical_required_child_paths"] == ["output"]
- assert result["data"]["value_bearing_convergence_directive"] == self._directive(
- result["data"], paths=["output"], rung=1
- )
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert emits[0]["canonical_required_child_paths"] == ["output"]
- assert emits[0]["rung"] == 1
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+def _auto_act_scout_ctx() -> AgentContext:
+ ctx = AgentContext.__new__(AgentContext)
+ ctx.browser_session_id = None
+ ctx.scouted_interactions = []
+ ctx.scout_trajectory = []
+ ctx.scout_observed_terminal_criterion_ids = set()
+ ctx.completion_criteria_turn_state = None
+ ctx.discovery_mcp_server = _AutoActClickServer()
+ return ctx
- def test_p8_bare_dynamic_return_emits_evaluator_root_guidance_on_both_rungs(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- 'rows = await page.locator("[data-testid=quickconnect-row]").all_text_contents()\nreturn rows'
- )
- metadata = [
- workflow_update_module._metadata_contract_template(
- block_label="extract_record",
- required_paths={"output.confirmation_number"},
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths=set(),
- )
- ]
- submitted_bytes = json.dumps(metadata, sort_keys=True)
- with capture_logs() as first_logs:
- first = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
- ctx.output_contract_bail_actuated_this_call = False
- with capture_logs() as second_logs:
- second = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+class _AutoActClickServer:
+ async def call_internal_tool(self, tool_name: str, args: dict[str, object]) -> dict[str, object]:
+ return {"ok": True, "data": {"selector": args.get("selector")}}
+
- assert first is not None
- assert second is not None
- assert first["data"]["reason_code"] == "value_bearing_output_required"
- assert second["data"]["reason_code"] == "value_bearing_output_required"
- assert first["data"]["canonical_required_child_paths"] == ["output"]
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=1
- )
- assert second["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=2
- )
- first_prearms = [
- log for log in first_logs if log["event"] == "copilot_output_contract_value_bearing_directive_emitted"
- ]
- second_prearms = [
- log for log in second_logs if log["event"] == "copilot_output_contract_value_bearing_directive_emitted"
+def _declaration_stamp_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "t-decl"
+ ctx.scout_trajectory = []
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_record",
+ outcome="The returned record includes record id.",
+ output_path="output.record_id",
+ ),
+ CompletionCriterion(
+ id="c_blocker",
+ outcome="A blocker is reported when the site blocks submission.",
+ contingent_on="the site blocks submission",
+ contingent_antecedent_output_path="output.blocker",
+ ),
]
- first_emits = [log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert first_prearms == []
- assert second_prearms == []
- assert len(first_emits) == 1
- assert len(second_emits) == 1
- assert first_emits[0]["rung"] == 1
- assert second_emits[0]["rung"] == 2
- assert json.dumps(metadata, sort_keys=True) == submitted_bytes
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
+ )
+ return ctx
- @pytest.mark.asyncio
- async def test_p8_value_bearing_directive_preserves_recorded_outcome_grounding_route(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- ctx.request_policy.completion_criteria.append(
+
+def _degraded_declaration_stamp_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "t-degraded-decl"
+ ctx.scout_trajectory = []
+ requested_paths = (
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ *[
+ CompletionCriterion(
+ id=f"slot_{index}",
+ outcome=f"The returned record includes {path}.",
+ request_slot_id=f"slot_{index}",
+ pinability="unpinnable",
+ mint_disposition="degraded",
+ mint_degrade="undecidable_judgment",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path=path,
+ )
+ for index, path in enumerate(requested_paths)
+ ],
CompletionCriterion(
- id="p8_reusable_inputs",
- outcome=(
- "The workflow uses business name, contact email, service address, and desired start date "
- "as reusable inputs."
- ),
- level="definition",
- output_path="workflow.parameters",
- )
- )
- parameters_yaml = """
- parameters:
- - {parameter_type: workflow, key: business_name, workflow_parameter_type: string}
- - {parameter_type: workflow, key: contact_email, workflow_parameter_type: string}
- - {parameter_type: workflow, key: service_address, workflow_parameter_type: string}
- - {parameter_type: workflow, key: desired_start_date, workflow_parameter_type: string}
- """
- directive_workflow = _yaml(
- f"""
- title: QuickConnect lookup
- workflow_definition:
-{parameters_yaml}
- blocks:
- - block_type: code
- label: extract_record
- parameter_keys: [business_name, contact_email, service_address, desired_start_date]
- code: |
- rows = await page.locator("[data-testid=quickconnect-row]").all_text_contents()
- reusable_inputs = (business_name, contact_email, service_address, desired_start_date)
- return rows if all(reusable_inputs) else rows
- """
- )
- definition_reject_workflow = _yaml(
- f"""
- title: QuickConnect lookup
- workflow_definition:
-{parameters_yaml}
- blocks:
- - block_type: code
- label: extract_record
- parameter_keys: []
- code: |
- return {{"output": {{"confirmation_number": "QC-1042"}}}}
- """
- )
+ id="c_blocker",
+ outcome="A blocker is reported when the site blocks submission.",
+ contingent_on="the site blocks submission",
+ contingent_antecedent_output_path="output.blocker",
+ ),
+ ]
+ )
+ return ctx
+
+
+class TestDeclarationContractStamp:
+ @pytest.mark.parametrize("empty_value", ["None", "''", "' '", "[]", "{}", "[None]", "[{}]", "{'a': None}"])
+ def test_static_empty_observation_value_is_non_run_eligible(self, empty_value: str) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {empty_value}, "blocker": None}}}}')
metadata = [
workflow_update_module._metadata_contract_template(
block_label="extract_record",
- required_paths={"output.confirmation_number"},
+ required_paths={"output.record_id"},
source="requested_output_contract",
reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths=set(),
+ declaration_paths={"output.blocker"},
)
]
- tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
-
- def arguments(workflow_yaml: str) -> str:
- return json.dumps(
- {
- "workflow_yaml": workflow_yaml,
- "block_labels": ["extract_record"],
- "code_artifact_metadata": metadata,
- }
- )
-
- with capture_logs() as logs:
- first = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments(directive_workflow))
- )
- second = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments(directive_workflow))
- )
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=1
- )
- assert second["data"]["value_bearing_convergence_directive"] == self._directive(
- second["data"], paths=["output"], rung=2
- )
- assert ctx.latest_recorded_build_test_outcome is None
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ metadata,
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
- for _ in range(2):
- definition_reject = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- tool_ctx,
- arguments(definition_reject_workflow),
- )
- )
- assert definition_reject["data"]["reason_code"] == "definition_contract_unsatisfied"
- assert "value_bearing_convergence_directive" not in definition_reject["data"]
-
- assert isinstance(ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome)
- assert ctx.latest_recorded_build_test_outcome.reason_code == "definition_contract_unsatisfied"
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.phase == "author_time_reject"
- assert requirement.reason_code == "definition_contract_unsatisfied"
-
- grounding = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- tool_ctx,
- arguments(definition_reject_workflow),
- )
- )
+ assert evaluation is not None
+ assert evaluation.missing_metadata_paths == []
+ assert evaluation.missing_schema_paths == []
+ assert evaluation.missing_return_paths == []
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
- assert grounding["ok"] is False
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert [event["rung"] for event in emits] == [1, 2]
- assert all(event["canonical_required_child_paths"] == ["output"] for event in emits)
-
- def test_value_bearing_guidance_rejects_non_output_root(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": None}')
- evaluation = self._live_root_evaluation(ctx, workflow_yaml, [])
- payload = dict(evaluation.payload)
- payload["canonical_required_child_paths"] = ["record_id"]
-
- assert workflow_update_module._value_bearing_evaluation_guidance(replace(evaluation, payload=payload)) is None
-
- def test_live_root_native_candidate_churn_preserves_metadata_bytes_and_escalates(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- first_workflow = _collapsed_spine_yaml('return {"output": None}')
- second_workflow = _collapsed_spine_yaml('return {"output": None, "unused": None}')
+ @pytest.mark.parametrize("value", ["0", "False", "'record-123'", "[None, 'x']", "{'a': 'x'}"])
+ def test_static_value_bearing_observation_remains_run_eligible(self, value: str) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {value}, "blocker": None}}}}')
metadata = [
workflow_update_module._metadata_contract_template(
block_label="extract_record",
- required_paths={"output.confirmation_number"},
+ required_paths={"output.record_id"},
source="requested_output_contract",
reason_code="requested_output_contract_missing_output_coverage",
- declaration_paths=set(),
+ declaration_paths={"output.blocker"},
)
]
- submitted_bytes = json.dumps(metadata, sort_keys=True)
- first_evaluation = self._live_root_evaluation(ctx, first_workflow, metadata)
- second_evaluation = self._live_root_evaluation(ctx, second_workflow, metadata)
- with capture_logs() as logs:
- first = workflow_update_module._record_output_contract_reject(
- ctx,
- first_evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=first_workflow,
- raw_metadata=metadata,
- )
- ctx.output_contract_bail_actuated_this_call = False
- with capture_logs() as second_logs:
- second = workflow_update_module._record_output_contract_reject(
- ctx,
- second_evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=second_workflow,
- raw_metadata=metadata,
- )
-
- assert first["reason_code"] == "value_bearing_output_required"
- assert second["reason_code"] == "value_bearing_output_required"
- assert first["canonical_required_child_paths"] == ["output"]
- assert first["value_bearing_convergence_directive"] == self._directive(first, paths=["output"], rung=1)
- assert second["value_bearing_convergence_directive"] == self._directive(first, paths=["output"], rung=2)
- assert "metadata_convergence_directive" not in first
- assert "metadata_convergence_directive" not in second
- assert json.dumps(metadata, sort_keys=True) == submitted_bytes
- emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- second_emits = [log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
- assert len(emits) == 1
- assert len(second_emits) == 1
- assert emits[0]["canonical_required_child_paths"] == ["output"]
- assert emits[0]["guidance_fingerprint"] == second_emits[0]["guidance_fingerprint"]
- assert emits[0]["candidate_fingerprint"] != second_emits[0]["candidate_fingerprint"]
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
- @pytest.mark.asyncio
- async def test_full_native_prearm_preserves_rung_across_candidate_churn(self) -> None:
- ctx = self._live_root_request_policy_ctx()
- first_workflow = _collapsed_spine_yaml('return {"output": None}')
- second_workflow = _collapsed_spine_yaml('return {"output": None, "unused": None}')
-
- with capture_logs() as first_logs:
- first = await _update_workflow(
- {"workflow_yaml": first_workflow, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- )
- with capture_logs() as second_logs:
- second = await _update_workflow(
- {"workflow_yaml": second_workflow, "code_artifact_metadata": []},
- ctx,
- allow_missing_credentials=True,
- )
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is False
- assert first["data"]["value_bearing_convergence_directive"] == self._directive(
- first["data"], paths=["output"], rung=1
- )
- assert second["data"]["value_bearing_convergence_directive"] == self._directive(
- second["data"], paths=["output"], rung=2
+ def test_full_metadata_advisory_rejects_declaration_only_return(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ required_paths = {"output.record_id", "output.blocker"}
+ signature = workflow_update_module._stable_output_contract_key(
+ workflow_update_module._output_contract_scope_key(ctx), required_paths
)
- first_prearms = [
- log for log in first_logs if log["event"] == "copilot_output_contract_value_bearing_directive_emitted"
- ]
- second_prearms = [
- log for log in second_logs if log["event"] == "copilot_output_contract_value_bearing_directive_emitted"
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
+ workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
+ metadata = [
+ workflow_update_module._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ declaration_paths={"output.blocker"},
+ )
]
- first_emit = next(log for log in first_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT)
- second_emit = next(log for log in second_logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT)
- assert len(first_prearms) == 1
- assert len(second_prearms) == 1
- assert first_emit["guidance_fingerprint"] == second_emit["guidance_fingerprint"]
- assert first_emit["candidate_fingerprint"] != second_emit["candidate_fingerprint"]
- assert ctx.output_contract_armed_directive_fingerprint_by_signature == {
- first["data"]["canonical_output_contract_signature"]: second_emit["guidance_fingerprint"]
- }
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
- def test_changed_evaluation_guidance_resets_to_rung_one(self) -> None:
- ctx = self._ctx()
- workflow_yaml, metadata = self._workflow_and_metadata()
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
ctx,
workflow_yaml,
metadata,
+ allow_static_return_advisory=True,
enforce_value_bearing_liveness=True,
)
+
assert evaluation is not None
+ assert evaluation.missing_metadata_paths == []
+ assert evaluation.missing_schema_paths == []
+ assert evaluation.missing_return_paths == ["output.record_id"]
+ assert evaluation.payload["static_return_advisory_paths"] == []
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
- first = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=metadata,
- )
- changed_templates = dict(evaluation.payload["satisfying_templates"])
- changed_templates["return_skeleton"] = 'return {"output": {"record_id": refreshed_record_id}}'
- changed_payload = dict(evaluation.payload)
- changed_payload["satisfying_templates"] = changed_templates
- ctx.output_contract_bail_actuated_this_call = False
-
- changed = workflow_update_module._record_output_contract_reject(
- ctx,
- replace(evaluation, payload=changed_payload),
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=metadata,
- )
+ def test_degraded_requested_slots_make_declaration_only_contract_non_run_eligible(self) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
- assert first["value_bearing_convergence_directive"]["rung"] == 1
- assert changed["value_bearing_convergence_directive"] == self._directive(
- changed_payload,
- paths=["output.account_number", "output.blocker", "output.record_id"],
- rung=1,
+ formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
-
- def test_value_bearing_prearm_and_metadata_family_state_do_not_escalate_first_rung(self) -> None:
- workflow_yaml, metadata = self._workflow_and_metadata()
- ctx = self._ctx()
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
ctx,
- workflow_yaml,
+ formed_yaml,
metadata,
+ allow_static_return_advisory=True,
enforce_value_bearing_liveness=True,
)
+
+ assert applied is False
+ assert formed_yaml == workflow_yaml
assert evaluation is not None
signature = evaluation.canonical_signature
- workflow_update_module._arm_output_contract_value_bearing_directive(
- ctx,
- signature=signature,
- workflow_yaml=workflow_yaml,
- liveness=workflow_update_module._OutputContractLiveness.DEGRADED_EMPTY,
+ assert ctx.output_contract_armed_directive_fingerprint_by_signature.get(signature) == (
+ workflow_update_module._VALUE_BEARING_PREARM_FINGERPRINT_PREFIX
+ + workflow_update_module._output_contract_structural_fingerprint(workflow_yaml, signature)
+ )
+ assert evaluation.payload["contract_liveness"] == "degraded_empty"
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+ assert "canonical_evaluation_paths" not in evaluation.payload
+ assert [row["request_slot_id"] for row in evaluation.payload["degraded_request_slots"]] == [
+ "slot_0",
+ "slot_1",
+ "slot_2",
+ "slot_3",
+ "slot_4",
+ ]
+ assert [row["floor_rekeyed_from_path"] for row in evaluation.payload["degraded_request_slots"]] == [
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ ]
+
+ def test_degraded_declaration_only_directive_arm_is_idempotent(self) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+
+ first_yaml, _, first_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ second_yaml, _, second_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, first_yaml, []
)
- ctx.output_contract_bail_actuated_this_call = False
- payload = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- authored_structural_fingerprint=workflow_update_module._output_contract_structural_fingerprint(
- workflow_yaml, signature
- ),
- workflow_yaml=workflow_yaml,
- raw_metadata=None,
+ assert first_applied is False
+ assert second_applied is False
+ assert first_yaml == workflow_yaml
+ assert second_yaml == workflow_yaml
+ assert len(ctx.output_contract_armed_directive_fingerprint_by_signature) == 1
+
+ def test_mixed_value_and_declaration_return_remains_valid(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ 'record_id = await page.locator("#record-id").text_content()\nreturn {"output": {"record_id": record_id}}'
+ )
+
+ formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, formed_yaml, metadata)
+
+ assert applied is True
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is False
+ code = str(workflow_blocks(parse_workflow_yaml(formed_yaml))[0].get("code") or "")
+ assert '"record_id": record_id' in code
+ assert '"blocker": None' in code
+
+ def test_formation_does_not_create_declaration_only_scaffold(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {}')
+
+ new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
- assert payload["value_bearing_convergence_directive"]["rung"] == 1
- assert "metadata_convergence_directive" not in payload
+ assert applied is True
+ assert new_yaml == workflow_yaml
+ code = str(workflow_blocks(parse_workflow_yaml(new_yaml))[0].get("code") or "")
+ assert "blocker" not in code
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, new_yaml, _metadata)
+ assert evaluation is not None
+ assert evaluation.can_attempt_run is False
+
+ def test_stamp_is_idempotent_across_calls(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")')
- baseline_ctx = _code_only_ctx()
- baseline = workflow_update_module._adjudicate_metadata_reject_ladder(
- baseline_ctx,
- workflow_yaml=_metadata_less_output_yaml(),
- raw_metadata=None,
- missing_labels=["extract_provider"],
- required_paths={"output.npi"},
+ first_yaml, _metadata, _applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ again_yaml, _metadata2, _applied2 = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, first_yaml, []
)
- crossed_ctx = _code_only_ctx()
- crossed_signature = workflow_update_module._output_contract_signature(
- ctx=crossed_ctx, required_paths={"output.npi"}
+
+ assert again_yaml == first_yaml
+
+ def test_stamp_skips_without_single_owner_block(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _yaml(
+ "title: Two blocks\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: first\n"
+ " code: |\n"
+ ' await page.click("#a")\n'
+ " - block_type: code\n"
+ " label: second\n"
+ " code: |\n"
+ ' await page.click("#b")\n'
)
- crossed_ctx.output_contract_armed_directive_fingerprint_by_signature[crossed_signature] = (
- "value-bearing-family-fingerprint"
+
+ new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
)
- crossed = workflow_update_module._adjudicate_metadata_reject_ladder(
- crossed_ctx,
- workflow_yaml=_metadata_less_output_yaml(),
- raw_metadata=None,
- missing_labels=["extract_provider"],
- required_paths={"output.npi"},
+
+ assert applied is False
+ assert new_yaml == workflow_yaml
+
+ def test_stamp_refused_for_statically_empty_observation_code(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"record_id": None}}')
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
+ ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
)
- assert baseline == crossed
- assert json.dumps(baseline, sort_keys=True) == json.dumps(crossed, sort_keys=True)
- assert baseline is not None and baseline["rung"] == 1
+ assert applied is False
+ assert stamped_yaml == workflow_yaml
- def test_empty_signature_retains_fail_closed_reject_without_directive(self) -> None:
- ctx = self._ctx()
- workflow_yaml, metadata = self._workflow_and_metadata()
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- metadata,
- enforce_value_bearing_liveness=True,
+ def test_stamp_applies_for_runtime_populated_observation_code(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ 'record_id = await page.inner_text("#record")\nreturn {"output": {"record_id": record_id}}'
)
- assert evaluation is not None
- evaluation = replace(evaluation, canonical_signature="")
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
+ ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
+ )
+
+ assert applied is True
+ code = str(workflow_blocks(parse_workflow_yaml(stamped_yaml))[0].get("code") or "")
+ assert '"blocker": None' in code
+
+
+def test_static_return_synthesis_refuses_statically_none_observation_local() -> None:
+ code = "record_id = None"
+
+ keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
+ )
+
+ assert violations
+ assert keyed == code
+
+
+def test_static_return_synthesis_accepts_runtime_populated_observation_local() -> None:
+ code = 'record_id = await page.inner_text("#record")'
+
+ keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
+ )
- payload = workflow_update_module._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-bearing output is still required.",
- workflow_yaml=workflow_yaml,
- raw_metadata=metadata,
- )
+ assert violations == []
+ assert {"output.record_id", "output.blocker"} <= workflow_update_module._code_block_produced_output_paths(keyed)
- assert payload["reason_code"] == "value_bearing_output_required"
- assert "value_bearing_convergence_directive" not in payload
- assert "output_contract_actuation" not in payload
- assert ctx.output_contract_reject_count_by_signature
- assert ctx.output_contract_armed_directive_fingerprint_by_signature == {}
+def test_static_return_synthesis_refuses_covered_statically_empty_return() -> None:
+ code = 'return {"output": {"record_id": None, "blocker": None}}'
-def test_non_value_deficiency_keeps_granted_advisory_arming_at_preflight() -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml(
- 'record_id = await page.locator("#record-id").text_content()\n'
- 'return {"output": {"record_id": record_id, "blocker": None}}'
- )
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
+ _, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
)
- assert evaluation is not None
- assert evaluation.has_deficiencies is True
- assert evaluation.can_attempt_run is False
- assert "value_bearing_output_required" not in evaluation.shape_violations
- workflow_update_module._grant_output_contract_advisory_run(ctx, evaluation.canonical_signature)
- preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+ assert violations
+
- assert preflight is None
- assert evaluation.canonical_signature in ctx.output_contract_pending_run_evidence
- assert (
- workflow_update_module._output_contract_advisory_state(ctx, evaluation.canonical_signature)
- == OutputContractAdvisoryState.GRANTED
- )
+def test_value_bearing_lattice_unpack_after_empty_literal_is_fail_open() -> None:
+ code = 'extra = await collect()\nreturn {"output": {"record_id": None, **extra}}'
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
-def test_degraded_criteria_reject_steers_to_root_output_repair() -> None:
- ctx = _degraded_declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- assert evaluation is not None
- assert evaluation.payload["contract_liveness"] == "degraded_empty"
- result = workflow_update_module._output_contract_reject_result(evaluation)
+def test_value_bearing_lattice_top_level_unpack_after_output_key_is_fail_open() -> None:
+ code = 'extra = await collect()\nreturn {"output": {"record_id": None}, **extra}'
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["canonical_required_child_paths"] == ["output"]
- assert result["data"]["declaration_only_child_paths"] == ["output.blocker"]
- assert "code_artifact_metadata" in result["error"]
- assert "return skeleton" in result["error"]
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
-def test_hollow_code_under_live_contract_keeps_code_repair_steering() -> None:
- ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('return {"output": {"record_id": None, "blocker": None}}')
- evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
- ctx,
- workflow_yaml,
- [],
- allow_static_return_advisory=True,
- enforce_value_bearing_liveness=True,
- )
- assert evaluation is not None
- assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+def test_value_bearing_lattice_empty_literal_after_unpack_stays_refused() -> None:
+ code = 'extra = await collect()\nreturn {"output": {**extra, "record_id": None}}'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
- result = workflow_update_module._output_contract_reject_result(evaluation)
- assert "code_artifact_metadata" in result["error"]
+ ("code", "expected"),
+ [
+ ('runtime_value = await collect()\nreturn {"output": runtime_value}', "proven"),
+ (
+ 'runtime_value = await collect()\npayload = {"output": runtime_value}\nalias = payload\nreturn alias',
+ "proven",
+ ),
+ ("runtime_value = await collect()\nreturn runtime_value", "unknown"),
+ ('runtime_value = await collect()\nreturn {"request_id": runtime_value}', "absent"),
+ ('runtime_value = await collect()\nreturn {"output.request_id": runtime_value}', "absent"),
+ ('runtime_value = await collect()\noutput_key = "output"\nreturn {output_key: runtime_value}', "unknown"),
+ (
+ 'runtime_value = await collect()\nextra = await collect_extra()\nreturn {"output": runtime_value, **extra}',
+ "unknown",
+ ),
+ (
+ 'if should_return_output:\n return {"output": runtime_value}\nreturn {"request_id": runtime_value}',
+ "absent",
+ ),
+ ('return payload\npayload = {"output": runtime_value}', "unknown"),
+ ("await page.click('#submit')", "absent"),
+ ],
+)
+def test_root_output_envelope_state_requires_literal_root_on_every_return(code: str, expected: str) -> None:
+ assert workflow_update_module._root_output_envelope_state(code) == expected
def _saved_code_workflow(
@@ -18060,51 +10038,6 @@ class TestValueBearingDispatchPreflight:
)
return prepare_workflow
- @pytest.mark.asyncio
- async def test_dispatch_of_saved_degraded_blocker_only_workflow_is_refused(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _degraded_declaration_stamp_ctx()
- ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
- prepare_workflow = self._guarded_app(monkeypatch)
-
- result = await run_execution_module._run_blocks_and_collect_debug(
- {"block_labels": ["finalize_service"], "parameters": {}}, ctx
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["contract_liveness"] == "degraded_empty"
- assert result["data"]["can_attempt_run"] is False
- assert [row["floor_rekeyed_from_path"] for row in result["data"]["degraded_request_slots"]] == [
- "output.confirmation_number",
- "output.account_number",
- "output.start_date",
- "output.deposit_amount",
- "output.next_owner",
- ]
- assert result["user_facing_summary"]
- assert ctx.last_executed_block_labels == []
- prepare_workflow.assert_not_called()
-
- @pytest.mark.asyncio
- async def test_dispatch_of_statically_hollow_code_under_value_contract_is_refused(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _declaration_stamp_ctx()
- ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
- prepare_workflow = self._guarded_app(monkeypatch)
-
- result = await run_execution_module._run_blocks_and_collect_debug(
- {"block_labels": ["finalize_service"], "parameters": {}}, ctx
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["contract_liveness"] == "value_required"
- assert result["data"]["canonical_required_child_paths"] == ["output.blocker", "output.record_id"]
- prepare_workflow.assert_not_called()
-
@pytest.mark.asyncio
async def test_dispatch_of_mixed_contract_workflow_passes_preflight(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _declaration_stamp_ctx()
@@ -18178,179 +10111,6 @@ class TestValueBearingDispatchPreflight:
assert result == {"ok": False, "error": "Organization not found"}
- @pytest.mark.asyncio
- async def test_all_hollow_workflow_is_refused_on_any_selected_subset(self, monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _declaration_stamp_ctx()
- ctx.staged_workflow = _saved_code_workflow(
- 'return {"output": {"blocker": None}}',
- label="finalize_service",
- extra={"extract_record": 'return {"output": {"record_id": None}}'},
- )
- prepare_workflow = self._guarded_app(monkeypatch)
-
- result = await run_execution_module._run_blocks_and_collect_debug(
- {"block_labels": ["extract_record"], "parameters": {}}, ctx
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert ctx.last_executed_block_labels == []
- prepare_workflow.assert_not_called()
-
- @pytest.mark.asyncio
- async def test_dispatch_with_drifted_empty_criteria_source_refuses_hollow_code(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(completion_criteria=[])
- ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
- prepare_workflow = self._guarded_app(monkeypatch)
-
- result = await run_execution_module._run_blocks_and_collect_debug(
- {"block_labels": ["finalize_service"], "parameters": {}}, ctx
- )
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "value_bearing_output_required"
- assert result["data"]["contract_liveness"] == "absent"
- assert result["data"]["statically_empty_return_envelope"] is True
- assert result["data"]["can_attempt_run"] is False
- assert result["user_facing_summary"]
- prepare_workflow.assert_not_called()
-
- def test_run_reject_fails_open_for_standard_policy(self) -> None:
- ctx = _declaration_stamp_ctx()
- ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx, {"finalize_service": 'return {"output": {"blocker": None}}'}
- )
-
- assert reject is None
-
- def test_run_reject_fails_open_for_runtime_populated_code(self) -> None:
- ctx = _declaration_stamp_ctx()
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx,
- {"finalize_service": 'record_id = await fetch()\nreturn {"output": {"record_id": record_id}}'},
- )
-
- assert reject is None
-
- def test_run_reject_fails_open_when_any_selected_block_may_bear_value(self) -> None:
- ctx = _declaration_stamp_ctx()
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx,
- {
- "finalize_service": 'return {"output": {"blocker": None}}',
- "extract_record": 'record_id = await fetch()\nreturn {"output": {"record_id": record_id}}',
- },
- )
-
- assert reject is None
-
- def test_run_reject_refuses_when_every_selected_block_is_statically_hollow(self) -> None:
- ctx = _declaration_stamp_ctx()
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx,
- {
- "finalize_service": 'return {"output": {"blocker": None}}',
- "extract_record": 'return {"output": {"record_id": None, "blocker": None}}',
- },
- )
-
- assert reject is not None
- assert reject["ok"] is False
- assert reject["data"]["reason_code"] == "value_bearing_output_required"
-
- @pytest.mark.parametrize(
- "code",
- [
- 'return {"output": {"blocker": None}}',
- 'return {"output": {}}',
- 'return {"output": None, "note": ""}',
- 'payload = {"output": {"blocker": None}}\nreturn payload',
- ],
- )
- def test_run_reject_refuses_statically_empty_envelope_with_zero_criteria(self, code: str) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(completion_criteria=[])
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(ctx, {"finalize_service": code})
-
- assert reject is not None
- assert reject["data"]["reason_code"] == "value_bearing_output_required"
- assert reject["data"]["statically_empty_return_envelope"] is True
-
- @pytest.mark.parametrize(
- "code",
- [
- 'return {"output": {"confirmation_number": "CN-1042"}}',
- 'value = await fetch()\nreturn {"output": {"confirmation_number": value}}',
- "return await collect()",
- 'await page.goto("https://example.com/plans")',
- ],
- )
- def test_run_reject_fails_open_with_zero_criteria_when_code_may_bear_value(self, code: str) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(completion_criteria=[])
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(ctx, {"finalize_service": code})
-
- assert reject is None
-
- def test_run_reject_fails_open_with_zero_criteria_when_any_block_may_bear_value(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(completion_criteria=[])
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx,
- {
- "finalize_service": 'return {"output": {"blocker": None}}',
- "extract_record": 'value = await fetch()\nreturn {"output": {"record_id": value}}',
- },
- )
-
- assert reject is None
-
- def test_run_reject_degraded_criteria_message_names_root_output_repair(self) -> None:
- ctx = _degraded_declaration_stamp_ctx()
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx, {"finalize_service": 'return {"output": {"blocker": None}}'}
- )
-
- assert reject is not None
- assert "non-empty top-level `output`" in reject["error"]
- assert "Revise the owning code block" in reject["error"]
- assert reject["data"]["reason_code"] == "value_bearing_output_required"
-
- def test_run_reject_zero_criteria_message_never_references_a_contract(self) -> None:
- ctx = _code_only_ctx()
- ctx.request_policy = RequestPolicy(completion_criteria=[])
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx, {"finalize_service": 'return {"output": {}}'}
- )
-
- assert reject is not None
- assert "output contract" not in reject["error"]
- assert "statically returns no values" in reject["error"]
- assert reject["data"]["reason_code"] == "value_bearing_output_required"
-
- def test_run_reject_hollow_code_under_live_contract_keeps_code_side_steering(self) -> None:
- ctx = _declaration_stamp_ctx()
-
- reject = workflow_update_module.output_contract_value_bearing_run_reject(
- ctx, {"finalize_service": 'return {"output": {"record_id": None, "blocker": None}}'}
- )
-
- assert reject is not None
- assert "Revise the owning code block" in reject["error"]
-
def _gate_blocks(sibling_code: str) -> list[dict[str, object]]:
return [
@@ -18366,223 +10126,12 @@ def _gate_validation(sibling_code: str, diagnostics: SynthesisDiagnostics, synth
selected_code_block=blocks[1],
submitted_selected_code=str(blocks[1]["code"]),
synthesized_code=synthesized_code if synthesized_code is not None else _SPINE_SYNTH_CODE,
- synthesized_diagnostics=diagnostics,
- )
-
-
-class TestBrowserSurfaceRejectionProvenance:
- def test_never_captured_mutation_rejected_with_rescout_move(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- workflow_update_module,
- "synthesize_code_block",
- lambda *a, **k: _fake_spine_synthesized(diagnostics=_spine_emission_diagnostics()),
- )
- ctx = _imposition_split_ctx()
-
- with capture_logs() as logs:
- result = workflow_update_module._maybe_impose_synthesized_code_block(
- _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
- )
-
- assert any("unscouted browser action" in violation for violation in result.violations)
- assert any(
- "never_captured" in violation and "re-scout that step" in violation for violation in result.violations
- )
- events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
- assert len(events) == 1
- assert events[0]["kind"] == "never_captured"
- assert events[0]["site"] == "whole_trajectory"
- assert "#hallucinated" in events[0]["action"]
-
- @pytest.mark.asyncio
- async def test_never_captured_reject_reopens_once_after_matching_canonical_capture(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _imposition_split_ctx()
- submitted_yaml = _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()')
-
- with capture_logs() as rejection_logs:
- result = await _update_workflow(
- {
- "workflow_yaml": submitted_yaml,
- "code_artifact_metadata": ctx.raw_code_artifact_metadata,
- },
- ctx,
- allow_static_output_uncertainty=True,
- )
-
- assert result["ok"] is False
- assert "never_captured" in result["error"]
- obligation = ctx.never_captured_obligation
- assert obligation is not None
- assert obligation.method == "click"
- assert obligation.normalized_receiver == "page.locator('#hallucinated')"
- assert obligation.block_label.endswith("_browser_stage_extra")
- assert obligation.state == "armed"
- assert obligation.replay_payload.params["workflow_yaml"] == submitted_yaml
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
- armed_events = [log for log in rejection_logs if log["event"] == "copilot_never_captured_obligation_armed"]
- assert len(armed_events) == 1
- assert armed_events[0]["turn_id"] == ctx.turn_id
- assert armed_events[0]["draft_fingerprint"] == obligation.draft_fingerprint
- assert armed_events[0]["call_shape_digest"] == obligation.call_shape_digest
- assert armed_events[0]["expected_tool_name"] == obligation.expected_tool_name
-
- scouting_module._record_scouted_interaction(
- ctx, tool_name="click", selector="#unrelated", source_url="https://example.com/form"
- )
-
- assert ctx.never_captured_obligation == obligation
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
-
- replay_call: dict[str, object] = {}
-
- async def _record_replay(
- params: dict[str, object],
- replay_ctx: CopilotContext,
- *,
- allow_missing_credentials: bool | None,
- allow_static_output_uncertainty: bool,
- formation_prepared: bool,
- ) -> dict[str, object]:
- replay_call.update(
- params=params,
- ctx=replay_ctx,
- allow_missing_credentials=allow_missing_credentials,
- allow_static_output_uncertainty=allow_static_output_uncertainty,
- formation_prepared=formation_prepared,
- )
- return {"ok": True, "data": {"message": "Workflow updated successfully."}}
-
- monkeypatch.setattr(workflow_update_module, "_update_workflow", _record_replay)
- ctx.pending_scout_source_url = "https://example.com/form"
- with capture_logs() as replay_logs:
- click_result = await tools_module._click_post_hook(
- {"ok": True, "data": {"selector": "#hallucinated"}},
- {"browser_context": {"url": "https://example.com/form", "title": "Example"}},
- ctx,
- )
-
- assert click_result["data"]["workflow_reconciliation"] == {
- "ok": True,
- "message": "Workflow updated successfully.",
- }
- assert replay_call == {
- "params": {
- "workflow_yaml": submitted_yaml,
- "code_artifact_metadata": obligation.replay_payload.params["code_artifact_metadata"],
- },
- "ctx": ctx,
- "allow_missing_credentials": None,
- "allow_static_output_uncertainty": True,
- "formation_prepared": False,
- }
- consumed = ctx.never_captured_obligation
- assert consumed is not None
- assert consumed.state == "consumed"
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
- completed_events = [log for log in replay_logs if log["event"] == "copilot_never_captured_obligation_completed"]
- assert len(completed_events) == 1
- assert completed_events[0]["identity_digest"] == obligation.identity_digest
- assert completed_events[0]["turn_id"] == ctx.turn_id
- assert completed_events[0]["workflow_permanent_id"] == ctx.workflow_permanent_id
- assert completed_events[0]["draft_fingerprint"] == obligation.draft_fingerprint
- assert completed_events[0]["block_label"] == obligation.block_label
- assert completed_events[0]["site"] == obligation.site
- assert completed_events[0]["trajectory_index"] == consumed.captured_trajectory_index
- replay_events = [log for log in replay_logs if log["event"] == "copilot_never_captured_obligation_replayed"]
- assert len(replay_events) == 1
- assert replay_events[0]["identity_digest"] == obligation.identity_digest
- assert replay_events[0]["turn_id"] == ctx.turn_id
- assert replay_events[0]["draft_fingerprint"] == obligation.draft_fingerprint
- assert replay_events[0]["block_label"] == obligation.block_label
- assert replay_events[0]["site"] == obligation.site
- assert replay_events[0]["captured_trajectory_index"] == consumed.captured_trajectory_index
- assert replay_events[0]["obligation_state"] == "consumed"
- assert replay_events[0]["ok"] is True
-
- scouting_module._record_scouted_interaction(
- ctx, tool_name="click", selector="#hallucinated", source_url="https://example.com/form"
- )
-
- assert ctx.never_captured_obligation == consumed
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
-
- @pytest.mark.asyncio
- async def test_replay_exception_preserves_captured_obligation(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _imposition_split_ctx()
- submitted_yaml = _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()')
- rejected = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)
- assert rejected["ok"] is False
- assert ctx.never_captured_obligation is not None
- scouting_module._record_scouted_interaction(
- ctx, tool_name="click", selector="#hallucinated", source_url="https://example.com/form"
- )
- captured = ctx.never_captured_obligation
- assert captured is not None
- assert captured.state == "captured"
-
- async def _raise_after_consuming(*_args: object, **_kwargs: object) -> dict[str, object]:
- ctx.never_captured_obligation = replace(captured, state="consumed")
- ctx.synthesized_block_reopened_for_capture_obligation = False
- raise RuntimeError("synthetic replay failure")
-
- monkeypatch.setattr(workflow_update_module, "_update_workflow", _raise_after_consuming)
-
- with pytest.raises(RuntimeError, match="synthetic replay failure"):
- await workflow_update_module._replay_captured_never_captured_obligation(ctx)
-
- assert ctx.never_captured_obligation == captured
- assert ctx.synthesized_block_reopened_for_capture_obligation is True
-
- @pytest.mark.asyncio
- async def test_captured_obligation_does_not_reopen_a_different_draft(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- monkeypatch.setattr(
- workflow_update_module,
- "synthesize_code_block",
- lambda *a, **k: _fake_spine_synthesized(diagnostics=_spine_emission_diagnostics()),
- )
- ctx = _imposition_split_ctx()
- rejected_yaml = _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()')
- rejected = await _update_workflow({"workflow_yaml": rejected_yaml}, ctx)
- assert rejected["ok"] is False
- assert ctx.never_captured_obligation is not None
-
- scouting_module._record_scouted_interaction(
- ctx, tool_name="click", selector="#hallucinated", source_url="https://example.com/form"
- )
- assert ctx.synthesized_block_reopened_for_capture_obligation is True
- ctx.update_workflow_called = True
- different_yaml = rejected_yaml.replace("title: Entry lookup", "title: Different draft")
-
- with capture_logs() as logs:
- await _update_workflow({"workflow_yaml": different_yaml}, ctx)
+ synthesized_diagnostics=diagnostics,
+ )
- obligation = ctx.never_captured_obligation
- assert obligation is not None
- assert obligation.state == "consumed"
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
- events = [log for log in logs if log["event"] == "copilot_imposition_skipped_after_update"]
- assert len(events) == 1
- @pytest.mark.asyncio
- @pytest.mark.parametrize(
- ("sibling_code", "provenance_kind"),
- [
- ('await page.locator("#stage-a").click(timeout=5000)', "shape_diverged"),
- ('do_click = page.locator("#x").click\nawait do_click()', "ambiguous"),
- ('await page.locator("#hallucinated").type("value")', "never_captured"),
- ],
- )
- async def test_fail_closed_provenance_does_not_arm_capture_obligation(
- self,
- monkeypatch: pytest.MonkeyPatch,
- sibling_code: str,
- provenance_kind: str,
- ) -> None:
+class TestBrowserSurfaceRejectionProvenance:
+ def test_never_captured_mutation_rejected_with_rescout_move(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
workflow_update_module,
"synthesize_code_block",
@@ -18590,15 +10139,20 @@ class TestBrowserSurfaceRejectionProvenance:
)
ctx = _imposition_split_ctx()
- result = await _update_workflow(
- {"workflow_yaml": _already_split_spine_yaml(extra_sibling_code=sibling_code)},
- ctx,
- )
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(
+ _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
+ )
- assert result["ok"] is False
- assert provenance_kind in result["error"]
- assert ctx.never_captured_obligation is None
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
+ assert any("unscouted browser action" in violation for violation in result.violations)
+ assert any(
+ "never_captured" in violation and "re-scout that step" in violation for violation in result.violations
+ )
+ events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
+ assert len(events) == 1
+ assert events[0]["kind"] == "never_captured"
+ assert events[0]["site"] == "whole_trajectory"
+ assert "#hallucinated" in events[0]["action"]
def test_same_receiver_divergent_call_shape_is_shape_diverged_with_nearest(self) -> None:
validation = _gate_validation(
@@ -18859,82 +10413,6 @@ class TestScoutCaptureParityAccounting:
indexes = [item["trajectory_index"] for item in ctx.scout_trajectory]
assert indexes == list(range(2, scouting_module._MAX_SCOUTED_INTERACTIONS + 2))
- def test_capture_obligation_matches_stable_index_after_trajectory_eviction(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "turn-eviction"
- ctx.scouted_interactions = []
- ctx.scout_trajectory = [
- {"tool_name": "click", "selector": f"#item-{index}", "trajectory_index": index}
- for index in range(scouting_module._MAX_SCOUTED_INTERACTIONS)
- ]
- ctx.never_captured_obligation = NeverCapturedObligation(
- identity_digest="eviction-identity",
- turn_id=ctx.turn_id,
- draft_fingerprint="draft",
- block_label="target",
- site="whole_trajectory",
- method="click",
- normalized_receiver="page.locator('#target')",
- call_shape_digest="shape",
- expected_tool_name="click",
- armed_after_trajectory_index=scouting_module._MAX_SCOUTED_INTERACTIONS - 1,
- )
-
- scouting_module._record_scouted_interaction(
- ctx, tool_name="click", selector="#target", source_url="https://example.com/list"
- )
-
- obligation = ctx.never_captured_obligation
- assert obligation is not None
- assert obligation.state == "captured"
- assert obligation.captured_trajectory_index == scouting_module._MAX_SCOUTED_INTERACTIONS
- assert ctx.synthesized_block_reopened_for_capture_obligation is True
-
- def test_capture_obligation_rejects_same_locator_with_different_press_key(self) -> None:
- ctx = _code_only_ctx()
- ctx.turn_id = "turn-key"
- ctx.scouted_interactions = []
- ctx.scout_trajectory = []
- ctx.never_captured_obligation = NeverCapturedObligation(
- identity_digest="key-identity",
- turn_id=ctx.turn_id,
- draft_fingerprint="draft",
- block_label="submit",
- site="whole_trajectory",
- method="press",
- normalized_receiver="page.locator('#request')",
- call_shape_digest="shape",
- expected_tool_name="press_key",
- expected_argument_literal="Enter",
- armed_after_trajectory_index=-1,
- )
-
- scouting_module._record_scouted_interaction(
- ctx,
- tool_name="press_key",
- selector="#request",
- source_url="https://example.com/form",
- key="Tab",
- )
-
- obligation = ctx.never_captured_obligation
- assert obligation is not None
- assert obligation.state == "armed"
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
-
- scouting_module._record_scouted_interaction(
- ctx,
- tool_name="press_key",
- selector="#request",
- source_url="https://example.com/form",
- key="Enter",
- )
-
- obligation = ctx.never_captured_obligation
- assert obligation is not None
- assert obligation.state == "captured"
- assert ctx.synthesized_block_reopened_for_capture_obligation is True
-
def test_dedup_replacement_is_not_an_eviction(self) -> None:
ctx = _code_only_ctx()
ctx.scouted_interactions = []
@@ -19147,29 +10625,6 @@ class TestScoutedSpineUnderBuild:
assert result.violations == []
assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- @pytest.mark.asyncio
- async def test_under_build_rejects_climb_churn_counter_to_stop(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- workflow_update_module, "synthesize_code_block", lambda *a, **k: _drifted_spine_synthesized()
- )
- ctx = _quote_ctx()
- workflow_yaml = _under_build_draft_yaml()
-
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
- result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
- assert result["ok"] is False
- assert "scouted_spine_under_build" in result["error"]
- assert ctx.code_authoring_guardrail_reject_count == index + 1
- assert ctx.blocker_signal is None
-
- final = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
-
- assert final["ok"] is False
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- held = ctx.blocker_signal
- assert isinstance(held, CopilotToolBlockerSignal)
- assert held.internal_reason_code == "code_authoring_guardrail_churn"
-
def _records_spine_ctx() -> CopilotContext:
ctx = _code_only_ctx()
@@ -19267,99 +10722,6 @@ def _credential_spine_block_yaml(synthesized: SynthesizedCodeBlock) -> str:
class TestScoutedSpinePersistSeamCoverage:
- @pytest.mark.asyncio
- async def test_early_partial_persist_then_stub_resubmission_rejected_and_turn_end_fires(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _records_spine_ctx()
- full_trajectory = list(ctx.scout_trajectory)
- ctx.scout_trajectory = full_trajectory[:2]
- partial_synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert partial_synthesized is not None
-
- persisted = await _update_workflow({"workflow_yaml": _records_block_yaml(partial_synthesized.code)}, ctx)
-
- assert persisted["ok"] is True
- assert ctx.persisted_draft_browser_calls is not None
- assert [pair for pair in ctx.persisted_draft_browser_calls if pair[0] == "click"] == [
- ("click", "page.locator('#stage-a')"),
- ("click", "page.locator('#stage-b')"),
- ]
-
- ctx.scout_trajectory = full_trajectory
-
- with capture_logs() as logs:
- rejected = await _update_workflow({"workflow_yaml": _records_block_yaml(partial_synthesized.code)}, ctx)
-
- assert rejected["ok"] is False
- assert "scouted_spine_under_build" in rejected["error"]
- assert 'await page.locator("#stage-c").click()' in rejected["error"]
- events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(events) == 1
- assert events[0]["site"] == "pre_persist"
-
- with capture_logs() as logs:
- nudge = enforcement_module._scouted_spine_turn_end_nudge(ctx)
-
- assert nudge is not None
- assert "scouted_spine_under_build" in nudge
- assert 'await page.locator("#stage-c").click()' in nudge
- turn_end_events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(turn_end_events) == 1
- assert turn_end_events[0]["site"] == "turn_end"
-
- @pytest.mark.asyncio
- async def test_post_update_skip_path_persist_under_coverage_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _records_spine_ctx()
- ctx.update_workflow_called = True
-
- with capture_logs() as logs:
- result = await _update_workflow(
- {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
- )
-
- assert result["ok"] is False
- assert "scouted_spine_under_build" in result["error"]
- assert 'await page.locator("#stage-b").click()' in result["error"]
- assert 'await page.locator("#stage-c").click()' in result["error"]
- assert ctx.code_authoring_guardrail_reject_count == 1
- assert [log for log in logs if log["event"] == "copilot_imposition_skipped_after_update"]
- events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(events) == 1
- assert events[0]["site"] == "pre_persist"
-
- @pytest.mark.asyncio
- async def test_second_persist_dropping_all_code_blocks_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _checkpoint_eligible_ctx()
- no_code_yaml = _yaml(
- """
- title: Records
- workflow_definition:
- blocks:
- - block_type: send_email
- label: notify
- recipients:
- subject: Records ready
- body: Records run finished
- """
- )
-
- with capture_logs() as logs:
- result = await _update_workflow({"workflow_yaml": no_code_yaml}, ctx)
-
- assert result["ok"] is False
- assert "scouted_spine_under_build" in result["error"]
- assert 'await page.locator("#stage-a").click()' in result["error"]
- assert 'await page.locator("#stage-c").click()' in result["error"]
- assert ctx.code_authoring_guardrail_reject_count == 1
- events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(events) == 1
- assert events[0]["site"] == "pre_persist"
-
def test_separated_split_branch_under_coverage_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
diagnostics = _spine_emission_diagnostics()
diagnostics.emitted_interactions.append(
@@ -19416,61 +10778,6 @@ class TestScoutedSpinePersistSeamCoverage:
]
assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None
- @pytest.mark.asyncio
- async def test_composite_route_resubmission_passes_credential_gate_and_persists(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- monkeypatch.setattr(
- workflow_update_module, "_credential_reference_validation_error", AsyncMock(return_value=None)
- )
- ctx = _credential_spine_ctx()
-
- with capture_logs() as logs:
- rejected = await _update_workflow(
- {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
- )
-
- assert rejected["ok"] is False
- assert "scouted_spine_under_build" in rejected["error"]
- assert "fill_credential_field" in rejected["error"]
- assert "click the submit control or press Enter" in rejected["error"]
- assert "Missing rung source to reuse verbatim" in rejected["error"]
- events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(events) == 1
- assert events[0]["site"] == "pre_persist"
- assert events[0]["credential_scout_precondition_pending"] is True
-
- ctx.scout_trajectory.append(_submit_interaction(source_url="https://example.com/records"))
- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
- assert synthesized is not None
-
- with capture_logs() as logs:
- accepted = await _update_workflow({"workflow_yaml": _credential_spine_block_yaml(synthesized)}, ctx)
-
- assert accepted["ok"] is True
- assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None
-
- @pytest.mark.asyncio
- async def test_satisfied_credential_precondition_keeps_single_step_route(self) -> None:
- ctx = _credential_spine_ctx()
- ctx.scout_trajectory.append(_submit_interaction(source_url="https://example.com/records"))
- ctx.synthesized_goal_complete_landed = True
-
- with capture_logs() as logs:
- rejected = await _update_workflow(
- {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
- )
-
- assert rejected["ok"] is False
- assert "scouted_spine_under_build" in rejected["error"]
- assert "fill_credential_field" not in rejected["error"]
- assert "Missing rung source to reuse verbatim" in rejected["error"]
- events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
- assert len(events) == 1
- assert events[0]["credential_scout_precondition_pending"] is False
-
@pytest.mark.asyncio
async def test_credential_scout_reject_carries_open_obligation_artifact(self) -> None:
ctx = _credential_spine_ctx()
@@ -19486,19 +10793,6 @@ class TestScoutedSpinePersistSeamCoverage:
assert "Missing rung source to reuse verbatim" in result["error"]
assert ".password" in result["error"]
- @pytest.mark.asyncio
- async def test_render_gate_preempts_pre_persist_spine_gate(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _checkpoint_eligible_ctx()
- draft = _records_block_yaml('await page.locator("#stage-a").click()\nnote = "{{ frobnicator }}"')
-
- with capture_logs() as logs:
- result = await _update_workflow({"workflow_yaml": draft}, ctx)
-
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "code_block_unrenderable"
- assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
-
class TestScoutedSpineTurnEndCheckpoint:
def test_checkpoint_is_single_shot_and_emits_unresolved_when_spent(self) -> None:
@@ -19673,35 +10967,6 @@ class TestFinalYamlStructuralGate:
assert violation is not None
assert "missing" in violation
- def test_final_yaml_structural_gate_is_scoped_to_code_authoring(self) -> None:
- ctx = _records_spine_ctx()
- ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
- dangling_yaml = _yaml(
- """
- title: Records
- workflow_definition:
- blocks:
- - block_type: code
- label: a
- next_block_label: missing
- - block_type: code
- label: b
- """
- )
- assert (
- workflow_update_module._final_yaml_structural_violations(
- ctx,
- dangling_yaml,
- minted_parameter_keys=[],
- promoted_parameter_keys=[],
- carried_by_imposition=False,
- )
- == []
- )
- assert (
- workflow_update_module._dangling_next_block_label_violation(parse_workflow_yaml(dangling_yaml)) is not None
- )
-
def test_resolved_next_block_label_passes(self) -> None:
parsed = {
"workflow_definition": {
@@ -19713,90 +10978,6 @@ class TestFinalYamlStructuralGate:
}
assert workflow_update_module._dangling_next_block_label_violation(parsed) is None
- def test_orphan_minted_parameter_is_flagged_and_referenced_is_not(self) -> None:
- parsed = {
- "workflow_definition": {
- "blocks": [
- {"block_type": "code", "label": "a", "code": "await page.locator('#x').fill(str(used_key))"},
- ]
- }
- }
- violation = workflow_update_module._orphan_minted_parameter_violation(parsed, ["used_key", "orphan_key"])
- assert violation is not None
- assert "orphan_key" in violation
- assert "used_key" not in violation
-
- def test_preexisting_authored_param_outside_manifest_is_untouched(self) -> None:
- parsed = {
- "workflow_definition": {
- "blocks": [
- {"block_type": "code", "label": "a", "code": "await page.locator('#x').click()"},
- ],
- "parameters": [{"key": "unused_authored_input", "parameter_type": "workflow"}],
- }
- }
- assert workflow_update_module._orphan_minted_parameter_violation(parsed, []) is None
-
- def test_scaffolding_only_body_flagged_only_when_carried_by_imposition(self) -> None:
- ctx = _records_spine_ctx()
- scaffolding_yaml = _records_block_yaml("await page.goto('https://example.com/records')")
- carried = workflow_update_module._final_yaml_structural_violations(
- ctx,
- scaffolding_yaml,
- minted_parameter_keys=[],
- promoted_parameter_keys=[],
- carried_by_imposition=True,
- )
- assert any("no load-bearing" in violation for violation in carried)
- not_carried = workflow_update_module._final_yaml_structural_violations(
- ctx,
- scaffolding_yaml,
- minted_parameter_keys=[],
- promoted_parameter_keys=[],
- carried_by_imposition=False,
- )
- assert not any("no load-bearing" in violation for violation in not_carried)
-
- def test_load_bearing_body_passes_the_gate(self) -> None:
- ctx = _records_spine_ctx()
- real_yaml = _records_block_yaml("await page.locator('#stage-a').click()")
- assert (
- workflow_update_module._final_yaml_structural_violations(
- ctx,
- real_yaml,
- minted_parameter_keys=[],
- promoted_parameter_keys=[],
- carried_by_imposition=True,
- )
- == []
- )
-
- def test_stub_reproducing_raw_prompt_is_flagged(self) -> None:
- ctx = _records_spine_ctx()
- ctx.turn_context_packet = SimpleNamespace(
- transcript_context=SimpleNamespace(earliest_user_turn="Log me in and stop")
- )
- stub_yaml = _yaml(
- """
- title: Records
- workflow_definition:
- blocks:
- - block_type: code
- label: search_registry
- description: Log me in and stop
- code: |
- await page.goto("https://example.com/records")
- """
- )
- violations = workflow_update_module._final_yaml_structural_violations(
- ctx,
- stub_yaml,
- minted_parameter_keys=[],
- promoted_parameter_keys=[],
- carried_by_imposition=True,
- )
- assert any("raw user request" in violation for violation in violations)
-
def _full_coverage_calls() -> list[tuple[str, str]]:
return [
@@ -21200,101 +12381,25 @@ class TestCommitOnlyReach:
ctx = _commit_only_ctx()
ctx.scout_trajectory = trajectory
- assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is False
-
- def test_commit_only_imposition_uses_goal_reaching_admission(self) -> None:
- ctx = _commit_only_ctx()
- ctx.update_workflow_called = True
-
- with capture_logs() as logs:
- result = workflow_update_module._maybe_impose_synthesized_code_block(_commit_only_submitted_yaml(), ctx)
-
- assert result.violations == []
- assert result.substitutions
- assert ctx.spine_imposition_owned_attempt is True
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert "openForm" in code
- assert "commitForm" in code
- admitted = [entry for entry in logs if entry["event"] == "copilot_imposition_admitted_after_update"]
- assert len(admitted) == 1
- assert admitted[0]["admission_key"] == "goal_reaching_spine_unlanded"
- assert admitted[0]["admission_key"] != "reopen_author_time_reject"
-
- @pytest.mark.asyncio
- async def test_first_commit_only_attempt_persists_imposition_and_dispatches_run(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _commit_only_ctx()
- ctx.request_policy = RequestPolicy()
- ctx.completion_criteria_turn_state = None
- update_result: dict[str, object] = {}
- run_calls: list[dict[str, object]] = []
-
- async def capture_update(
- payload: dict[str, object], update_ctx: AgentContext, **kwargs: object
- ) -> dict[str, object]:
- result = await workflow_update_module._update_workflow(payload, update_ctx, **kwargs)
- update_result.update(result)
- return result
-
- async def fake_run_blocks(
- params: dict[str, object], _ctx: AgentContext, **_kwargs: object
- ) -> dict[str, object]:
- run_calls.append(params)
- return {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_commit_only",
- "overall_status": "completed",
- "blocks": [{"label": "start_service", "status": "completed"}],
- },
- }
-
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *_args: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_update_workflow", capture_update)
- monkeypatch.setattr(tools_module, "_plan_frontier", lambda *_args: (["start_service"], {}, "start_service"))
- monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *_args: None)
- monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
- monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))
- monkeypatch.setattr(
- tools_module,
- "_tool_visible_result_after_completion_verification",
- lambda _ctx, result, _verification: result,
- )
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *_args: None)
+ assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is False
+
+ def test_commit_only_imposition_uses_goal_reaching_admission(self) -> None:
+ ctx = _commit_only_ctx()
+ ctx.update_workflow_called = True
with capture_logs() as logs:
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": _commit_only_submitted_yaml(),
- "block_labels": ["start_service"],
- }
- ),
- )
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_commit_only_submitted_yaml(), ctx)
- assert json.loads(result)["ok"] is True
- assert update_result["ok"] is True
- data = update_result["data"]
- assert isinstance(data, dict)
- substitutions = data.get("imposed_substitutions")
- assert isinstance(substitutions, dict)
- assert substitutions["block_label"] == "start_service"
- assert ctx.update_workflow_called is True
- persisted_code = str(_single_code_block(parse_workflow_yaml(ctx.workflow_yaml))["code"])
- assert "openForm" in persisted_code
- assert "commitForm" in persisted_code
- assert run_calls == [{"block_labels": ["start_service"], "parameters": {}}]
- owned = [entry for entry in logs if entry["event"] == "copilot_spine_imposition_owned_attempt"]
- assert len(owned) == 1
- assert not [entry for entry in logs if entry["event"] == "copilot_imposition_admitted_after_update"]
+ assert result.violations == []
+ assert result.substitutions
+ assert ctx.spine_imposition_owned_attempt is True
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert "openForm" in code
+ assert "commitForm" in code
+ admitted = [entry for entry in logs if entry["event"] == "copilot_imposition_admitted_after_update"]
+ assert len(admitted) == 1
+ assert admitted[0]["admission_key"] == "goal_reaching_spine_unlanded"
+ assert admitted[0]["admission_key"] != "reopen_author_time_reject"
class TestReachingTrajectoryOwnershipDeterminism:
@@ -21525,8 +12630,6 @@ def test_producer_mixed_merged_single_return_passes_download_gate() -> None:
assert envelope.code.count("return {") == 1
assert '"downloaded_file_name": downloaded_file_name' in envelope.code
assert workflow_update_module._code_uses_expect_download(envelope.code)
- artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}
- assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None
_, violations = workflow_update_module._extraction_code_with_required_static_return(
envelope.code, required_paths={"output.statement_amount", "output.statement_date"}
)
@@ -22014,8 +13117,6 @@ def test_producer_preserves_the_models_download_descriptor_key() -> None:
assert envelope.code.count("return {") == 1
assert '"saved_as": dl_info.value.suggested_filename' in envelope.code
assert "downloaded_file_name" not in envelope.code
- artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}
- assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None
assert {"output.statement_amount"} <= workflow_update_module._code_block_produced_output_paths(envelope.code)
@@ -22319,67 +13420,6 @@ class TestSynthesizedParameterMultiFillReconciliation:
assert reconciliation.repair_context is not None
assert reconciliation.repair_context.reason_code == "synthesized_parameter_binding_ambiguous"
- @pytest.mark.asyncio
- async def test_relaxed_multi_fill_binding_still_fails_closed_on_empty_envelope(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#location",
- "source_url": "https://example.com/directory",
- "typed_length": 7,
- "role": "textbox",
- "accessible_name": "Address City County or Zip Code",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#specialty",
- "source_url": "https://example.com/directory",
- "typed_length": 10,
- "role": "textbox",
- "accessible_name": "Provider Specialty",
- "trajectory_index": 1,
- },
- {
- "tool_name": "click",
- "selector": "#search",
- "source_url": "https://example.com/directory",
- "role": "button",
- "accessible_name": "Search",
- "trajectory_index": 2,
- },
- ]
- submitted = _yaml(
- """
- title: Directory lookup
- workflow_definition:
- parameters:
- - {parameter_type: output, key: directory_result}
- blocks:
- - block_type: code
- label: search_directory
- prompt: Search the directory and return structured provider result data.
- code: |
- await page.locator("#location").fill("Raleigh")
- await page.locator("#specialty").fill("Cardiology")
- await page.locator("#search").click()
- """
- )
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "synthesized_parameter_binding_ambiguous" not in json.dumps(result)
- assert "must pass `code_artifact_metadata`" in result["error"]
- assert ctx.workflow_yaml == ""
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
-
def _directory_output_intent_yaml(self) -> str:
return _yaml(
"""
@@ -22491,48 +13531,6 @@ class TestSynthesizedParameterMultiFillReconciliation:
"""
)
- def test_owned_carrier_keyed_output_forms_goal_value_paths_when_union_empty(self) -> None:
- ctx = self._owned_carrier_ctx()
- workflow_yaml = self._carrier_yaml(
- 'npi = (await page.locator("#npi").inner_text()).strip()\n'
- ' return {"provider": {"npi": npi}}'
- )
-
- scaffolded, scaffold_applied = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, workflow_yaml, None
- )
-
- assert scaffold_applied is True
- assert json.dumps(scaffolded).count("provider.npi") > 0
- assert workflow_update_module._missing_code_artifact_metadata_error(workflow_yaml, ctx, scaffolded) is None
-
- def test_owned_carrier_without_keyed_output_stays_fail_closed(self) -> None:
- ctx = self._owned_carrier_ctx()
- workflow_yaml = self._carrier_yaml('await page.locator("#search").click()')
-
- scaffolded, scaffold_applied = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, workflow_yaml, None
- )
-
- assert scaffold_applied is False
- assert scaffolded is None
- assert workflow_update_module._missing_code_artifact_metadata_error(workflow_yaml, ctx, scaffolded) is not None
-
- def test_non_owned_carrier_does_not_form_metadata(self) -> None:
- ctx = self._owned_carrier_ctx()
- ctx.spine_imposition_owned_attempt = False
- workflow_yaml = self._carrier_yaml(
- 'npi = (await page.locator("#npi").inner_text()).strip()\n'
- ' return {"provider": {"npi": npi}}'
- )
-
- scaffolded, scaffold_applied = workflow_update_module._scaffold_metadata_contract_for_update(
- ctx, workflow_yaml, None
- )
-
- assert scaffold_applied is False
- assert workflow_update_module._missing_code_artifact_metadata_error(workflow_yaml, ctx, scaffolded) is not None
-
def _divergent_multi_fill_scout(self) -> list[dict[str, object]]:
return [
{
@@ -22587,63 +13585,6 @@ class TestSynthesizedParameterMultiFillReconciliation:
"""
)
- @pytest.mark.asyncio
- async def test_two_repair_rounds_never_realternate_to_binding_reject(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = self._divergent_multi_fill_scout()
-
- first = await _update_workflow({"workflow_yaml": self._divergent_multi_fill_yaml(keyed_return=False)}, ctx)
-
- assert first["ok"] is False
- assert "synthesized_parameter_binding_ambiguous" not in json.dumps(first)
- assert "must pass `code_artifact_metadata`" in first["error"]
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
-
- schema = workflow_update_module._schema_template_text_for_required_paths({"provider.npi"})
- repaired_metadata = [
- {
- "block_label": "search_directory",
- "declared_goal": "return the provider npi",
- "claimed_outcomes": [
- {
- "id": "claim:search_directory",
- "scope": "outcome",
- "text": "return the provider npi",
- "status": "observed_not_verified",
- "goal_value_paths": ["provider.npi"],
- "extraction_schema": schema,
- }
- ],
- "terminal_verifier_expectations": [
- {
- "id": "expectation:search_directory",
- "text": "return the provider npi",
- "goal_value_paths": ["provider.npi"],
- "extraction_schema": schema,
- }
- ],
- }
- ]
-
- second = await _update_workflow(
- {
- "workflow_yaml": self._divergent_multi_fill_yaml(keyed_return=True),
- "code_artifact_metadata": repaired_metadata,
- },
- ctx,
- )
-
- assert "synthesized_parameter_binding_ambiguous" not in json.dumps(second)
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code != "synthesized_parameter_binding_ambiguous"
- # The two author-time rejects no longer alternate. This block's code fills hardcoded literals
- # rather than the synthesized parameters, so it terminates with a single honest fail-closed
- # reject (literal grounding is out of this ticket's plane); it does not oscillate to the ceiling.
- assert second["ok"] is False
-
def _goal_reaching_freehand_ctx(*, credential: bool = False) -> CopilotContext:
ctx = _code_only_ctx()
@@ -22668,317 +13609,41 @@ def _goal_reaching_freehand_ctx(*, credential: bool = False) -> CopilotContext:
"trajectory_index": 1,
},
]
- else:
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#search",
- "source_url": "https://example.com/find",
- "typed_length": 5,
- "role": "textbox",
- "accessible_name": "Search",
- "trajectory_index": 0,
- },
- {
- "tool_name": "click",
- "selector": "button",
- "source_url": "https://example.com/find",
- "role": "button",
- "accessible_name": "Sign In",
- "trajectory_index": 1,
- },
- ]
- return ctx
-
-
-def _freehand_block_yaml(code: str, *, label: str = "find_address", parameters: str = "") -> str:
- indented = "\n".join(f" {line}" for line in code.splitlines())
- return (
- "title: t\n"
- "workflow_definition:\n"
- f"{parameters}"
- " blocks:\n"
- " - block_type: code\n"
- f" label: {label}\n"
- " code: |\n"
- f"{indented}\n"
- )
-
-
-class TestFreehandSurfacePersistSeam:
- def test_get_by_label_unassociated_shape_rejected_with_scout_route(self) -> None:
- code = (
- 'await page.locator("#search").fill("x")\n'
- 'await page.get_by_role("button", name="Sign In", exact=True).click()\n'
- 'await page.get_by_label("State").select_option("CA")'
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert "get_by_label" in result.violations[0]
- assert "never_captured" in result.violations[0]
-
- def test_attribute_selector_button_shape_rejected(self) -> None:
- code = (
- 'await page.locator("#search").fill("x")\n'
- "await page.locator(\"button[data-action='orderDocuments']\").click()"
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is not None
- assert "orderDocuments" in result.violations[0]
-
- def test_strict_synthesis_parity_rejects_bare_locator_when_emitted_literal_is_role_form(self) -> None:
- code = 'await page.locator("#search").fill("x")\nawait page.locator("button").click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is not None
- assert "page.locator('button')" in result.violations[0]
-
- def test_receiver_matching_literal_fill_is_admitted(self) -> None:
- code = (
- 'await page.locator("#search").fill("hardcoded")\n'
- 'await page.get_by_role("button", name="Sign In", exact=True).click()'
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is None
-
- def test_selectorless_navigation_block_is_not_gated(self) -> None:
- code = 'await page.goto("https://example.com/find")\nawait page.reload()'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label="nav"), _goal_reaching_freehand_ctx()
- )
- assert result is None
-
- def _fragment_scout_ctx(self) -> CopilotContext:
- ctx = _goal_reaching_freehand_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#search",
- "source_url": "https://example.com/find",
- "role": "textbox",
- "accessible_name": "Search",
- "trajectory_index": 0,
- }
- ]
- assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is False
- return ctx
-
- def test_fragment_scout_rejects_ungrounded_get_by_label(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_label("State").select_option("CA")'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert "get_by_label" in result.violations[0]
-
- def test_fragment_scout_admits_reused_scouted_receiver(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("#search").fill("x")'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is None
-
- def test_fragment_scout_admits_not_yet_scouted_get_by_role(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_role("button", name="Continue", exact=True).click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is None
-
- def test_fragment_scout_rejects_nameless_get_by_role(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_role("button").click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
-
- def test_fragment_scout_rejects_dynamic_role_get_by_role(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = "await page.get_by_role(role_var).click()"
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
-
- def test_fragment_scout_rejects_dynamic_name_get_by_role(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_role("button", name=btn_name).click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
-
- def test_fragment_scout_rejects_non_literal_kwarg_get_by_role(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_role("button", name="Continue", exact=flag).click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
-
- def test_fragment_scout_reject_names_get_by_role_site(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.get_by_role("button").click()'
- with capture_logs() as logs:
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
- assert events and events[0]["site"] == "fragment_scout"
-
- def test_goal_reaching_rejects_unscouted_get_by_role(self) -> None:
- code = 'await page.get_by_role("button", name="Unscouted", exact=True).click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is not None
- assert "unscouted browser action(s)" in result.violations[0]
-
- def test_fragment_scout_rejects_freehand_bare_locator_guess(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("#search").fill("x")\nawait page.locator("tr").click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert result.repair_context.block_label == "find_address"
- assert "page.locator('tr')" in result.violations[0]
- assert "never_captured" in result.violations[0]
-
- def test_fragment_scout_rejects_freehand_attribute_selector_guess(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = "await page.locator('button[data-action=\"orderDocuments\"]').click()"
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert "orderDocuments" in result.violations[0]
-
- def test_fragment_scout_rejects_aliased_locator_mutation(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'target = page.locator("#guessed")\nawait target.click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert "ambiguous browser action(s)" in result.violations[0]
- assert "target.click()" in result.violations[0]
-
- def test_fragment_scout_rejects_nth_wrapped_locator_guess(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("tr").nth(2).click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert "ambiguous browser action(s)" in result.violations[0]
-
- def test_fragment_scout_rejects_has_text_refined_locator_guess(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("tr", has_text="Order").click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert "unscouted browser action(s)" in result.violations[0]
- assert "never_captured" in result.violations[0]
-
- def test_fragment_scout_rejects_first_wrapped_bare_locator_guess(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("tr").first.click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- assert "page.locator('tr')" in result.violations[0]
-
- def test_fragment_scout_admits_first_wrapped_scouted_selector(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("#search").first.fill("x")'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is None
-
- def test_fragment_scout_reject_carries_fragment_scout_site(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("tr").click()'
- with capture_logs() as logs:
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is not None
- events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
- assert events and events[0]["site"] == "fragment_scout"
-
- def test_fragment_scout_admits_input_prefixed_capture_replay(self) -> None:
- ctx = self._fragment_scout_ctx()
- ctx.scout_trajectory[0]["selector"] = "input#search"
- code = 'await page.locator("#search").fill("x")'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is None
-
- def test_fragment_scout_skips_unchanged_block(self) -> None:
- ctx = self._fragment_scout_ctx()
- code = 'await page.locator("tr").click()'
- submitted = _freehand_block_yaml(code)
- ctx.workflow_yaml = submitted
- result = workflow_update_module._persist_seam_freehand_surface_result(submitted, ctx)
- assert result is None
-
- def test_goal_reaching_freehand_bare_locator_reject_unchanged(self) -> None:
- code = 'await page.locator("#search").fill("x")\nawait page.locator("tr").click()'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code), _goal_reaching_freehand_ctx()
- )
- assert result is not None
- assert "unscouted browser action(s)" in result.violations[0]
-
- def test_unguarded_credential_fill_routes_to_guarded_entry_rung(self) -> None:
- parameters = " parameters:\n - {parameter_type: credential, key: portal}\n"
- code = 'await page.locator("#password").fill(portal.password)'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, parameters=parameters), _goal_reaching_freehand_ctx(credential=True)
- )
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unguarded_credential_fill"
-
- def test_fragment_scout_rejects_unguarded_credential_fill(self) -> None:
- ctx = _goal_reaching_freehand_ctx(credential=True)
- ctx.scout_trajectory = ctx.scout_trajectory[:1]
- assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is False
- parameters = " parameters:\n - {parameter_type: credential, key: portal}\n"
- code = 'await page.locator("#password").fill(portal.password)'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, parameters=parameters), ctx
- )
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unguarded_credential_fill"
-
- def test_presence_guarded_credential_fill_is_admitted(self) -> None:
- parameters = " parameters:\n - {parameter_type: credential, key: portal}\n"
- code = (
- '_scout_entry_target = page.locator("#username")\n'
- "if await _scout_entry_target.count() == 1:\n"
- ' await page.locator("#username").fill(portal.username)'
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, parameters=parameters), _goal_reaching_freehand_ctx(credential=True)
- )
- assert result is None
+ else:
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#search",
+ "source_url": "https://example.com/find",
+ "typed_length": 5,
+ "role": "textbox",
+ "accessible_name": "Search",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "button",
+ "source_url": "https://example.com/find",
+ "role": "button",
+ "accessible_name": "Sign In",
+ "trajectory_index": 1,
+ },
+ ]
+ return ctx
- def test_non_credential_attribute_fill_is_not_flagged_as_credential(self) -> None:
- parameters = " parameters:\n - {parameter_type: credential, key: portal}\n"
- code = 'await page.locator("#search").fill(profile.email)'
- assert workflow_update_module._block_has_unguarded_credential_fill(code, {"portal"}) is False
- _ = parameters
- def test_reaches_goal_gate_off_returns_none_without_imposition(self) -> None:
- ctx = _goal_reaching_freehand_ctx()
- ctx.impose_synthesized_code_block = False
- code = 'await page.get_by_label("State").select_option("CA")'
- result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
- assert result is None
+def _freehand_block_yaml(code: str, *, label: str = "find_address", parameters: str = "") -> str:
+ indented = "\n".join(f" {line}" for line in code.splitlines())
+ return (
+ "title: t\n"
+ "workflow_definition:\n"
+ f"{parameters}"
+ " blocks:\n"
+ " - block_type: code\n"
+ f" label: {label}\n"
+ " code: |\n"
+ f"{indented}\n"
+ )
_RENDER_PREFLIGHT_FIXTURE_PATH = (
@@ -23049,39 +13714,6 @@ _RENDERABLE_JINJA_CODE_YAML = _yaml(
class TestCodeBlockRenderPreflightSeam:
- @pytest.mark.asyncio
- async def test_code_block_preflight_rejects_parameters_namespace_at_persist_seam(self) -> None:
- ctx = _code_only_ctx()
- with capture_logs() as logs:
- result = await _update_workflow({"workflow_yaml": _PARAMETERS_NAMESPACE_CODE_YAML}, ctx)
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "code_block_unrenderable"
- assert result["data"]["block_label"] == "submit_request"
- assert result["data"]["failing_expression"] == "{{ parameters.business_name }}"
- assert "{{ business_name }}" in result["error"]
- outcome = ctx.latest_recorded_build_test_outcome
- assert isinstance(outcome, RecordedBuildTestOutcome)
- assert outcome.reason_code == "code_block_unrenderable"
- assert any(
- log.get("reason_code") == "code_block_unrenderable"
- and log.get("failing_expression") == "{{ parameters.business_name }}"
- for log in logs
- )
-
- @pytest.mark.asyncio
- async def test_code_block_preflight_rejects_undeclared_root_at_persist_seam(self) -> None:
- result = await _update_workflow({"workflow_yaml": _UNDECLARED_ROOT_CODE_YAML}, _code_only_ctx())
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "code_block_unrenderable"
- assert result["data"]["failing_expression"] == "{{ frobnicator }}"
-
- @pytest.mark.asyncio
- async def test_code_block_preflight_rejects_template_syntax_error_at_persist_seam(self) -> None:
- result = await _update_workflow({"workflow_yaml": _TEMPLATE_SYNTAX_ERROR_CODE_YAML}, _code_only_ctx())
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "code_block_unrenderable"
- assert "syntax error" in result["error"]
-
@pytest.mark.asyncio
async def test_code_block_preflight_renderable_draft_persists(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -23140,23 +13772,6 @@ class TestCodeBlockRenderPreflightSeam:
assert (result.get("data") or {}).get("reason_code") != "code_block_unrenderable"
assert result["ok"] is True
- @pytest.mark.asyncio
- async def test_code_block_preflight_rejects_recorded_unrenderable_draft_replay(self) -> None:
- artifact = json.loads(_RENDER_PREFLIGHT_FIXTURE_PATH.read_text())
- ctx = _code_only_ctx()
- with capture_logs() as logs:
- result = await _update_workflow({"workflow_yaml": artifact["_copilot_yaml"]}, ctx)
- assert result["ok"] is False
- assert result["data"]["reason_code"] == "code_block_unrenderable"
- assert result["data"]["block_label"] == "submit_commercial_water_request"
- assert "parameters.business_name" in result["data"]["failing_expression"]
- assert "{{ business_name }}" in result["error"]
- assert any(
- log.get("reason_code") == "code_block_unrenderable"
- and "parameters.business_name" in str(log.get("failing_expression"))
- for log in logs
- )
-
def _under_scouted_order_status_ctx() -> CopilotContext:
ctx = _code_only_ctx()
@@ -23176,40 +13791,6 @@ def _under_scouted_order_status_ctx() -> CopilotContext:
return ctx
-class TestUnderScoutedOrderStatusReplay:
- _LOOKUP_BLOCK_CODE = (
- 'await page.goto("https://portal.example.com/order-status")\n'
- 'await page.locator("#confirmation").fill("ABC123")\n'
- "await page.locator('button[data-action=\"orderLookup\"]').click()\n"
- "await page.locator('button[data-action=\"orderDocuments\"]').click()\n"
- 'rows = page.locator("tr")\n'
- "row_count = await rows.count()\n"
- )
-
- def test_freehand_button_guesses_rejected_instead_of_admitted(self) -> None:
- ctx = _under_scouted_order_status_ctx()
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(self._LOOKUP_BLOCK_CODE, label="lookup_order"), ctx
- )
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert result.repair_context.block_label == "lookup_order"
- assert "orderLookup" in result.violations[0]
- assert "orderDocuments" in result.violations[0]
-
- def test_scouted_confirmation_replay_admitted(self) -> None:
- ctx = _under_scouted_order_status_ctx()
- code = (
- 'await page.goto("https://portal.example.com/order-status")\n'
- 'await page.locator("#confirmation").fill("ABC123")\n'
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label="lookup_order"), ctx
- )
- assert result is None
-
-
def _pre_goal_wizard_ctx() -> CopilotContext:
ctx = _code_only_ctx()
_enable_imposition(ctx)
@@ -23440,35 +14021,6 @@ class TestPrePersistScoutedSpineGate:
assert result is None
assert ctx.code_authoring_guardrail_reject_count == 0
- @pytest.mark.asyncio
- async def test_pre_persist_reject_leaves_canonical_definition_unwritten(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- monkeypatch.setattr(workflow_update_module, "_workflow_requires_canonical_persist", lambda *_a, **_k: True)
- canonical_writes: list[dict[str, object]] = []
-
- async def _spy_update_workflow_definition(**kwargs: object) -> None:
- canonical_writes.append(kwargs)
-
- monkeypatch.setattr(
- workflow_update_module.app,
- "WORKFLOW_SERVICE",
- SimpleNamespace(update_workflow_definition=_spy_update_workflow_definition),
- )
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- submitted = _freehand_block_yaml('print(await page.locator("body").inner_text())', label="report_only")
- ctx.workflow_yaml = submitted
-
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "under-builds the scouted spine" in result["error"]
- assert canonical_writes == []
- assert ctx.has_staged_proposal is False
- assert ctx.persisted_draft_browser_calls is None
-
@pytest.mark.asyncio
async def test_covering_draft_persists_through_canonical_path(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -23564,71 +14116,6 @@ _P6_RECORDED_FREEHAND_BLOCKS: dict[str, tuple[str, str, tuple[str, ...]]] = {
class TestRecordedGauntletPacketReplay:
- @pytest.mark.parametrize("run_id", _P6_UNDER_SCOUT_RECORDED_RUN_IDS)
- def test_p6_recorded_freehand_block_rejected(self, run_id: str) -> None:
- label, code, expected_tokens = _P6_RECORDED_FREEHAND_BLOCKS[run_id]
- ctx = _under_scouted_order_status_ctx()
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label=label), ctx
- )
- assert result is not None
- assert result.repair_context is not None
- assert result.repair_context.reason_code == "freehand_unresolvable_selector"
- assert result.repair_context.block_label == label
- for token in expected_tokens:
- assert token in result.violations[0]
-
- def test_p6_recorded_freehand_reject_carries_fragment_scout_site(self) -> None:
- label, code, _ = _P6_RECORDED_FREEHAND_BLOCKS[_P6_UNDER_SCOUT_RECORDED_RUN_IDS[0]]
- ctx = _under_scouted_order_status_ctx()
- with capture_logs() as logs:
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label=label), ctx
- )
- assert result is not None
- events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
- assert events and events[0]["site"] == "fragment_scout"
-
- def test_p6_recorded_scouted_confirmation_replay_admitted(self) -> None:
- ctx = _under_scouted_order_status_ctx()
- code = (
- 'await page.goto("https://example.com/order-status")\nawait page.locator("#confirmation").fill("ABC123")\n'
- )
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label="lookup_order"), ctx
- )
- assert result is None
-
- def test_p6_cap_evicted_scouted_selector_still_admitted(self) -> None:
- ctx = _under_scouted_order_status_ctx()
- ctx.scout_trajectory = [
- {
- "tool_name": "click",
- "selector": "#search-submit",
- "source_url": "https://example.com/order-status",
- "role": "button",
- "accessible_name": "Search",
- "trajectory_index": 0,
- }
- ]
- ctx.scouted_interactions = [
- {
- "tool_name": "type_text",
- "selector": "#confirmation",
- "source_url": "https://example.com/order-status",
- "role": "textbox",
- "accessible_name": "Confirmation number",
- "typed_length": 8,
- "trajectory_index": 0,
- }
- ]
- assert enforcement_module.synthesized_trajectory_reaches_goal(ctx) is False
- code = 'await page.locator("#confirmation").fill("ABC123")'
- result = workflow_update_module._persist_seam_freehand_surface_result(
- _freehand_block_yaml(code, label="lookup_order"), ctx
- )
- assert result is None
-
@pytest.mark.parametrize("run_id", [_P10_SPINE_COLLAPSE_RECORDED_RUN_ID])
def test_p10_recorded_sign_in_only_collapse_rejected_naming_dropped_rungs(self, run_id: str) -> None:
ctx = _pre_goal_wizard_ctx()
@@ -23774,7 +14261,7 @@ class TestDefinitionContractRecordedPacketReplay:
assert stored["reason_code"] == "definition_contract_unsatisfied"
assert stored["verdict"] == "authoring_rejected"
- def test_run1_recorded_reject_error_is_byte_identical_to_recorded_halt_keys(self) -> None:
+ def test_run1_recorded_reject_error_names_exactly_the_recorded_halt_keys(self) -> None:
packet = _P3_PACKETS_BY_RUN["run1"]
recorded_keys = packet["turn_halts"][0]["unresolved_parameter_keys"]
assert sorted(_P3_REPLAY_BINDINGS["run1"]) == recorded_keys
@@ -23784,212 +14271,10 @@ class TestDefinitionContractRecordedPacketReplay:
rejection = workflow_update_module._definition_plane_preflight_reject(ctx, draft)
assert rejection is not None
- assert workflow_update_module._definition_plane_reject_error(rejection) == (
- "The submitted workflow declares reusable parameters that no block references: "
- "`account_number`, `billing_end_date`, `billing_start_date`."
- )
-
- @pytest.mark.asyncio
- @pytest.mark.parametrize("run_name", ["run1", "run2", "run3"])
- async def test_recorded_packets_converge_next_attempt_into_bound_dataflow(
- self, monkeypatch: pytest.MonkeyPatch, run_name: str
- ) -> None:
- _stub_successful_update(monkeypatch)
- packet = _P3_PACKETS_BY_RUN[run_name]
- bindings = _P3_REPLAY_BINDINGS[run_name]
- values = {key: value for key, (_selector, value) in bindings.items()}
- ctx = _p3_definition_ctx()
- _enable_imposition(ctx)
- draft = _p3_draft_yaml(packet, bindings)
- for _ in range(2):
- rejected = await _update_workflow({"workflow_yaml": draft}, ctx)
- assert rejected["ok"] is False
- assert rejected["data"]["reason_code"] == "definition_contract_unsatisfied"
- for summary in packet["reject_summaries"]:
- assert ("Failed: " + rejected["error"]).startswith(
- _recorded_summary_prefix(summary["recorded_summary"])
- )
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- ctx.scout_trajectory = _p3_scout_trajectory(packet)
- ctx.composition_page_evidence = _p3_composition_evidence(packet, bindings)
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": draft, "parameters": values},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == sorted(bindings)
- code = str(block["code"])
- terminal_selector = _p3_recorded_clicks(packet)[-1]["selector"]
- terminal_call = f'page.locator("{terminal_selector}").click()'
- for key, (selector, _value) in bindings.items():
- fill = f'page.locator("{selector}").fill(str({key}))'
- assert code.count(fill) == 1
- assert code.index(fill) < code.index(terminal_call)
- for interaction in _p3_recorded_clicks(packet):
- assert f'page.locator("{interaction["selector"]}")' in code
- authored = authored_selector_parameter_bindings(code, set(bindings))
- assert authored is not None
- for key, (selector, _value) in bindings.items():
- assert key in authored.get(selector, set())
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
-
- @pytest.mark.asyncio
- async def test_run1_recorded_packet_ambiguous_mapping_arms_value_free_directive_then_consumes(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- packet = _P3_PACKETS_BY_RUN["run1"]
- bindings = _P3_REPLAY_BINDINGS["run1"]
- values = {key: value for key, (_selector, value) in bindings.items()}
- ctx = _p3_definition_ctx()
- _enable_imposition(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- draft = _p3_draft_yaml(packet, bindings)
- ctx.scout_trajectory = _p3_scout_trajectory(packet)
- ctx.composition_page_evidence = _p3_composition_evidence(
- packet,
- bindings,
- extra_fields=[{"selector": "#scopeSearch", "value": values["account_number"]}],
- )
-
- first_result = json.loads(
- await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": draft,
- "block_labels": [packet["block_labels"][0]],
- "parameters": values,
- }
- ),
- )
- )
-
- assert first_result["ok"] is False
- repair_context = ctx.last_code_authoring_repair_context
- assert repair_context is not None
- directive = repair_context.parameter_binding_directive
- assert isinstance(directive, AuthoringParameterBindingDirective)
- candidate_pairs = {(candidate.declared_key, candidate.field_selector) for candidate in directive.candidates}
- assert candidate_pairs >= {
- ("account_number", "#accountNumberFilter"),
- ("account_number", "#scopeSearch"),
- }
- for value in values.values():
- assert value not in str(directive)
-
- fills = "".join(
- f'await page.locator("{selector}").fill(str({key}))\n' for key, (selector, _value) in bindings.items()
- )
- clicks = "".join(
- f'await page.locator("{interaction["selector"]}").click()\n' for interaction in _p3_recorded_clicks(packet)
- )
- authored_parsed = parse_workflow_yaml(draft)
- authored_block = _single_code_block(authored_parsed)
- authored_block["code"] = fills + clicks
- authored_block["parameter_keys"] = sorted(bindings)
- authored_yaml = yaml.safe_dump(authored_parsed, sort_keys=False)
-
- with capture_logs() as logs:
- second_result = await _update_workflow(
- {"workflow_yaml": authored_yaml, "parameters": values},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert second_result["ok"] is True
- assert any(log["event"] == "copilot authoring parameter binding directive consumed" for log in logs)
- final_block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))
- assert final_block["parameter_keys"] == sorted(bindings)
- final_code = str(final_block["code"])
- for key, (selector, _value) in bindings.items():
- assert final_code.count(f'page.locator("{selector}").fill(str({key}))') == 1
- assert workflow_update_module._definition_plane_preflight_reject(ctx, ctx.workflow_yaml) is None
-
- @pytest.mark.asyncio
- async def test_run1_recorded_halt_packet_negative_stays_byte_identical_without_targeting_fill(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- _stub_successful_update(monkeypatch)
- packet = _P3_PACKETS_BY_RUN["run1"]
- bindings = _P3_REPLAY_BINDINGS["run1"]
- values = {key: value for key, (_selector, value) in bindings.items()}
- ctx = _p3_definition_ctx()
- _enable_imposition(ctx)
- draft = _p3_draft_yaml(packet, bindings)
- for _ in range(2):
- assert (await _update_workflow({"workflow_yaml": draft}, ctx))["ok"] is False
- assert arm_recorded_outcome_grounding_requirement(ctx) is not None
- ctx.blocker_signal = ctx.turn_halt = ctx.turn_ownership = None
- ctx.scout_trajectory = _p3_scout_trajectory(packet)
- ctx.composition_page_evidence = {
- "source_tool": "inspect_page_for_composition",
- "current_url": _p3_recorded_clicks(packet)[0]["source_url"],
- "forms": [],
- }
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- result = await _update_workflow(
- {"workflow_yaml": draft, "parameters": values},
- ctx,
- allow_missing_credentials=True,
- )
-
- assert result["ok"] is False
- halt_packet = packet["turn_halts"][0]
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.extra == {
- "unresolved_parameter_keys": halt_packet["unresolved_parameter_keys"],
- "grounding_unresolved": halt_packet["grounding_unresolved"],
- }
- assert ctx.blocker_signal.user_facing_reason == (
- "I kept the workflow draft, but I could not safely connect the current page fields to these reusable "
- "inputs: `account_number`, `billing_end_date`, `billing_start_date`. No workflow run was started."
- )
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind.value == "definition_contract_unsatisfied"
-
- def test_definition_contract_blocker_signal_outputs_stay_byte_identical(self) -> None:
- packet = _P3_PACKETS_BY_RUN["run1"]
- recorded_keys = packet["turn_halts"][0]["unresolved_parameter_keys"]
- shuffled_keys = list(reversed(recorded_keys))
-
- grounding = build_definition_contract_unsatisfied_blocker_signal(
- unresolved_parameter_keys=shuffled_keys,
- grounding_unresolved=True,
- )
- assert grounding.user_facing_reason == (
- "I kept the workflow draft, but I could not safely connect the current page fields to these reusable "
- "inputs: `account_number`, `billing_end_date`, `billing_start_date`. No workflow run was started."
- )
- assert grounding.agent_steering_text == (
- "The exact candidate failed its definition contract before execution. Preserve the draft, stop this "
- "turn, and explain the unresolved reusable inputs without dispatching the candidate."
- )
- assert grounding.extra == {"unresolved_parameter_keys": recorded_keys, "grounding_unresolved": True}
-
- unreferenced = build_definition_contract_unsatisfied_blocker_signal(
- unresolved_parameter_keys=shuffled_keys,
- )
- assert unreferenced.user_facing_reason == (
- "I kept the workflow draft, but it cannot run because these reusable inputs are not used: "
- "`account_number`, `billing_end_date`, `billing_start_date`. Connect each input to the workflow "
- "before trying again."
- )
- assert unreferenced.extra == {"unresolved_parameter_keys": recorded_keys, "grounding_unresolved": False}
-
- keyless = build_definition_contract_unsatisfied_blocker_signal(unresolved_parameter_keys=[])
- assert keyless.user_facing_reason == (
- "I kept the workflow draft, but it does not yet satisfy the required workflow definition. "
- "Complete the missing definition requirements before trying again."
- )
+ error = workflow_update_module._definition_plane_reject_error(rejection)
+ # The contract is that the error names every unreferenced key and nothing else; the
+ # surrounding prose is free to change.
+ assert sorted(re.findall(r"`([^`]+)`", error)) == recorded_keys
class TestScoutedSpineRepeatedOmissionSeam:
@@ -23997,26 +14282,6 @@ class TestScoutedSpineRepeatedOmissionSeam:
def _uncovered_record(selector: str) -> dict[str, object]:
return {"tool_name": "click", "method": "click", "selector": selector, "trajectory_index": 1}
- def test_two_consecutive_identical_omissions_set_repeated_flag(self) -> None:
- ctx = _code_only_ctx()
- digest = workflow_update_module._scouted_spine_omission_digest([self._uncovered_record("#submit")])
- workflow_update_module._record_scouted_spine_omission_reject(ctx, digest)
- assert ctx.scouted_spine_repeated_identical_missing_steps is False
- assert ctx.scouted_spine_previous_omission_digest == digest
- workflow_update_module._record_scouted_spine_omission_reject(ctx, digest)
- assert ctx.scouted_spine_repeated_identical_missing_steps is True
-
- def test_changed_omission_clears_repeated_flag(self) -> None:
- ctx = _code_only_ctx()
- first = workflow_update_module._scouted_spine_omission_digest([self._uncovered_record("#a")])
- second = workflow_update_module._scouted_spine_omission_digest([self._uncovered_record("#b")])
- workflow_update_module._record_scouted_spine_omission_reject(ctx, first)
- workflow_update_module._record_scouted_spine_omission_reject(ctx, first)
- assert ctx.scouted_spine_repeated_identical_missing_steps is True
- workflow_update_module._record_scouted_spine_omission_reject(ctx, second)
- assert ctx.scouted_spine_repeated_identical_missing_steps is False
- assert ctx.scouted_spine_previous_omission_digest == second
-
def test_repeated_flag_reopens_imposition_for_attempt_three(self) -> None:
ctx = _code_only_ctx()
assert workflow_update_module._should_impose_after_update_attempt(ctx) is False
@@ -24026,70 +14291,6 @@ class TestScoutedSpineRepeatedOmissionSeam:
def _omitting_draft() -> str:
return _freehand_block_yaml('print(await page.locator("body").inner_text())')
- def test_two_consecutive_rejections_at_seam_force_attempt_three_impose_or_name(self) -> None:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- draft = self._omitting_draft()
-
- first = workflow_update_module._pre_persist_scouted_spine_result(draft, ctx)
- assert first is not None
- assert "#search-submit" in first.violations[0]
- workflow_update_module._record_scouted_spine_omission_reject(ctx, first.omission_digest)
- assert ctx.scouted_spine_repeated_identical_missing_steps is False
-
- second = workflow_update_module._pre_persist_scouted_spine_result(draft, ctx)
- assert second is not None
- assert second.omission_digest == first.omission_digest
- workflow_update_module._record_scouted_spine_omission_reject(ctx, second.omission_digest)
- assert ctx.scouted_spine_repeated_identical_missing_steps is True
-
- assert workflow_update_module._current_draft_repeats_prior_scouted_spine_omission(draft, ctx) is True
- assert workflow_update_module._should_impose_after_update_attempt(ctx, repeated_identical_omission=True) is True
- third = workflow_update_module._pre_persist_scouted_spine_result(draft, ctx)
- assert third is not None
- assert "#search-submit" in third.violations[0]
-
- @pytest.mark.asyncio
- async def test_third_differing_draft_not_admitted_after_two_identical_omissions(self) -> None:
- ctx = _code_only_ctx()
- _enable_imposition(ctx)
- ctx.scout_trajectory = [
- {
- "tool_name": "type_text",
- "selector": "#a",
- "source_url": "https://example.com/records",
- "trajectory_index": 0,
- },
- {
- "tool_name": "type_text",
- "selector": "#b",
- "source_url": "https://example.com/records",
- "trajectory_index": 1,
- },
- ]
- omitting = _freehand_block_yaml('await page.goto("https://example.com/records", wait_until="domcontentloaded")')
- ctx.update_workflow_called = True
-
- first = await _update_workflow({"workflow_yaml": omitting}, ctx)
- assert first["ok"] is False
- assert ctx.scouted_spine_repeated_identical_missing_steps is False
-
- second = await _update_workflow({"workflow_yaml": omitting}, ctx)
- assert second["ok"] is False
- assert ctx.scouted_spine_repeated_identical_missing_steps is True
- latched_digest = ctx.scouted_spine_previous_omission_digest
- assert latched_digest is not None
-
- differing = _freehand_block_yaml('await page.locator("#a").fill("x")')
- assert ctx.scouted_spine_previous_omission_digest == latched_digest
- assert workflow_update_module._current_draft_repeats_prior_scouted_spine_omission(omitting, ctx) is True
- assert workflow_update_module._current_draft_repeats_prior_scouted_spine_omission(differing, ctx) is False
-
- third = await _update_workflow({"workflow_yaml": differing}, ctx)
- assert third["ok"] is False
- assert ctx.spine_imposition_owned_attempt is False
- assert ctx.scouted_spine_previous_omission_digest != latched_digest
-
def test_returning_omission_after_differing_draft_does_not_misfire_imposition(self) -> None:
ctx = _code_only_ctx()
_enable_imposition(ctx)
diff --git a/tests/unit/test_copilot_code_mode_opt_out.py b/tests/unit/test_copilot_code_mode_opt_out.py
index 02466d345..888f1774b 100644
--- a/tests/unit/test_copilot_code_mode_opt_out.py
+++ b/tests/unit/test_copilot_code_mode_opt_out.py
@@ -257,19 +257,32 @@ def test_with_copilot_code_mode_metadata_preserves_turn_outcome_fields() -> None
def test_derive_copilot_code_mode_diagnostics_uses_context_state() -> None:
- ctx = type("Ctx", (), {})()
- ctx.last_test_ok = False
- ctx.last_failed_workflow_yaml = None
- ctx.code_native_pending_capability = "credential-typed code synthesis"
- ctx.turn_halt = type("Halt", (), {"kind": type("Kind", (), {"value": "repair_ceiling_reached"})()})()
-
- diagnostics = derive_copilot_code_mode_diagnostics(ctx)
+ ctx = SimpleNamespace(
+ last_test_ok=False,
+ last_failed_workflow_yaml=None,
+ code_native_pending_capability="credential-typed code synthesis",
+ turn_halt=SimpleNamespace(kind=SimpleNamespace(value="repair_ceiling_reached")),
+ )
- assert diagnostics == {
+ assert derive_copilot_code_mode_diagnostics(ctx) == {
"copilot_last_code_build_failed": True,
"copilot_repair_ceiling_hit": True,
"copilot_pending_capability": "credential-typed code synthesis",
- "copilot_schema_incompatibility": None,
+ }
+
+
+def test_derive_copilot_code_mode_diagnostics_on_a_clean_turn() -> None:
+ ctx = SimpleNamespace(
+ last_test_ok=True,
+ last_failed_workflow_yaml=None,
+ code_native_pending_capability=None,
+ turn_halt=None,
+ )
+
+ assert derive_copilot_code_mode_diagnostics(ctx) == {
+ "copilot_last_code_build_failed": False,
+ "copilot_repair_ceiling_hit": False,
+ "copilot_pending_capability": None,
}
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index a99f926b4..846765d01 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1823,29 +1823,6 @@ def test_terminal_goal_record_self_asserted_boolean_without_identifier_remains_u
)
-def test_submit_terminal_still_rejects_false_submit_with_author_time_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": _terminal_goal_payload(submitted=False)})
-
- assert (
- grade_terminal_goal_record_criteria(
- [
- _criterion(
- "c0",
- "a commercial water service request is submitted",
- kind="terminal_action",
- terminal_action_family="request",
- )
- ],
- snapshot,
- )
- == []
- )
-
-
@pytest.mark.parametrize(
"payload",
[
@@ -11729,21 +11706,6 @@ async def test_maybe_run_completion_verification_unavailable_on_low_budget(monke
assert no_handler_result.is_fully_satisfied() is False
-async def test_completion_verification_still_fails_closed_with_author_time_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- _patch_completion_handler(monkeypatch, None)
-
- result = await _maybe_run_completion_verification(_run_ctx(), _clean_success_result(), time.monotonic())
-
- assert result is not None
- assert result.status == "unavailable"
- assert result.is_fully_satisfied() is False
-
-
def test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None:
ctx = SimpleNamespace(
completion_verification_result=CompletionVerificationResult("unavailable"),
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index e5e1495dd..978c863e0 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -35,7 +35,6 @@ from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module
-from skyvern.forge.sdk.copilot import enforcement as enforcement_module
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import RequestPolicyGuardrailInputs, _derive_turn_intent_on_context
from skyvern.forge.sdk.copilot.config import CopilotConfig
@@ -1030,7 +1029,7 @@ def test_elapsed_run_seconds_subtracts_pause_time() -> None:
@pytest.mark.asyncio
async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: pytest.MonkeyPatch) -> None:
- """A pause longer than the work budget is excluded when the loop resumes."""
+ """Fails on old code: a slow pause would consume the (tiny) real timeout budget."""
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.15)
ctx = make_copilot_context()
@@ -1043,18 +1042,13 @@ async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: py
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.2)
- monotonic_seconds = 100.0
- fake_time = SimpleNamespace(monotonic=lambda: monotonic_seconds)
- monkeypatch.setattr(enforcement_module, "time", fake_time)
- monkeypatch.setattr(credential_pause_module, "time", fake_time)
-
- async def _resolve_after_pause(*args: Any, **kwargs: Any) -> CredentialPauseResolution:
- nonlocal monotonic_seconds
- monotonic_seconds += 0.25
- return CredentialPauseResolution(action="skip")
-
- monkeypatch.setattr(credential_pause_module, "_wait_for_credential_response", _resolve_after_pause)
+ async def _populate_after_first_poll() -> None:
+ await asyncio.sleep(0.25)
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-credit")] = encode_credential_response(
+ "skip", None
+ )
stream = _make_stream()
fake_result = _fake_result()
@@ -1072,17 +1066,20 @@ async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: py
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
- returned = await run_with_enforcement(
- agent=MagicMock(),
- initial_input="hello",
- ctx=ctx,
- stream=stream,
- run_config=RunConfig(),
- copilot_config=config,
- )
+ populate_task = asyncio.ensure_future(_populate_after_first_poll())
+ try:
+ returned = await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+ finally:
+ await populate_task
assert returned is fake_result
- assert ctx.copilot_credential_pause_seconds == 0.25
assert len(calls) == 2, "pause time must be credited so the resumed iteration isn't timed out"
assert ctx.copilot_total_timeout_exceeded is False
@@ -1491,34 +1488,40 @@ async def test_missing_credential_run_failure_pauses_the_loop(monkeypatch: pytes
# ---------------------------------------------------------------------------
-async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypatch: pytest.MonkeyPatch) -> None:
- """Fails on old code: the flag stays True after the second, successful call."""
+def _skip_flag_workflow_yaml() -> str:
+ return "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
+
+
+def _skip_flag_ctx() -> CopilotContext:
ctx = make_copilot_context()
ctx.turn_intent = TurnIntent(
mode=TurnIntentMode.BUILD,
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
)
ctx.request_policy = RequestPolicy()
+ return ctx
- workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
- async def fake_prior_definition(update_ctx: Any) -> object:
- return None
+async def _no_prior_definition(update_ctx: CopilotContext) -> object:
+ return None
+
- async def fake_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:
+async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _skip_flag_ctx()
+
+ async def fake_update_workflow(payload: dict, update_ctx: CopilotContext, **kwargs: object) -> dict:
workflow = SimpleNamespace(workflow_definition={"blocks": [{"label": "step_one"}]})
update_ctx.last_workflow = workflow
update_ctx.last_update_block_count = 1
return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}
- async def fake_run_blocks(params: dict, run_ctx: Any, **kwargs: object) -> dict:
+ async def fake_run_blocks(params: dict, run_ctx: CopilotContext, **kwargs: object) -> dict:
return {"ok": True, "data": {"workflow_run_id": "wr-1", "overall_status": "completed", "blocks": []}}
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", _no_prior_definition)
monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["step_one"], {}, "step_one"))
monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
@@ -1526,9 +1529,10 @@ async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypat
monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
- call_args = json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}})
+ call_args = json.dumps(
+ {"workflow_yaml": _skip_flag_workflow_yaml(), "block_labels": ["step_one"], "parameters": {}}
+ )
- # Call 1: policy forces a skip (unbound credential).
monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)
result_1 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args
@@ -1536,7 +1540,6 @@ async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypat
assert json.loads(result_1)["data"]["skip_reason"] == "workflow_credential_inputs_unbound"
assert ctx.last_run_skipped_unbound_credentials is True
- # Call 2: credential now bound, policy allows the real run.
monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
result_2 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args
@@ -1550,36 +1553,23 @@ async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypat
async def test_last_run_skipped_flag_stays_false_when_update_workflow_fails_before_skip_branch(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """Fails on old code: the flag was set True from the policy check alone,
- before _update_workflow ever ran, so an unrelated authoring failure got
- misreported as a credential ask.
- """
- ctx = make_copilot_context()
- ctx.turn_intent = TurnIntent(
- mode=TurnIntentMode.BUILD,
- authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
- )
- ctx.request_policy = RequestPolicy()
- workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
-
- async def fake_prior_definition(update_ctx: Any) -> object:
- return None
+ """The policy would allow a skip, but the authoring failure lands first — reporting it as a
+ credential ask would misattribute an unrelated error."""
+ ctx = _skip_flag_ctx()
- async def failing_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:
+ async def failing_update_workflow(payload: dict, update_ctx: CopilotContext, **kwargs: object) -> dict:
return {"ok": False, "error": "workflow_yaml is not valid: bad block reference"}
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", _no_prior_definition)
monkeypatch.setattr(tools_module, "_update_workflow", failing_update_workflow)
monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
- # The policy would allow a skip if we got that far — but we never do.
monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)
result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}}),
+ json.dumps({"workflow_yaml": _skip_flag_workflow_yaml(), "block_labels": ["step_one"], "parameters": {}}),
)
assert json.loads(result)["ok"] is False
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index c9d9b4fd2..b5eccc4ff 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -4,16 +4,13 @@ import hashlib
import json
from collections.abc import Callable
from types import SimpleNamespace
-from unittest.mock import AsyncMock
import pytest
from structlog.testing import capture_logs
-from skyvern.config import settings
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
- recorded_outcome_from_author_time_reject,
)
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
@@ -445,18 +442,6 @@ def _uncovered_output_turn_state(output_path: str) -> SimpleNamespace:
return SimpleNamespace(decision=SimpleNamespace(criteria=(criterion,)))
-def _uncovered_output_author_reject(output_path: str) -> RecordedBuildTestOutcome:
- return recorded_outcome_from_author_time_reject(
- reason_code="metadata_reject",
- block_labels=["extract_order"],
- structural_payload={
- "reason_code": "recorded_outcome_missing_output_coverage",
- "missing_output_paths": [output_path],
- },
- missing_requested_output_facts=[{"output_path": output_path, "output_root": output_path.split(".", 1)[0]}],
- )
-
-
def _run_repair_loop_state(ctx: CopilotContext) -> RepairLoopState:
repair_context = CodeAuthoringRepairContext(
block_label="extract_order",
@@ -472,25 +457,6 @@ def _run_repair_loop_state(ctx: CopilotContext) -> RepairLoopState:
return contract.repair_loop_state
-def test_uncovered_output_author_reject_reopens_once_then_counts_to_ceiling() -> None:
- ctx = _ctx()
- ctx.completion_criteria_turn_state = _uncovered_output_turn_state("output.document_name")
- ctx.latest_recorded_build_test_outcome = _uncovered_output_author_reject("output.document_name")
-
- first = _run_repair_loop_state(ctx)
- assert first.consecutive_identical_repair_count == 0
- assert first.ceiling_reached is False
- assert ctx.synthesized_block_reopened_for_output_coverage is True
- assert ctx.consecutive_non_converging_repair_count == 0
-
- states = [_run_repair_loop_state(ctx) for _ in range(3)]
- counts = [state.consecutive_identical_repair_count for state in states]
- assert counts == [1, 2, 3]
- assert counts[-1] == settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL
- assert states[-1].ceiling_reached is True
- assert ctx.blocker_signal is None
-
-
def test_persisted_run_outcome_is_not_excluded_from_repair_streak() -> None:
ctx = _ctx()
ctx.completion_criteria_turn_state = _uncovered_output_turn_state("output.document_name")
@@ -504,26 +470,9 @@ def test_persisted_run_outcome_is_not_excluded_from_repair_streak() -> None:
)
state = _run_repair_loop_state(ctx)
assert state.consecutive_identical_repair_count == 1
- assert ctx.synthesized_block_reopened_for_output_coverage is False
repeat = _run_repair_loop_state(ctx)
assert repeat.consecutive_identical_repair_count == 2
- assert ctx.synthesized_block_reopened_for_output_coverage is False
-
-
-def test_uncovered_output_reopen_preserves_prior_streak_count() -> None:
- ctx = _ctx()
- assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 1
- assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 2
-
- ctx.completion_criteria_turn_state = _uncovered_output_turn_state("output.document_name")
- ctx.latest_recorded_build_test_outcome = _uncovered_output_author_reject("output.document_name")
- reopened = _run_repair_loop_state(ctx)
-
- assert ctx.synthesized_block_reopened_for_output_coverage is True
- assert reopened.consecutive_identical_repair_count == 2
- assert reopened.ceiling_reached is False
- assert ctx.consecutive_non_converging_repair_count == 2
def test_failed_run_finalizes_runtime_authoring_repair_context_after_matching_page_observation() -> None:
@@ -1149,34 +1098,6 @@ def test_runtime_authoring_repair_context_sanitizes_failure_and_page_summaries()
assert repair_context.current_origin == "https://example.test"
-def test_schema_incompatibility_failure_type_stops_without_repair() -> None:
- # SKY-11380: the typed schema-incompatibility reject must route to STOP, never repair,
- # so the agent reports the mismatch instead of churning toward repair_ceiling_reached.
- contract = build_diagnosis_repair_contract(
- source_tool="update_and_run_blocks",
- result={
- "ok": False,
- "error": "STOP: the edited extraction_schema declares field(s) [shoebox] that map to no output.",
- "data": {
- "failure_type": "schema_incompatibility",
- "workflow_updated": False,
- "schema_incompatibility": {
- "block_label": "capture_row",
- "incompatible_paths": ["shoebox"],
- "known_output_paths": ["order_date", "order_total"],
- },
- },
- },
- ctx=_ctx(),
- workflow_updated=False,
- )
-
- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY
- assert contract.repair_decision.next_action == RepairNextAction.STOP
- assert contract.repair_decision.next_action != RepairNextAction.REPAIR
- assert contract.repair_decision.target_blocks == []
-
-
def test_judge_confirmed_suspicious_success_forces_no_change() -> None:
ctx = _ctx()
ctx.last_test_suspicious_success = True
@@ -2301,39 +2222,6 @@ def test_contract_trace_exposes_stable_root_cause_identity() -> None:
}
-async def test_update_workflow_failure_records_diagnosis_contract(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge.sdk.copilot import tools as tools_module
-
- error = (
- "Unable to impose synthesized code block: dropped scout interaction 0 from `click` (ambiguous_bare_selector)."
- )
- recorded: list[tuple[str, dict[str, object], str]] = []
-
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_request_policy_allows_credential_deferred_draft", lambda *args: False)
- monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_record_workflow_update_result", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_workflow", AsyncMock(return_value={"ok": False, "error": error}))
-
- def fake_record_contract(ctx: CopilotContext, *, source_tool: str, result: dict[str, object]) -> None:
- contract = build_diagnosis_repair_contract(source_tool=source_tool, result=result, ctx=ctx)
- recorded.append((source_tool, result, contract.to_trace_data()["root_cause_error_class"]))
-
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", fake_record_contract)
-
- result = await tools_module.update_workflow_tool.on_invoke_tool(
- SimpleNamespace(context=_ctx(), tool_name="update_workflow"),
- json.dumps({"workflow_yaml": "title: Test\nworkflow_definition:\n parameters: []\n blocks: []\n"}),
- )
-
- assert json.loads(result)["ok"] is False
- assert recorded == [
- ("update_workflow", {"ok": False, "error": error}, "code_block_synthesis_ambiguous_bare_selector")
- ]
-
-
def test_diagnosis_tool_error_preserves_terminal_challenge_blocker_category() -> None:
from skyvern.forge.sdk.copilot.tools.run_execution import _diagnosis_repair_tool_error
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 6d7179791..4af09ae67 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -20,7 +20,6 @@ from structlog.testing import capture_logs
from skyvern.config import Settings, settings
from skyvern.forge.sdk.copilot.blocker_signal import (
- UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
CopilotToolBlockerSignal,
stash_blocker_signal,
)
@@ -29,9 +28,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
PostRunPagePathTarget,
RecordedBuildTestOutcome,
_post_run_page_path_failure,
- author_time_reject_missing_output_paths,
bind_post_run_page_path_failure,
- recorded_outcome_from_author_time_reject,
)
from skyvern.forge.sdk.copilot.code_block_synthesis import SynthesizedCodeBlock
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
@@ -46,7 +43,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
KEEP_RECENT_TOOL_OUTPUTS,
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
CopilotGoalSatisfied,
- _canonical_output_path,
_check_enforcement,
_maybe_synthesized_block_offer_msg,
_needs_suspicious_success_nudge,
@@ -57,10 +53,8 @@ from skyvern.forge.sdk.copilot.enforcement import (
_should_force_advisory_run_dispatch,
_should_force_synthesized_block_persistence,
_summarize_tool_output,
- _uncovered_output_reject_admits_evaluate,
aggressive_prune,
arm_credential_scout_reopen,
- consume_uncovered_output_reopen_event,
mint_scout_observation_contract_for_ctx,
pre_run_gated_outputs_without_path,
record_scouted_output_coverage,
@@ -72,7 +66,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
synthesized_persistence_reopened_after_failed_run,
synthesized_trajectory_is_goal_complete,
synthesized_trajectory_reaches_goal,
- uncovered_output_reject_scout_steer_signal,
uncovered_requested_output_paths,
)
from skyvern.forge.sdk.copilot.mcp_adapter import (
@@ -89,8 +82,6 @@ from skyvern.forge.sdk.copilot.output_contracts import (
from skyvern.forge.sdk.copilot.output_extraction_plan import ShapeExpectation, ValueCardinality, ValueShape
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
-from skyvern.forge.sdk.copilot.runtime import NeverCapturedObligation
-from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool
from skyvern.forge.sdk.copilot.tools import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
_analyze_run_blocks,
@@ -115,11 +106,6 @@ from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerification
from tests.unit.conftest import make_copilot_context
[email protected](autouse=True)
-def _disable_author_time_gate_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", False)
-
-
class _Ctx:
"""Minimal stand-in for CopilotContext used in enforcement checks.
@@ -156,10 +142,7 @@ class _Ctx:
self.latest_diagnosis_repair_contract = None
self.last_code_authoring_repair_context = None
self.synthesized_block_reopened_after_failed_run = False
- self.synthesized_block_reopened_for_output_coverage = False
self.synthesized_block_reopened_for_credential_scout = False
- self.synthesized_block_reopened_for_capture_obligation = False
- self.never_captured_obligation = None
self.credential_scout_rescout_context_key = None
self.synthesized_goal_complete_landed = False
self.impose_synthesized_code_block = False
@@ -169,8 +152,6 @@ class _Ctx:
self.flow_evidence: list[dict[str, object]] = []
self.composition_page_evidence = None
self.copilot_config: CopilotConfig | None = None
- self.uncovered_output_rescout_context_key = None
- self.uncovered_output_rescout_steer_key = None
self.latest_recorded_build_test_outcome = None
self.last_run_blocks_workflow_run_id = None
self.post_run_page_observation_tool = None
@@ -183,8 +164,6 @@ class _Ctx:
self.workflow_verification_evidence = WorkflowVerificationEvidence()
self.completion_criteria_turn_state = None
self.reached_download_target: ReachedDownloadTarget | None = None
- self.author_time_gate_log_only_ids: frozenset[str] = frozenset()
- self.author_time_gate_ablation_events = []
self.request_policy = None
self.blocker_signal = None
self.blocker_signal_claimant = None
@@ -728,38 +707,6 @@ class TestSynthesizedOfferPersistenceGate:
assert synthesized_block_persistence_signal(ctx, "click") is None
- def test_never_captured_obligation_admits_only_its_expected_scout_tool(self) -> None:
- ctx = _Ctx()
- ctx.turn_id = "turn-a"
- ctx.turn_intent = TurnIntent(
- mode=TurnIntentMode.BUILD,
- authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
- )
- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
- ctx.synthesized_block_offered = True
- ctx.synthesized_block_offered_trajectory_len = 1
- ctx.synthesized_block_offered_goal_complete = True
- ctx.scout_trajectory = [{"tool_name": "click", "selector": "#existing"}]
- ctx.never_captured_obligation = NeverCapturedObligation(
- identity_digest="identity",
- turn_id=ctx.turn_id,
- draft_fingerprint="draft",
- block_label="submit",
- site="whole_trajectory",
- method="click",
- normalized_receiver="page.locator('#submit')",
- call_shape_digest="shape",
- expected_tool_name="click",
- armed_after_trajectory_index=0,
- )
-
- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#submit"}) is None
- assert isinstance(
- synthesized_block_persistence_signal(ctx, "click", {"selector": "#other"}),
- CopilotToolBlockerSignal,
- )
- assert isinstance(synthesized_block_persistence_signal(ctx, "type_text"), CopilotToolBlockerSignal)
-
def test_unresolved_recorded_outcome_blocks_page_mutating_tool_until_update_and_run_blocks(self) -> None:
ctx = _Ctx()
ctx.turn_intent = TurnIntent(
@@ -3751,30 +3698,6 @@ def _entry_commit_trajectory() -> list[dict[str, object]]:
]
-def _author_time_reject_outcome(*output_paths: str) -> RecordedBuildTestOutcome:
- paths = sorted(output_paths)
- return recorded_outcome_from_author_time_reject(
- reason_code="metadata_reject",
- block_labels=["extract_order"],
- structural_payload={
- "reason_code": "recorded_outcome_missing_output_coverage",
- "missing_output_paths": paths,
- "block_labels": ["extract_order"],
- "recorded_reason_code": "outcome_not_demonstrated",
- },
- observed_evidence_summary="missing requested output coverage",
- missing_requested_output_facts=[
- {
- "output_path": path,
- "output_root": path.split(".", 1)[0],
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- }
- for path in paths
- ],
- )
-
-
class TestScoutOutputCoverageGate:
def _authoring_ctx(self, *criteria: CompletionCriterion) -> _Ctx:
ctx = _Ctx()
@@ -4352,251 +4275,18 @@ class TestScoutOutputCoverageGate:
)
assert uncovered_requested_output_paths(ctx) == set()
- def test_reopen_fires_after_prior_run_then_author_reject(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.last_run_blocks_workflow_run_id = "wr_prior_run"
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is True
- assert consume_uncovered_output_reopen_event(ctx) is True
- assert ctx.synthesized_block_reopened_for_output_coverage is True
-
- def test_stream_recorded_authoring_success_clears_latch_and_steer_key(self) -> None:
- ctx = _Ctx()
- ctx.synthesized_block_reopened_for_output_coverage = True
- ctx.uncovered_output_rescout_steer_key = "steer-key"
- ctx.uncovered_output_rescout_context_key = "context-key"
- _update_enforcement_from_tool(ctx, "update_workflow", {"ok": True, "data": {"block_count": 1}})
- assert ctx.synthesized_block_reopened_for_output_coverage is False
- assert ctx.uncovered_output_rescout_steer_key is None
- assert ctx.uncovered_output_rescout_context_key == "context-key"
-
- def test_recorded_workflow_update_clears_latch_and_steer_key(self) -> None:
- ctx = make_copilot_context("title: Updated")
- ctx.synthesized_block_reopened_for_output_coverage = True
- ctx.uncovered_output_rescout_steer_key = "steer-key"
- ctx.uncovered_output_rescout_context_key = "context-key"
- _record_workflow_update_result(
- ctx,
- {
- "ok": True,
- "data": {"block_count": 1},
- "_workflow": SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()])),
- },
- )
- assert ctx.synthesized_block_reopened_for_output_coverage is False
- assert ctx.uncovered_output_rescout_steer_key is None
- assert ctx.uncovered_output_rescout_context_key == "context-key"
-
- def test_accessor_empty_when_no_recorded_outcome(self) -> None:
- assert author_time_reject_missing_output_paths(None) == set()
-
- def test_fact_paths_canonicalize_into_uncovered_set(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- outcome = _author_time_reject_outcome("output.document_name")
- canonical = {_canonical_output_path(path) for path in author_time_reject_missing_output_paths(outcome)}
- assert canonical & uncovered_requested_output_paths(ctx) == {"output.document_name"}
-
- def test_admission_allows_evaluate_while_uncovered_output_reject_active(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is True
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is True
- assert synthesized_block_persistence_signal(ctx, "evaluate") is None
-
- def test_consume_reopen_event_arms_latch_fire_once(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- assert consume_uncovered_output_reopen_event(ctx) is True
- assert ctx.synthesized_block_reopened_for_output_coverage is True
- assert synthesized_persistence_reopened(ctx) is True
- assert consume_uncovered_output_reopen_event(ctx) is False
-
- def test_steer_redirects_reauthor_to_scout_once_then_lets_through(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- consume_uncovered_output_reopen_event(ctx)
- steer = uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks")
- assert isinstance(steer, CopilotToolBlockerSignal)
- assert steer.cleared_by_tools == frozenset({"evaluate"})
- assert steer.renders_final_reply is False
- assert "output.document_name" in steer.agent_steering_text
- assert steer.extra["uncovered_output_paths"] == ["output.document_name"]
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is None
-
- def test_steer_redirects_update_workflow_reauthor_before_clear_consumes_reopen(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- consume_uncovered_output_reopen_event(ctx)
- steer = uncovered_output_reject_scout_steer_signal(ctx, "update_workflow")
- assert isinstance(steer, CopilotToolBlockerSignal)
- assert steer.cleared_by_tools == frozenset({"evaluate"})
- assert steer.blocked_tool == "update_workflow"
-
- def test_log_only_rescout_steer_records_without_consuming(
- self,
- monkeypatch: pytest.MonkeyPatch,
- ) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- consume_uncovered_output_reopen_event(ctx)
-
- steer = uncovered_output_reject_scout_steer_signal(ctx, "update_workflow")
-
- assert steer is None
- assert ctx.uncovered_output_rescout_steer_key is None
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == "uncovered_output_rescout_steer"
- assert event.reason_code == UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE
- assert event.blocked_tool == "update_workflow"
- assert event.fingerprint == (
- f"{ctx.latest_recorded_build_test_outcome.structural_failure_identity}|output.document_name"
- )
- assert event.log_only is True
-
- def test_steer_yields_to_live_ladder_and_one_shot_key_survives(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- consume_uncovered_output_reopen_event(ctx)
- ctx.output_contract_actuation_by_signature = {"sig_a": OutputContractAdvisoryState.GRANTED}
- ctx.output_contract_actuation_count_by_signature = {}
-
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is None
- assert ctx.uncovered_output_rescout_steer_key is None
-
- ctx.output_contract_actuation_by_signature = {"sig_a": OutputContractAdvisoryState.CONSUMED}
- steer = uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks")
- assert isinstance(steer, CopilotToolBlockerSignal)
- assert ctx.uncovered_output_rescout_steer_key is not None
-
- def test_steer_inert_without_reopen_latch(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is None
-
- def test_no_contradictory_blockers_steer_and_force_persist_disjoint(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- consume_uncovered_output_reopen_event(ctx)
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is not None
- assert _should_force_synthesized_block_persistence(ctx) is False
- ctx.scouted_output_covered_paths = {"output.document_name"}
- ctx.uncovered_output_rescout_steer_key = None
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is None
-
- def test_persisted_run_outcome_does_not_trigger_author_reopen(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.completion_verification_result = TestSynthesizedOfferPersistenceGate._unsatisfied_verification()
- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
- phase="persisted_block_run",
- attempted_tool="update_and_run_blocks",
- verdict="repairable_failure",
- reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:unsatisfied-output",
- missing_requested_output_facts=[
- {
- "output_path": "output.document_name",
- "output_root": "output",
- "reason_code": "outcome_not_demonstrated",
- "value_status": "no_typed_value",
- }
- ],
- )
- assert author_time_reject_missing_output_paths(ctx.latest_recorded_build_test_outcome) == set()
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is False
- assert consume_uncovered_output_reopen_event(ctx) is False
- assert ctx.synthesized_block_reopened_for_output_coverage is False
- assert uncovered_output_reject_scout_steer_signal(ctx, "update_and_run_blocks") is None
- assert isinstance(synthesized_block_persistence_signal(ctx, "click"), CopilotToolBlockerSignal)
-
- def test_author_reject_inert_when_no_missing_output_facts(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.latest_recorded_build_test_outcome = recorded_outcome_from_author_time_reject(
- reason_code="metadata_reject",
- block_labels=["extract_order"],
- structural_payload={"reason_code": "recorded_outcome_missing_output_coverage"},
- )
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is False
- assert consume_uncovered_output_reopen_event(ctx) is False
- assert ctx.synthesized_block_reopened_for_output_coverage is False
-
- def test_author_reject_inert_when_paths_already_covered(self) -> None:
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- ctx.scouted_output_covered_paths = {"output.document_name"}
- ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("output.document_name")
- assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is False
- assert consume_uncovered_output_reopen_event(ctx) is False
-
- def test_coverage_reopen_without_plan_does_not_refresh_offer(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.enforcement.synthesize_code_block",
- lambda *args, **kwargs: SynthesizedCodeBlock(code="await page.click('button')"),
- )
- ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
- ctx.update_workflow_called = True
- assert _maybe_synthesized_block_offer_msg(ctx) is None
- ctx.synthesized_block_reopened_for_output_coverage = True
- assert _maybe_synthesized_block_offer_msg(ctx) is None
-
def test_post_hook_failure_rolls_back_coverage_credit(self) -> None:
assert "scouted_output_covered_paths" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
- assert "synthesized_block_reopened_for_output_coverage" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
assert "synthesized_business_required_parameter_keys" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
ctx = _Ctx()
ctx.scouted_output_covered_paths = {"output.document_name"}
ctx.synthesized_business_required_parameter_keys = {"service_address"}
- ctx.synthesized_block_reopened_for_output_coverage = False
snapshot = _snapshot_post_hook_context(ctx)
ctx.scouted_output_covered_paths.add("output.leaked")
ctx.synthesized_business_required_parameter_keys.add("leaked_input")
- ctx.synthesized_block_reopened_for_output_coverage = True
_restore_post_hook_context(ctx, snapshot)
assert ctx.scouted_output_covered_paths == {"output.document_name"}
assert ctx.synthesized_business_required_parameter_keys == {"service_address"}
- assert ctx.synthesized_block_reopened_for_output_coverage is False
-
- def test_post_hook_failure_rolls_back_capture_obligation_credit(self) -> None:
- assert "never_captured_obligation" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
- assert "synthesized_block_reopened_for_capture_obligation" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
- ctx = _Ctx()
- armed = NeverCapturedObligation(
- identity_digest="identity",
- turn_id="turn-a",
- draft_fingerprint="draft",
- block_label="submit",
- site="whole_trajectory",
- method="click",
- normalized_receiver="page.locator('#submit')",
- call_shape_digest="shape",
- expected_tool_name="click",
- armed_after_trajectory_index=0,
- )
- ctx.never_captured_obligation = armed
- snapshot = _snapshot_post_hook_context(ctx)
- ctx.never_captured_obligation = NeverCapturedObligation(
- **{
- **armed.__dict__,
- "captured_trajectory_index": 1,
- "state": "captured",
- }
- )
- ctx.synthesized_block_reopened_for_capture_obligation = True
-
- _restore_post_hook_context(ctx, snapshot)
-
- assert ctx.never_captured_obligation == armed
- assert ctx.synthesized_block_reopened_for_capture_obligation is False
def test_post_hook_failure_rolls_back_scout_observation_contract(self) -> None:
assert "scout_observation_contract" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
@@ -5321,44 +5011,3 @@ class TestCredentialScoutReopen:
assert _should_force_synthesized_block_persistence(ctx) is False
ctx.synthesized_block_reopened_for_credential_scout = True
assert _should_force_synthesized_block_persistence(ctx) is True
-
-
-class TestNeverCapturedObligationAdmission:
- def _offered_complete_ctx(self) -> _Ctx:
- helper = TestCredentialFlowGoalComplete()
- ctx = helper._ctx_with_inventory(
- helper._two_screen_full_login(),
- inventory={"cred_1": frozenset({"username", "password"})},
- )
- ctx.turn_id = "turn-capture"
- ctx.never_captured_obligation = NeverCapturedObligation(
- identity_digest="capture-identity",
- turn_id=ctx.turn_id,
- draft_fingerprint="draft",
- block_label="submit",
- site="whole_trajectory",
- method="click",
- normalized_receiver="page.locator('#gasSubmit')",
- call_shape_digest="shape",
- expected_tool_name="click",
- armed_after_trajectory_index=0,
- )
- return ctx
-
- def test_exact_target_admission_registers_precedence_claim(self) -> None:
- ctx = self._offered_complete_ctx()
-
- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#gasSubmit"}) is None
-
- assert ctx.turn_ownership is not None
- assert TurnClaimant.CAPTURE_OBLIGATION_REOPEN in ctx.turn_ownership.claims
- assert current_turn_owner(ctx) is None
-
- def test_same_tool_for_different_target_remains_blocked(self) -> None:
- ctx = self._offered_complete_ctx()
-
- signal = synthesized_block_persistence_signal(ctx, "click", {"selector": "#unrelated"})
-
- assert isinstance(signal, CopilotToolBlockerSignal)
- assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
- assert ctx.turn_ownership is None
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 57d036087..02eb59adf 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -444,7 +444,6 @@ class TestMCPFailedStepLoopDetection:
pending_scout_source_url="https://source",
pending_scout_typed_value="typed",
completion_criteria_turn_state=None,
- synthesized_block_reopened_for_output_coverage=False,
last_code_authoring_repair_context=None,
scouted_output_covered_paths=set(),
reached_download_target=None,
@@ -829,14 +828,19 @@ class TestMCPToolOverlayCompleteness:
for name in browser_tools:
assert overlays[name].requires_browser, f"{name} should have requires_browser=True"
- def test_intent_not_hidden_on_browser_tools(self) -> None:
+ def test_intent_hidden_on_element_action_tools(self) -> None:
+ """An `intent` runs a second LLM agent to pick the element, duplicating reasoning the
+ copilot loop already did. The element-action tools take a selector only."""
from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlays
overlays = _build_skyvern_mcp_overlays()
- tools_with_intent = {"click", "type_text", "scroll", "select_option", "press_key"}
- for name in tools_with_intent:
+ for name in {"click", "type_text", "select_option", "press_key"}:
hidden = overlays[name].hide_params or frozenset()
- assert "intent" not in hidden, f"{name} should NOT hide intent"
+ assert "intent" in hidden, f"{name} should hide intent"
+
+ # `scroll` deliberately keeps it: scrolling a named element into view is not the
+ # element-picking this removes, and the agent prompt still offers it.
+ assert "intent" not in (overlays["scroll"].hide_params or frozenset())
class TestNewToolOverlayConfigs:
@@ -862,7 +866,7 @@ class TestNewToolOverlayConfigs:
from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlays
overlay = _build_skyvern_mcp_overlays()["select_option"]
- assert overlay.hide_params == frozenset({"session_id", "cdp_url", "timeout"})
+ assert overlay.hide_params == frozenset({"session_id", "cdp_url", "timeout", "intent"})
assert overlay.required_overrides == ["value"]
assert overlay.requires_browser is True
assert overlay.timeout == 15
@@ -872,25 +876,24 @@ class TestNewToolOverlayConfigs:
from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlays
overlay = _build_skyvern_mcp_overlays()["press_key"]
- assert overlay.hide_params == frozenset({"session_id", "cdp_url"})
+ assert overlay.hide_params == frozenset({"session_id", "cdp_url", "intent"})
assert overlay.required_overrides == ["key"]
assert overlay.requires_browser is True
assert overlay.post_hook is not None
- def test_click_and_type_overlays_steer_selector_first(self) -> None:
- # The tool contract must steer toward selector-only (deterministic) acting;
- # an `intent` routes the action through a slow full-page AI scan, so the
- # description must not invite "both" as the default (regression guard).
+ def test_click_and_type_overlays_are_selector_only(self) -> None:
+ # Acting by selector is deterministic; an `intent` would spawn a second LLM agent to
+ # choose the element. The parameter is off the schema, so the description must not
+ # reference it and must point at re-observing when a selector fails (regression guard).
from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlays
overlays = _build_skyvern_mcp_overlays()
for name in ("click", "type_text"):
desc = overlays[name].description or ""
- assert "selector ALONE" in desc, f"{name} should steer toward selector-only"
- assert "slower full-page AI scan" in desc, f"{name} should name the intent cost"
- assert "or both for resilient targeting" not in desc, f"{name} must not invite both by default"
- # intent must remain available for the genuine no-selector case
- assert "intent" not in overlays[name].hide_params
+ assert "CSS selector" in desc, f"{name} should name the selector contract"
+ assert "intent" not in desc, f"{name} description must not reference intent"
+ assert "inspect the page again" in desc, f"{name} should steer to re-observation on failure"
+ assert "intent" in overlays[name].hide_params
def test_browser_action_overlays_force_direct_selector_mode(self) -> None:
# The copilot keeps deterministic selector actions by binding selector_mode="direct"
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index 6b350f842..2af1b68d9 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
assert_clean_user_facing_text,
loop_blocker_evidence_from_ctx,
)
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedOutcomeGroundingRequirement
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
@@ -185,6 +186,32 @@ def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:
assert dict(ctx.blocker_signal.extra) == {}
+def test_unsatisfied_grounding_requirement_does_not_suppress_loop_guards() -> None:
+ requirement = RecordedOutcomeGroundingRequirement(
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ structural_key="key",
+ workflow_run_id="wr_1",
+ block_labels=["block_a"],
+ )
+
+ consecutive_ctx = _ctx(consecutive_tool_tracker=_streak("click"))
+ consecutive_ctx.recorded_outcome_grounding_requirement = requirement
+ consecutive_payload = _tool_loop_error(consecutive_ctx, "click", None)
+
+ assert consecutive_payload is not None
+ assert isinstance(consecutive_ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert consecutive_ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+
+ failed_step_ctx = _ctx(failed_tool_step_tracker={tool_step_identity("update_workflow", {}): 3})
+ failed_step_ctx.recorded_outcome_grounding_requirement = requirement
+ failed_step_payload = _tool_loop_error(failed_step_ctx, "update_workflow", {})
+
+ assert failed_step_payload is not None
+ assert isinstance(failed_step_ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert failed_step_ctx.blocker_signal.internal_reason_code == "loop_detected_repeated_failed_step"
+
+
def test_consecutive_evaluate_loop_without_composition_steer_uses_generic_copy() -> None:
ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))
payload = _tool_loop_error(ctx, "evaluate", None)
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 1b7fdf6f9..d3f23ad61 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -7,21 +7,16 @@ OSS-synced: only synthetic labels and RFC-2606 placeholder identifiers.
from __future__ import annotations
-import dataclasses
import itertools
import json
from types import SimpleNamespace
-import pytest
from structlog.testing import capture_logs
-from skyvern.config import settings
from skyvern.forge.sdk.copilot import enforcement
from skyvern.forge.sdk.copilot.blocker_signal import (
- OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
- blocker_signal_is_genuinely_terminal,
build_output_source_unobservable_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
@@ -40,27 +35,12 @@ from skyvern.forge.sdk.copilot.output_contracts import (
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
- _bind_criterion_to_request_slot,
)
-from skyvern.forge.sdk.copilot.request_slots import (
- CanonicalRequestSlotV1,
- RequestSlotAntecedentFamily,
- RequestSlotPinability,
- RequestSlotPlane,
-)
-from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page
from skyvern.forge.sdk.copilot.runtime import (
- RejectedCodeArtifactMetadataCapture,
output_contract_ladder_unresolved,
- record_author_time_gate_ablation_event,
)
from skyvern.forge.sdk.copilot.tools import workflow_update as wu
from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind, turn_halt_from_blocker_signal
-from skyvern.forge.sdk.copilot.turn_ownership import (
- TurnClaimant,
- claim_and_stash_blocker_signal,
- current_turn_owner,
-)
from tests.unit.copilot_test_helpers import make_copilot_ctx
_ALL_SPLIT_BLOCKERS = [
@@ -74,11 +54,6 @@ _ALL_SPLIT_BLOCKERS = [
]
[email protected](autouse=True)
-def _disable_author_time_gate_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", False)
-
-
def test_resolver_is_total_over_the_evidence_family_matrix() -> None:
kinds = set()
for (
@@ -493,47 +468,6 @@ def _counter_ctx() -> SimpleNamespace:
)
-def test_counter_resets_on_structural_fingerprint_delta() -> None:
- ctx = _counter_ctx()
- required = {"output.confirmation_number"}
- first = wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_a"
- )
- second = wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_a"
- )
- assert (first, second) == (1, 2)
- changed = wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_b"
- )
- assert changed == 1
-
-
-def test_counter_holds_streak_on_cosmetic_churn() -> None:
- ctx = _counter_ctx()
- required = {"output.confirmation_number"}
- for expected in (1, 2, 3):
- count = wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_same"
- )
- assert count == expected
-
-
-def test_counter_resets_after_imposition_marker() -> None:
- ctx = _counter_ctx()
- required = {"output.confirmation_number"}
- wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_same"
- )
- signature = next(iter(ctx.output_contract_reject_count_by_signature))
- ctx.output_contract_imposed_since_last_reject_by_signature[signature] = True
- count = wu._record_output_contract_family_reject(
- ctx, required, reject_family="metadata_reject", authored_structural_fingerprint="fp_same"
- )
- assert count == 1
- assert ctx.output_contract_imposed_since_last_reject_by_signature[signature] is False
-
-
_STATIC_RETURN_BLOCKERS = ["static_return_envelope_unavailable"]
_PAGE_READ_CODE = "value = page.inner_text('#confirmation')\nresult = value"
_FINGERPRINT = "fp_stage_topology"
@@ -561,8 +495,6 @@ def _advisory_ctx() -> SimpleNamespace:
output_contract_bail_actuated_this_call=False,
submitted_code_artifact_metadata_snapshot=None,
rejected_code_artifact_metadata_captures=[],
- author_time_gate_log_only_ids=frozenset(),
- author_time_gate_ablation_events=[],
latest_recorded_build_test_outcome=None,
recorded_build_test_outcome_history=[],
scouted_output_covered_paths=set(),
@@ -600,174 +532,6 @@ def _actuate(ctx: SimpleNamespace, signature: str) -> OutputContractActuationKin
return actuation.kind
-def test_author_time_gate_log_only_requires_local_opt_in(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _advisory_ctx()
- monkeypatch.setattr(settings, "ENV", "prod")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- assert (
- record_author_time_gate_ablation_event(
- ctx,
- gate_id="metadata_run_preflight_reject",
- reason_code="metadata_contract_required_before_run",
- fingerprint="sig",
- blocked_tool="update_and_run_blocks",
- )
- is False
- )
- assert ctx.author_time_gate_ablation_events == []
-
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", False)
- assert (
- record_author_time_gate_ablation_event(
- ctx,
- gate_id="metadata_run_preflight_reject",
- reason_code="metadata_contract_required_before_run",
- fingerprint="sig",
- blocked_tool="update_and_run_blocks",
- )
- is False
- )
- assert ctx.author_time_gate_ablation_events == []
-
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- assert (
- record_author_time_gate_ablation_event(
- ctx,
- gate_id="metadata_run_preflight_reject",
- reason_code="metadata_contract_required_before_run",
- fingerprint="sig",
- blocked_tool="update_and_run_blocks",
- )
- is True
- )
- assert ctx.author_time_gate_ablation_events[-1].log_only is True
-
-
-def test_log_only_output_contract_actuation_records_without_granting(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- signature = "sig_log_only"
- ctx = _arm_static_return_advisory_ctx(signature)
-
- assert _actuate(ctx, signature) == OutputContractActuationKind.ADVISORY_RUN
-
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_actuation_count_by_signature == {}
- assert ctx.output_contract_declick_attempted_by_signature == {}
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == "output_contract_actuation"
- assert event.reason_code == "advisory_run"
- assert event.blocked_tool == "update_workflow"
- assert event.fingerprint == _FINGERPRINT
- assert event.log_only is True
-
-
-def test_log_only_terminal_stash_does_not_duplicate_output_contract_event(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- signature = "sig_terminal_log_only"
- ctx = _advisory_ctx()
- ctx.turn_halt = None
- _mark_consumed_empty_run(ctx, signature, page_extraction=True)
-
- actuation = wu._actuate_output_contract_bail(
- ctx,
- blockers=list(_STATIC_RETURN_BLOCKERS),
- target_code=_PAGE_READ_CODE,
- required_paths={"output.confirmation_number"},
- signature=signature,
- current_fingerprint=_FINGERPRINT,
- )
- assert actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL
- wu._stash_output_source_unobservable_terminal(
- ctx,
- reason_code=actuation.reason_code,
- required_paths={"output.confirmation_number"},
- block_label="collect_confirmation",
- signature=signature,
- blockers=list(_STATIC_RETURN_BLOCKERS),
- )
-
- assert len(ctx.author_time_gate_ablation_events) == 1
- assert ctx.author_time_gate_ablation_events[0].fingerprint == _FINGERPRINT
- assert ctx.turn_halt is None
-
-
-def test_log_only_metadata_preflight_records_without_rejecting_or_consuming(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- signature = "sig_metadata"
- ctx = _advisory_ctx()
- ctx.output_contract_actuation_by_signature[signature] = OutputContractAdvisoryState.GRANTED
- evaluation = SimpleNamespace(
- has_deficiencies=True,
- canonical_signature=signature,
- can_attempt_run=False,
- payload={"reason_code": "output_contract_required"},
- reason_code="output_contract_required",
- block_label="collect_confirmation",
- required_paths={"output.confirmation_number"},
- missing_metadata_paths=["output.confirmation_number"],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=[],
- observation_paths=set(),
- )
- monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))
- monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))
- monkeypatch.setattr(wu, "_evaluate_output_contract_for_code_block", lambda *args, **kwargs: evaluation)
-
- result = wu._metadata_contract_run_preflight_reject(ctx, "workflow: yaml", {})
-
- assert result is None
- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
- assert ctx.output_contract_reject_count_by_signature == {}
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == "metadata_run_preflight_reject"
- assert event.reason_code == "output_contract_required"
- assert event.blocked_tool == "update_and_run_blocks"
- assert event.fingerprint == wu._output_contract_structural_fingerprint("workflow: yaml", signature)
- assert event.log_only is True
-
-
-def test_log_only_metadata_preflight_skips_run_attemptable_without_advisory(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- ctx = _advisory_ctx()
- evaluation = SimpleNamespace(
- has_deficiencies=True,
- canonical_signature="sig_run_attemptable",
- can_attempt_run=True,
- payload={"reason_code": "output_contract_required"},
- reason_code="output_contract_required",
- block_label="collect_confirmation",
- required_paths={"output.confirmation_number"},
- missing_metadata_paths=["output.confirmation_number"],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=[],
- observation_paths=set(),
- )
- monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))
- monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))
- monkeypatch.setattr(wu, "_evaluate_output_contract_for_code_block", lambda *args, **kwargs: evaluation)
-
- result = wu._metadata_contract_run_preflight_reject(ctx, "workflow: yaml", {})
-
- assert result is None
- assert ctx.author_time_gate_ablation_events == []
- assert ctx.output_contract_reject_count_by_signature == {}
-
-
def _genuine_terminal_signal() -> CopilotToolBlockerSignal:
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
@@ -781,122 +545,6 @@ def _genuine_terminal_signal() -> CopilotToolBlockerSignal:
)
-def test_metadata_preflight_yields_to_live_ladder_and_records_conflict() -> None:
- ctx = make_copilot_ctx()
- ctx.output_contract_actuation_by_signature["sig_live"] = OutputContractAdvisoryState.GRANTED
-
- assert wu._metadata_preflight_reject_yields_to_ladder(ctx) is True
- assert any(
- event.owner == "output_contract_actuation" and event.yielded == "metadata_run_preflight_reject"
- for event in ctx.gate_precedence_conflict_events
- )
-
-
-def test_metadata_preflight_does_not_yield_without_a_live_grant() -> None:
- ctx = make_copilot_ctx()
-
- assert wu._metadata_preflight_reject_yields_to_ladder(ctx) is False
- assert ctx.gate_precedence_conflict_events == []
-
-
-def test_metadata_preflight_does_not_yield_during_ordinary_reject_without_a_grant() -> None:
- ctx = make_copilot_ctx()
- ctx.output_contract_actuation_count_by_signature["sig_landed"] = 1
- assert output_contract_ladder_unresolved(ctx) is True
-
- assert wu._metadata_preflight_reject_yields_to_ladder(ctx) is False
-
-
-def test_metadata_preflight_does_not_yield_to_a_genuine_terminal_owner() -> None:
- ctx = make_copilot_ctx()
- ctx.output_contract_actuation_by_signature["sig_live"] = OutputContractAdvisoryState.GRANTED
- terminal = _genuine_terminal_signal()
- claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal)
- ctx.turn_halt = turn_halt_from_blocker_signal(terminal, source="test")
- assert current_turn_owner(ctx).claimant is TurnClaimant.GENUINELY_TERMINAL
-
- assert wu._metadata_preflight_reject_yields_to_ladder(ctx) is False
-
-
-def test_metadata_preflight_reject_returns_none_at_call_site_while_ladder_owns(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- ctx = make_copilot_ctx()
- ctx.output_contract_actuation_by_signature["sig_live"] = OutputContractAdvisoryState.GRANTED
- monkeypatch.setattr(
- wu,
- "_recorded_outcome_convergence_reject",
- lambda *args, **kwargs: wu._ConvergenceReject("sig_target", "frontier_unchanged", False),
- )
- monkeypatch.setattr(wu, "_workflow_yaml_code_blocks_by_label", lambda *args: {})
- monkeypatch.setattr(wu, "_record_author_time_reject_outcome", lambda *args, **kwargs: None)
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda *args, **kwargs: None)
-
- result = wu._metadata_contract_run_preflight_reject(ctx, "workflow: yaml", {})
-
- assert result is None
- assert any(event.yielded == "metadata_run_preflight_reject" for event in ctx.gate_precedence_conflict_events)
-
-
-def test_metadata_preflight_reject_still_rejects_without_a_live_ladder(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- ctx = make_copilot_ctx()
- monkeypatch.setattr(
- wu,
- "_recorded_outcome_convergence_reject",
- lambda *args, **kwargs: wu._ConvergenceReject("sig_target", "frontier_unchanged", False),
- )
- monkeypatch.setattr(wu, "_workflow_yaml_code_blocks_by_label", lambda *args: {})
- monkeypatch.setattr(wu, "_record_author_time_reject_outcome", lambda *args, **kwargs: None)
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda *args, **kwargs: None)
-
- result = wu._metadata_contract_run_preflight_reject(ctx, "workflow: yaml", {})
-
- assert result is not None
- assert result["ok"] is False
-
-
-def test_credential_scout_submit_gate_still_blocks_with_author_time_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- ctx = SimpleNamespace(
- block_authoring_policy=wu.BlockAuthoringPolicy.CODE_ONLY_BROWSER,
- workflow_yaml="",
- scout_trajectory=[
- {
- "tool_name": "fill_credential_field",
- "credential_id": "cred_safe",
- "credential_field": "password",
- "source_url": "https://login.example.test/login",
- }
- ],
- )
- workflow_yaml = """
-workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: credential_id
- key: login_credentials
- default_value: cred_safe
- blocks:
- - block_type: code
- label: login
- parameter_keys:
- - login_credentials
- code: |
- await page.locator("#password").fill(login_credentials.password)
- await page.get_by_role("button", name="Submit").click()
-"""
-
- errors = wu._credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
-
- assert len(errors) == 1
- assert "a later submit action on the same page" in errors[0]
-
-
def test_advisory_grant_survives_double_preflight_pass() -> None:
signature = "sig_double_pass"
ctx = _arm_static_return_advisory_ctx(signature)
@@ -1169,44 +817,6 @@ def test_steer_only_never_terminals_click_only_and_skips_declick() -> None:
assert second.reason_code != OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE
-def test_value_bearing_reject_uses_steer_only_structure_directive() -> None:
- signature = "sig_value_bearing_steer"
- evaluation = _make_evaluation(
- signature,
- block_label="collect",
- shape_violations=["value_bearing_output_required"],
- )
- non_steer_ctx = _ladder_ctx()
-
- inert = wu._adjudicate_output_contract_ladder_after_reject(
- non_steer_ctx,
- evaluation,
- workflow_yaml=_PAGE_READ_YAML,
- current_fingerprint="value-bearing:fp",
- )
-
- assert inert is None
- assert non_steer_ctx.output_contract_armed_directive_fingerprint_by_signature == {}
-
- ctx = _ladder_ctx()
- actuation = wu._adjudicate_output_contract_ladder_after_reject(
- ctx,
- evaluation,
- workflow_yaml=_PAGE_READ_YAML,
- current_fingerprint="value-bearing:fp",
- steer_only=True,
- )
-
- assert actuation is not None
- assert actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE
- assert ctx.output_contract_armed_directive_fingerprint_by_signature[signature] == "value-bearing:fp"
- assert ctx.output_contract_actuation_by_signature == {}
- assert ctx.output_contract_pending_run_evidence == {}
- assert ctx.output_contract_actuation_count_by_signature == {}
- assert ctx.output_contract_declick_attempted_by_signature == {}
- assert ctx.turn_halt is None
-
-
def test_observed_required_values_exact_and_lineage_match() -> None:
ctx = SimpleNamespace(scouted_output_covered_paths={"output.order.id"}, composition_page_evidence=None)
assert wu._observed_required_output_values(ctx, {"output.order.id"}) is True
@@ -1333,192 +943,6 @@ def test_ladder_resolved_when_no_output_contract_activity() -> None:
assert output_contract_ladder_unresolved(_ladder_ctx()) is False
-def test_reject_seam_adjudication_grants_advisory_within_caps_when_imposition_early_outs() -> None:
- ctx = _ladder_ctx()
- signature = "sig_early_out"
- evaluation = _make_evaluation(signature, block_label="collect")
- for _ in range(wu._MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN + 1):
- ctx.output_contract_bail_actuated_this_call = False
- wu._adjudicate_output_contract_ladder_after_reject(
- ctx, evaluation, workflow_yaml=_PAGE_READ_YAML, current_fingerprint="fp_early_out"
- )
- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
- assert ctx.turn_halt is None
-
-
-def test_same_fresh_signature_emits_one_reject_before_typed_advisory(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _ladder_ctx()
- signature = "sig_first_contact_example"
- evaluation = _make_evaluation(signature, block_label="collect")
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: None)
-
- first = wu._record_output_contract_reject(
- ctx,
- evaluation,
- summary="First typed deficiency.",
- authored_structural_fingerprint="fp_first_contact",
- workflow_yaml=_PAGE_READ_YAML,
- )
- first_outcome = ctx.latest_recorded_build_test_outcome
- ctx.output_contract_bail_actuated_this_call = False
- second = wu._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Same typed deficiency.",
- authored_structural_fingerprint="fp_first_contact",
- workflow_yaml=_PAGE_READ_YAML,
- )
-
- assert first["output_contract_actuation"] == OutputContractActuationKind.STRUCTURE_DIRECTIVE.value
- assert first_outcome is not None
- assert first_outcome.phase == "author_time_reject"
- assert second["output_contract_actuation"] == OutputContractActuationKind.ADVISORY_RUN.value
- assert ctx.latest_recorded_build_test_outcome is None
- assert [entry["phase"] for entry in ctx.recorded_build_test_outcome_history] == ["author_time_reject"]
- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
-
-
-def test_has_metadata_reject_attaches_no_metadata_convergence_directive(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _ladder_ctx()
- evaluation = _make_evaluation("sig_complete_metadata", block_label="collect")
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: None)
- complete_metadata = [
- {
- "block_label": "collect",
- "declared_goal": "Collect the provider summary",
- "claimed_outcomes": ["summary extracted"],
- "page_dependencies": ["provider page"],
- "completion_criteria": ["summary non-empty"],
- "terminal_verifier_expectations": ["summary rendered"],
- "evidence_refs": ["scout:provider-page"],
- }
- ]
-
- payload = wu._record_output_contract_reject(
- ctx,
- evaluation,
- summary="Value-required deficiency on a metadata-complete block.",
- authored_structural_fingerprint="fp_complete_metadata",
- workflow_yaml=_PAGE_READ_YAML,
- raw_metadata=complete_metadata,
- )
-
- assert payload["output_contract_actuation"] == OutputContractActuationKind.STRUCTURE_DIRECTIVE.value
- assert "metadata_convergence_directive" not in payload
-
-
-def test_synthesized_metadata_less_reject_arms_convergence_directive() -> None:
- ctx = _ladder_ctx()
- ctx.turn_id = "turn-synth-metadata-less"
-
- with capture_logs() as logs:
- directive = wu._synthesized_metadata_reject_directive(
- ctx,
- workflow_yaml=_PAGE_READ_YAML,
- raw_metadata=[],
- label_candidates=["collect"],
- required_paths={"output.confirmation_number"},
- )
-
- assert directive is not None
- assert any(entry["event"] == "copilot_output_contract_spine_structure_directive_emitted" for entry in logs)
-
-
-def test_synthesized_metadata_complete_reject_arms_no_directive() -> None:
- ctx = _ladder_ctx()
- ctx.turn_id = "turn-synth-metadata-complete"
- complete_metadata = [
- {
- "block_label": "collect",
- "declared_goal": "Collect the confirmation number",
- "claimed_outcomes": ["confirmation captured"],
- "page_dependencies": ["confirmation page"],
- "completion_criteria": ["confirmation non-empty"],
- "terminal_verifier_expectations": ["confirmation rendered"],
- "evidence_refs": ["scout:confirmation"],
- }
- ]
-
- directive = wu._synthesized_metadata_reject_directive(
- ctx,
- workflow_yaml=_PAGE_READ_YAML,
- raw_metadata=complete_metadata,
- label_candidates=["collect"],
- required_paths={"output.confirmation_number"},
- )
-
- assert directive is None
-
-
-def test_reject_seam_metadata_required_reaches_advisory_before_no_source_terminal() -> None:
- ctx = _ladder_ctx()
- signature = "sig_metadata_required"
- ctx.output_contract_declick_attempted_by_signature[signature] = True
- for _ in range(wu._MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN):
- wu._record_output_contract_actuation_progress(ctx, signature)
- workflow_yaml = (
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: collect\n"
- " code: \"page.click('#submit')\"\n"
- )
-
- wu._adjudicate_output_contract_ladder_after_reject(
- ctx, _make_evaluation(signature, block_label="collect"), workflow_yaml=workflow_yaml, current_fingerprint="fp"
- )
-
- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
- assert ctx.turn_halt is None
-
-
-def test_reject_seam_adjudication_skips_when_imposition_already_actuated_this_call() -> None:
- ctx = _ladder_ctx()
- ctx.output_contract_bail_actuated_this_call = True
- signature = "sig_already_actuated"
- wu._adjudicate_output_contract_ladder_after_reject(
- ctx, _make_evaluation(signature, block_label="collect"), workflow_yaml=_PAGE_READ_YAML, current_fingerprint="fp"
- )
- assert signature not in ctx.output_contract_actuation_count_by_signature
- assert signature not in ctx.output_contract_actuation_by_signature
-
-
-def test_reject_seam_adjudication_skips_owner_ambiguity_without_owner_block() -> None:
- ctx = _ladder_ctx()
- signature = "sig_owner_ambiguous"
- wu._adjudicate_output_contract_ladder_after_reject(
- ctx, _make_evaluation(signature, block_label=""), workflow_yaml="", current_fingerprint="fp"
- )
- assert signature not in ctx.output_contract_actuation_count_by_signature
- assert signature not in ctx.output_contract_actuation_by_signature
-
-
-def test_reject_seam_adjudication_stashes_terminal_after_declick_cycle(monkeypatch: object) -> None:
- ctx = _ladder_ctx()
- signature = "sig_no_source"
- ctx.output_contract_declick_attempted_by_signature[signature] = True
- stashed: list[str] = []
-
- def capture(_ctx: object, *, reason_code: str, **_kwargs: object) -> None:
- stashed.append(reason_code)
-
- monkeypatch.setattr(wu, "_stash_output_source_unobservable_terminal", capture) # type: ignore[attr-defined]
- workflow_yaml = (
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: collect\n"
- " code: \"page.click('#toggle-service')\"\n"
- )
- wu._adjudicate_output_contract_ladder_after_reject(
- ctx,
- _make_evaluation(signature, block_label="collect", shape_violations=list(_STRUCTURAL_BLOCKERS)),
- workflow_yaml=workflow_yaml,
- current_fingerprint="fp_no_source",
- )
- assert stashed == [OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE]
-
-
def test_observed_but_unbound_run_without_page_extraction_is_not_exhaustion() -> None:
signature = "sig_empty_run"
ctx = _advisory_ctx()
@@ -1727,55 +1151,6 @@ def test_advisory_grant_downgraded_to_directive_when_run_authority_forbids_dispa
assert signature not in ctx.output_contract_actuation_by_signature
-def test_loaded_result_carrier_is_selector_bound_and_claimed_by_one_contract_signature(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- ctx = _advisory_ctx()
- events: list[dict[str, object]] = []
-
- def capture_event(_ctx: object, **kwargs: object) -> bool:
- events.append(kwargs)
- return False
-
- monkeypatch.setattr(wu, "record_author_time_gate_ablation_event", capture_event)
- ctx.latest_evaluate_result_composition_steer = loaded_result_composition_evidence_from_page(
- {"result_containers": [{"selector": "#results", "row_count": 1, "sample_rows": ["Example customer record"]}]},
- source_tool="evaluate",
- source_url="https://example.com/results",
- )
-
- first = wu._actuate_output_contract_bail(
- ctx,
- blockers=["static_return_envelope_unavailable"],
- target_code='await page.locator("#results").click()',
- required_paths={"output.record_id"},
- signature="sig-a",
- current_fingerprint="fp-a",
- )
-
- assert ctx.latest_evaluate_result_composition_signature == "sig-a"
-
- ctx.output_contract_bail_actuated_this_call = False
- second = wu._actuate_output_contract_bail(
- ctx,
- blockers=["static_return_envelope_unavailable"],
- target_code='await page.locator("#results").click()',
- required_paths={"output.other_id"},
- signature="sig-b",
- current_fingerprint="fp-b",
- )
-
- assert ctx.latest_evaluate_result_composition_signature == "sig-a"
- assert first.kind is not OutputContractActuationKind.BLOCKED_TERMINAL
- assert second.kind is OutputContractActuationKind.STRUCTURE_DIRECTIVE
- first_payload = events[0]["payload"]
- second_payload = events[1]["payload"]
- assert isinstance(first_payload, dict)
- assert isinstance(second_payload, dict)
- assert first_payload["loaded_result_source_producible"] is True
- assert second_payload["loaded_result_source_producible"] is False
-
-
def _antecedent_ctx(*criteria: CompletionCriterion) -> CopilotContext:
ctx = make_copilot_ctx()
ctx.block_authoring_policy = wu.BlockAuthoringPolicy.CODE_ONLY_BROWSER
@@ -2167,45 +1542,6 @@ def test_face_c_degraded_mint_preserves_satisfiable_value_bearing_authoring_path
assert "None" not in skeleton
-def test_canonically_bound_shapeless_valid_criterion_stays_dispatchable() -> None:
- slot_hash = "a" * 48
- slot = CanonicalRequestSlotV1(
- slot_id="a" * 64,
- canonical_path=f"output.request_slot_{slot_hash}_00",
- path_segments=("output", f"request_slot_{slot_hash}_00"),
- ordinal=0,
- source_id="u0",
- source_start=0,
- source_end=10,
- plane=RequestSlotPlane.RUN,
- pinability=RequestSlotPinability.SHAPELESS_VALID,
- antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
- )
- bound = _bind_criterion_to_request_slot(
- CompletionCriterion(
- id="c1",
- outcome="The visible action label is returned.",
- output_path="output.visible_action_label",
- ),
- slot=slot,
- source_quote="visible action label",
- )
-
- assert bound.request_slot_id == slot.slot_id
- assert bound.output_path == "output.visible_action_label"
- assert bound.expected_output_shape == "value_present"
- assert bound.mint_disposition != "degraded"
- assert bound.mint_degrade is None
-
- ctx = _antecedent_ctx(bound)
- contract = wu._output_contract_required_paths_source(ctx)
-
- assert contract.degraded_request_slots == ()
- assert contract.liveness is not wu._OutputContractLiveness.DEGRADED_EMPTY
- code = 'label = await page.inner_text("#action")\nreturn {"output": {"visible_action_label": label}}\n'
- assert wu.output_contract_value_bearing_run_reject(ctx, {"b1": code}) is None
-
-
def test_classifier_minted_download_path_excluded_from_author_coverage() -> None:
ctx = _antecedent_ctx(
CompletionCriterion(
@@ -2412,234 +1748,6 @@ def test_observation_production_is_not_click_only_spine() -> None:
_METADATA_REQUIRED = {"output.confirmation_number"}
-def test_metadata_family_reject_holds_streak_across_fingerprint_rotation() -> None:
- ctx = _counter_ctx()
- for family in ("missing_code_artifact_metadata", "metadata_normalization", "output_contract_required"):
- sub = _counter_ctx()
- counts = [
- wu._record_output_contract_family_reject(
- sub, _METADATA_REQUIRED, reject_family=family, authored_structural_fingerprint=f"fp_{i}"
- )
- for i in range(4)
- ]
- assert counts == [1, 2, 3, 4], family
- imposed = wu._record_output_contract_family_reject(
- ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_0"
- )
- signature = next(iter(ctx.output_contract_reject_count_by_signature))
- ctx.output_contract_imposed_since_last_reject_by_signature[signature] = True
- after_imposition = wu._record_output_contract_family_reject(
- ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_1"
- )
- assert (imposed, after_imposition) == (1, 1)
-
-
-def test_out_of_family_reject_resets_on_fingerprint_rotation() -> None:
- ctx = _counter_ctx()
- first = wu._record_output_contract_family_reject(
- ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_a"
- )
- rotated = wu._record_output_contract_family_reject(
- ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_b"
- )
- assert (first, rotated) == (1, 1)
-
-
-def test_reject_budget_exhausted_reason_code_is_genuinely_terminal() -> None:
- signal = build_output_source_unobservable_blocker_signal(
- reason_code=OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
- required_paths=_METADATA_REQUIRED,
- block_label="collect",
- )
- assert blocker_signal_is_genuinely_terminal(signal) is True
-
-
-def test_metadata_family_budget_reaches_typed_terminal_at_bound_without_churn() -> None:
- ctx = make_copilot_ctx(turn_id="turn_budget_bound")
- terminal_at = None
- for candidate in range(1, 7):
- count = wu._record_output_contract_family_reject(
- ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata"
- )
- assert count == candidate
- exhausted = wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=count, block_label="collect")
- if exhausted and terminal_at is None:
- terminal_at = candidate
- assert terminal_at == 6
- assert ctx.turn_halt is not None
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
- assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True
- assert ctx.code_authoring_guardrail_reject_count == 0
-
-
-def test_budget_terminal_names_paths_and_omits_generic_churn_string() -> None:
- ctx = make_copilot_ctx(turn_id="turn_budget_copy")
- wu._stash_output_contract_reject_budget_terminal(
- ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_copy"
- )
- assert ctx.blocker_signal is not None
- user_facing = ctx.blocker_signal.user_facing_reason
- assert_clean_user_facing_text(user_facing)
- assert "output.confirmation_number" in user_facing
- assert "safety checks rejected each version" not in user_facing
- assert ctx.blocker_signal.renders_final_reply is True
-
-
-def test_main_gate_defers_churn_to_budget_ladder_when_reject_counted(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = make_copilot_ctx(turn_id="turn_budget_defer")
- calls: list[object] = []
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))
- count = wu._record_output_contract_reject(
- ctx,
- _make_evaluation("", block_label="collect"),
- summary="Typed deficiency.",
- authored_structural_fingerprint="fp_defer",
- workflow_yaml=_PAGE_READ_YAML,
- )
- assert count["output_contract_reject_count"] >= 1
- assert calls == []
-
-
-def test_undeclared_contract_rejects_count_toward_budget_and_reach_typed_terminal(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # A candidate deriving NO required child paths must still consume the reject budget: skipping
- # the count degrades repeated preflight rejects to the generic churn stop (loop_detected).
- ctx = make_copilot_ctx(turn_id="turn_budget_undeclared")
- calls: list[object] = []
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))
- malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}
- ctx.submitted_code_artifact_metadata_snapshot = malformed
-
- def undeclared_evaluation() -> wu._OutputContractEvaluation:
- return wu._OutputContractEvaluation(
- block_label="collect",
- artifact_id="",
- required_paths=set(),
- observation_paths=set(),
- declaration_paths=set(),
- source="requested_output",
- reason_code="output_contract_required",
- missing_metadata_paths=[],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=[],
- canonical_signature="",
- payload={},
- repair_context=None,
- )
-
- terminal_at = None
- for candidate in range(1, 8):
- payload = wu._record_output_contract_reject(
- ctx,
- undeclared_evaluation(),
- summary="Undeclared contract deficiency.",
- authored_structural_fingerprint=f"fp_{candidate}",
- workflow_yaml="",
- )
- assert payload["output_contract_reject_count"] == candidate
- if ctx.turn_halt is not None and terminal_at is None:
- terminal_at = candidate
- break
- assert terminal_at is not None and terminal_at <= 7
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
- assert calls == []
- assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)
-
-
-def test_recorded_identical_reject_shape_replays_to_budget_terminal_with_per_reject_capture(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # Replays the recorded live churn shape — identical value_bearing_output_required rejects on a
- # candidate deriving no required paths — through the real seam to budget exhaustion.
- ctx = make_copilot_ctx(turn_id="turn_recorded_shape_replay")
- churn_calls: list[object] = []
- monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: churn_calls.append(_ctx))
- malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}
- ctx.submitted_code_artifact_metadata_snapshot = malformed
-
- def recorded_shape_evaluation() -> wu._OutputContractEvaluation:
- return wu._OutputContractEvaluation(
- block_label="collect",
- artifact_id="",
- required_paths=set(),
- observation_paths=set(),
- declaration_paths=set(),
- source="requested_output",
- reason_code=wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
- missing_metadata_paths=[],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=[wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE],
- canonical_signature="",
- payload={"reason_code": wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE},
- repair_context=None,
- )
-
- terminal_at = None
- for candidate in range(1, 8):
- payload = wu._record_output_contract_reject(
- ctx,
- recorded_shape_evaluation(),
- summary="Identical value-bearing deficiency.",
- authored_structural_fingerprint="fp_identical",
- workflow_yaml="",
- )
- assert payload["output_contract_reject_count"] == candidate
- assert len(ctx.rejected_code_artifact_metadata_captures) == min(candidate, 5)
- if ctx.turn_halt is not None:
- terminal_at = candidate
- break
- assert terminal_at is not None and terminal_at <= 7
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
- assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True
- assert churn_calls == []
-
-
-def test_no_reject_leaves_no_budget_terminal() -> None:
- ctx = make_copilot_ctx(turn_id="turn_budget_yes")
- assert wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=0, block_label="collect") is False
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
-
-
-def test_rejected_code_artifact_metadata_capture_is_byte_exact_and_bounded() -> None:
- ctx = make_copilot_ctx(turn_id="turn_capture")
- malformed = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}
- ctx.submitted_code_artifact_metadata_snapshot = malformed
- wu._capture_rejected_code_artifact_metadata(ctx)
- assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)
- for index in range(7):
- ctx.submitted_code_artifact_metadata_snapshot = {"collect": {"variant": index}}
- wu._capture_rejected_code_artifact_metadata(ctx)
- captures = ctx.rejected_code_artifact_metadata_captures
- assert len(captures) == 5
- assert [c.payload["collect"]["variant"] for c in captures] == [2, 3, 4, 5, 6]
-
-
-def test_rejected_capture_log_carries_payload_kept_off_user_facing_text() -> None:
- ctx = make_copilot_ctx(turn_id="turn_capture_log")
- payload = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}
- ctx.submitted_code_artifact_metadata_snapshot = payload
- with capture_logs() as logs:
- wu._capture_rejected_code_artifact_metadata(ctx)
- entries = [line for line in logs if line.get("event") == "copilot_rejected_code_artifact_metadata_captured"]
- assert entries
- assert entries[0]["rejected_code_artifact_metadata_payload"] == payload
- assert entries[0]["submitted_metadata_labels"] == ["collect"]
-
- wu._stash_output_contract_reject_budget_terminal(
- ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_capture"
- )
- assert ctx.blocker_signal is not None
- assert "extraction_schema" not in ctx.blocker_signal.user_facing_reason
- assert "goal_value_paths" not in ctx.blocker_signal.user_facing_reason
-
-
def test_contract_satisfying_candidate_admits_run_no_reject() -> None:
ctx = _antecedent_ctx(
CompletionCriterion(
@@ -2755,256 +1863,6 @@ def test_credit_flag_admits_uncertain_root_envelope_and_keeps_absent_rejected()
assert wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in absent.shape_violations
-def test_ceiling_credits_uncertain_but_bound_submission() -> None:
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
-
- # Below the ceiling the server keeps steering: no credit, no imposition.
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
-
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract(
- ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
- )
- assert resolution is not None
- assert resolution.mode == "credited"
- assert resolution.workflow_yaml == workflow_yaml
- assert resolution.code_artifact_metadata is metadata
- assert resolution.evaluation.has_deficiencies is False
- assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is not True
- assert any(log.get("event") == "copilot_output_contract_submission_credited_at_ceiling" for log in logs)
-
-
-def test_credit_preserves_reject_streak_so_downstream_reject_keeps_ceiling() -> None:
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
-
- credited = wu._credit_output_contract_submission_at_ceiling(
- ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
- )
- assert credited is not None
-
- downstream = wu._record_output_contract_family_reject(
- ctx,
- strict.required_paths,
- reject_family="requested_output_contract_missing_output_coverage",
- authored_structural_fingerprint=wu._output_contract_structural_fingerprint(
- workflow_yaml, strict.canonical_signature
- ),
- )
- assert downstream == wu._MAX_OUTPUT_CONTRACT_REJECTS
-
-
-def test_ceiling_imposes_binding_from_scouted_observation() -> None:
- ctx = _record_criterion_ctx()
- workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')
- metadata = _record_contract_metadata()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None
- assert "output.record_id" in strict.missing_return_paths
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
-
- # Enforcing-mode negative: without a recorded source, the reject path stays untouched.
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
-
- ctx.scouted_output_covered_paths = {"output.record_id"}
- with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract(
- ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
- )
- assert resolution is not None
- assert resolution.mode == "imposed"
- imposed_code = str(
- wu._workflow_yaml_code_blocks_by_label(resolution.workflow_yaml)["extract_record"].get("code") or ""
- )
- assert 'return {"output": {"record_id": record_id}}' in imposed_code
- assert resolution.evaluation.has_deficiencies is False
- assert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)
-
-
-def test_ceiling_abstains_when_no_binding_composable_even_with_observation() -> None:
- ctx = _record_criterion_ctx()
- workflow_yaml = _declaration_waiver_yaml('await page.click("#next")\nreturn "done"')
- metadata = _record_contract_metadata()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- ctx.scouted_output_covered_paths = {"output.record_id"}
-
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
- assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None
-
-
-def test_pass_path_resolves_the_same_way_under_log_only_as_when_enforcing() -> None:
- resolutions = []
- for log_only in (False, True):
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- if log_only:
- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- resolution = wu._credit_or_impose_output_contract(
- ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
- )
- assert resolution is not None
- resolutions.append(resolution.mode)
- assert resolutions == ["credited", "credited"]
-
-
-def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejects() -> None:
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- assert ctx.output_contract_reject_count_by_signature == {}
-
- with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract(
- ctx,
- strict,
- workflow_yaml=workflow_yaml,
- raw_metadata=metadata,
- )
- assert resolution is not None
- assert resolution.mode == "credited"
- assert resolution.evaluation.has_deficiencies is False
- assert any(log.get("event") == "copilot_output_contract_submission_credited_at_ceiling" for log in logs)
-
-
-def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> None:
- ctx = _record_criterion_ctx()
- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
- ctx.scouted_output_covered_paths = {"output.record_id"}
- workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')
- metadata = _record_contract_metadata()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- assert ctx.output_contract_reject_count_by_signature == {}
-
- with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract(
- ctx,
- strict,
- workflow_yaml=workflow_yaml,
- raw_metadata=metadata,
- )
- assert resolution is not None
- assert resolution.mode == "imposed"
- imposed_code = str(
- wu._workflow_yaml_code_blocks_by_label(resolution.workflow_yaml)["extract_record"].get("code") or ""
- )
- assert 'return {"output": {"record_id": record_id}}' in imposed_code
- assert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)
-
-
-def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_as_today() -> None:
- ctx = _record_criterion_ctx()
- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
- ctx.scouted_output_covered_paths = set()
- omitted_yaml = _declaration_waiver_yaml('return {"status": "done"}')
- metadata = _record_contract_metadata()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)
-
- assert (
- wu._credit_or_impose_output_contract(
- ctx,
- strict,
- workflow_yaml=omitted_yaml,
- raw_metadata=metadata,
- )
- is None
- )
- assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None
-
-
-def test_ceiling_refuses_omitted_value_without_observation_byte_identically() -> None:
- ctx, _, metadata = _declaration_only_root_contract()
- ctx.scouted_output_covered_paths = set()
- omitted_yaml = _declaration_waiver_yaml('return {"status": "done"}')
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)
-
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=omitted_yaml, raw_metadata=metadata) is None
- credited = wu._evaluate_output_contract_for_code_block(
- ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True, credit_convergence_ceiling=True
- )
- assert credited is not None
- assert credited.payload == strict.payload
- assert wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in credited.shape_violations
- assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None
-
-
-def test_steering_exhausted_requires_an_armed_directive_the_draft_did_not_move() -> None:
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None
- assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False
-
- ctx.output_contract_actuation_count_by_signature[strict.canonical_signature] = 1
- ctx.output_contract_last_reject_fingerprint_by_signature[strict.canonical_signature] = "fp_from_another_draft"
- assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False
-
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is True
-
-
-def test_loaded_result_evidence_claimed_by_another_contract_does_not_authorize_imposition() -> None:
- ctx = _record_criterion_ctx()
- ctx.scouted_output_covered_paths = set()
- workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')
- metadata = _record_contract_metadata()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None and strict.has_deficiencies
- _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
- ctx.latest_evaluate_result_composition_steer = loaded_result_composition_evidence_from_page(
- {"result_containers": [{"selector": "#record", "row_count": 1, "sample_rows": ["Example record"]}]},
- source_tool="evaluate",
- source_url="https://example.com/results",
- )
- ctx.latest_evaluate_result_composition_signature = "sig_for_a_different_contract"
-
- assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is False
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
- assert ctx.latest_evaluate_result_composition_signature == "sig_for_a_different_contract"
-
- ctx.latest_evaluate_result_composition_signature = None
- assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is True
- assert ctx.latest_evaluate_result_composition_signature == strict.canonical_signature
-
-
def _steering_exhausted_evidence(**overrides: object) -> OutputContractActuationEvidence:
base = dict(
imposed_available=False,
@@ -3027,72 +1885,6 @@ def _arm_steering_exhausted(ctx: object, signature: str, workflow_yaml: str) ->
)
-def test_resolver_credits_before_imposing_once_steering_is_exhausted() -> None:
- credit_and_impose = _steering_exhausted_evidence(credit_available=True, scouted_binding_impose_available=True)
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=credit_and_impose).kind
- == OutputContractActuationKind.CREDIT
- )
-
- impose_only = _steering_exhausted_evidence(scouted_binding_impose_available=True)
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=impose_only).kind
- == OutputContractActuationKind.IMPOSE_SCOUTED_BINDING
- )
-
-
-def test_resolver_pass_path_preempts_the_unobservable_source_terminal() -> None:
- would_terminal = _steering_exhausted_evidence(
- click_only_spine=True,
- declick_attempt_failed=True,
- credit_available=True,
- )
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=would_terminal).kind
- == OutputContractActuationKind.CREDIT
- )
-
- without_pass_path = _steering_exhausted_evidence(click_only_spine=True, declick_attempt_failed=True)
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=without_pass_path).kind
- == OutputContractActuationKind.BLOCKED_TERMINAL
- )
-
-
-def test_resolver_ignores_credit_and_impose_until_steering_is_exhausted() -> None:
- for kind_flags in ({"credit_available": True}, {"scouted_binding_impose_available": True}):
- armed = _steering_exhausted_evidence(steering_exhausted=False, **kind_flags)
- unarmed = _steering_exhausted_evidence(steering_exhausted=False)
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=armed).kind
- == resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=unarmed).kind
- )
-
-
-def test_resolver_keeps_existing_imposition_ahead_of_the_pass_path() -> None:
- both = _steering_exhausted_evidence(imposed_available=True, credit_available=True)
- assert (
- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=both).kind
- == OutputContractActuationKind.IMPOSED
- )
-
-
-def test_observing_one_path_does_not_authorize_imposing_a_multi_path_contract() -> None:
- ctx = _record_criterion_ctx()
- ctx.scouted_output_covered_paths = {"output.record_id"}
- evaluation = SimpleNamespace(
- observation_paths={"output.record_id", "output.record_total"},
- required_paths={"output.record_id", "output.record_total"},
- block_label="extract_record",
- canonical_signature="sig_multi",
- )
-
- assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is False
-
- ctx.scouted_output_covered_paths = {"output.record_id", "output.record_total"}
- assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is True
-
-
# ---------------------------------------------------------------------------
# The code-authoring guardrail give-up consults the ceiling actuation.
# ---------------------------------------------------------------------------
@@ -3105,116 +1897,3 @@ def _guardrail_impose_ctx() -> tuple[CopilotContext, str, list[dict[str, object]
ctx.scouted_output_covered_paths = {"output.record_id"}
ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
return ctx, workflow_yaml, metadata
-
-
-def test_guardrail_ceiling_consult_imposes_before_stop_under_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
-
- with capture_logs() as logs:
- resolution = wu._guardrail_ceiling_actuation_resolution(
- ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=False
- )
-
- assert resolution is not None
- assert resolution.mode == "imposed"
- events = [log.get("event") for log in logs]
- assert "copilot_guardrail_ceiling_actuation_consulted" in events
- assert events.index("copilot_guardrail_ceiling_actuation_consulted") < events.index(
- "copilot_output_contract_binding_imposed_at_ceiling"
- )
- assert ctx.blocker_signal is None
- assert ctx.code_authoring_guardrail_reject_count == enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
- assert ctx.output_contract_reject_count_by_signature.get(resolution.evaluation.canonical_signature) in (None, 0)
-
-
-def test_ceiling_bookkeeping_imposed_arms_evidence_and_marks_imposed() -> None:
- seed_ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
- resolution = wu._guardrail_ceiling_actuation_resolution(
- seed_ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=True
- )
- assert resolution is not None and resolution.mode == "imposed"
- signature = resolution.evaluation.canonical_signature
- observation_paths = sorted(resolution.evaluation.observation_paths)
- assert observation_paths
-
- ctx = _record_criterion_ctx()
- returned = wu._apply_output_contract_ceiling_bookkeeping(ctx, resolution)
-
- assert returned == signature
- assert ctx.output_contract_pending_run_evidence.get(signature) == observation_paths
- assert ctx.output_contract_imposed_since_last_reject_by_signature.get(signature) is True
-
-
-def test_ceiling_bookkeeping_credited_arms_evidence_without_marking_imposed() -> None:
- seed_ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
- imposed = wu._guardrail_ceiling_actuation_resolution(
- seed_ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=True
- )
- assert imposed is not None
- credited = dataclasses.replace(imposed, mode="credited")
- signature = credited.evaluation.canonical_signature
- observation_paths = sorted(credited.evaluation.observation_paths)
- assert observation_paths
-
- ctx = _record_criterion_ctx()
- returned = wu._apply_output_contract_ceiling_bookkeeping(ctx, credited)
-
- assert returned == ""
- assert ctx.output_contract_pending_run_evidence.get(signature) == observation_paths
- assert ctx.output_contract_imposed_since_last_reject_by_signature == {}
-
-
-def test_guardrail_ceiling_consult_abstains_without_recorded_observation() -> None:
- ctx = _record_criterion_ctx()
- workflow_yaml = _declaration_waiver_yaml('await page.click("#next")\nreturn "done"')
- metadata = _record_contract_metadata()
- ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
-
- with capture_logs() as logs:
- resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
-
- assert resolution is None
- assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
- assert ctx.blocker_signal is None
- assert ctx.code_authoring_guardrail_reject_count == enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
- assert ctx.output_contract_imposed_since_last_reject_by_signature == {}
-
-
-def test_guardrail_ceiling_consult_skipped_below_churn_ceiling() -> None:
- ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
- ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2
-
- with capture_logs() as logs:
- resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
-
- assert resolution is None
- assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
-
-
-def test_guardrail_ceiling_consult_yields_to_stronger_live_owner() -> None:
- ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
- ctx.output_contract_actuation_by_signature = {"sig": OutputContractAdvisoryState.GRANTED}
-
- with capture_logs() as logs:
- resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
-
- assert resolution is None
- assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
- assert ctx.blocker_signal is None
-
-
-def test_guardrail_exhaustion_evidence_reaches_the_pass_path() -> None:
- ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None
-
- assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
-
- resolution = wu._credit_or_impose_output_contract(
- ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata, steering_exhausted=True
- )
- assert resolution is not None
- assert resolution.mode == "imposed"
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 12b29ffd9..8567bff15 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -12,8 +12,14 @@ from agents import GuardrailFunctionOutput, OutputGuardrail, ToolInputGuardrailD
from agents.run_context import RunContextWrapper
from agents.tool_context import ToolContext
-from skyvern.config import settings
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.author_time_block import (
+ AUTHOR_TIME_HARD_BLOCKS,
+ BANNED_BLOCKS_BLOCK_ID,
+ CODE_SAFETY_BLOCK_ID,
+ CREDENTIAL_SCOUT_BLOCK_ID,
+ AuthorTimeBlock,
+)
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -34,6 +40,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyVerdict,
_contains_internal_tool_vocab_leak,
_contains_yaml_authoring_vocab_leak,
+ demote_author_time_steer_reasons,
derive_output_kind,
evaluate_actuation_obligation,
evaluate_output_policy,
@@ -257,19 +264,6 @@ def test_rejects_raw_secret_echo_in_user_response() -> None:
assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
-def test_output_policy_still_blocks_with_author_time_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
-
- verdict = evaluate_output_policy(
- request_policy=_policy(),
- user_response="I used password: hunter2 to test the login.",
- )
-
- assert not verdict.allowed
- assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
-
-
@pytest.mark.parametrize(
("response_type", "user_response", "expected_type", "expected_response"),
[
@@ -647,22 +641,6 @@ def test_rejects_reply_when_request_policy_required_clarification() -> None:
assert OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS in verdict.reason_codes
-def test_request_policy_clarification_still_blocks_with_author_time_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
-
- verdict = evaluate_output_policy(
- request_policy=_policy(user_response_policy="ask_clarification", allow_update_workflow=False),
- response_type="REPLY",
- user_response="I created the workflow with the credential.",
- )
-
- assert not verdict.allowed
- assert OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS in verdict.reason_codes
-
-
def test_avoidable_output_field_confirmation_is_hard_blocked() -> None:
policy = _policy(
user_response_policy="proceed",
@@ -1270,19 +1248,6 @@ def test_rejects_approved_credential_on_different_login_origin() -> None:
assert OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED in verdict.reason_codes
-def test_credential_scope_still_blocks_with_author_time_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
-
- verdict = evaluate_output_policy(
- request_policy=_policy(),
- workflow_yaml=_workflow_yaml(url="https://evil.example.test/login"),
- )
-
- assert not verdict.allowed
- assert OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED in verdict.reason_codes
-
-
def test_rejects_approved_credential_on_templated_login_origin() -> None:
verdict = evaluate_output_policy(
request_policy=_policy(),
@@ -2142,106 +2107,6 @@ workflow_definition:
assert ctx.consecutive_tool_tracker == ["update_workflow", "update_workflow"]
-async def test_update_and_run_blocks_rejects_unobserved_page_before_workflow_update(monkeypatch) -> None:
- from skyvern.forge.sdk.copilot import tools as tools_module
-
- workflow_yaml = """
-workflow_definition:
- parameters: []
- blocks:
- - block_type: goto_url
- label: open_lookup
- url: https://example.com/lookup
- - block_type: navigation
- label: search_lookup
- navigation_goal: Enter the observed field and submit.
-"""
- ctx = _ctx(
- build_phase=BuildPhase.COMPOSING,
- turn_intent=TurnIntent(mode=TurnIntentMode.BUILD),
- )
-
- async def unexpected_prior_definition(*args: object, **kwargs: object) -> object:
- raise AssertionError("composition evidence precheck must run before reading prior workflow")
-
- async def unexpected_update_workflow(*args: object, **kwargs: object) -> object:
- raise AssertionError("composition evidence precheck must run before updating workflow")
-
- sanitized_tool_names: list[str] = []
-
- def fake_sanitize_tool_result_for_llm(tool_name: str, result: dict[str, object]) -> dict[str, object]:
- sanitized_tool_names.append(tool_name)
- return result
-
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", unexpected_prior_definition)
- monkeypatch.setattr(tools_module, "_update_workflow", unexpected_update_workflow)
- monkeypatch.setattr(tools_module, "sanitize_tool_result_for_llm", fake_sanitize_tool_result_for_llm)
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps(
- {
- "workflow_yaml": workflow_yaml,
- "block_labels": ["search_lookup"],
- "parameters": {},
- }
- ),
- )
-
- payload = json.loads(result)
- assert payload["ok"] is False
- assert "inspect_page_for_composition" in payload["error"]
- assert "https://example.com/lookup" in payload["error"]
- assert ctx.block_observation_refs == {}
- assert sanitized_tool_names == ["update_and_run_blocks"]
-
-
-async def test_update_workflow_runs_composition_evidence_precheck_before_saving(monkeypatch) -> None:
- from skyvern.forge.sdk.copilot import tools as tools_module
-
- workflow_yaml = """
-workflow_definition:
- parameters: []
- blocks:
- - block_type: goto_url
- label: open_lookup
- url: https://example.com/lookup
- - block_type: navigation
- label: search_lookup
- navigation_goal: Enter the observed field and submit.
- - block_type: action
- label: expand_first_result
- navigation_goal: Expand the first result row.
-"""
- ctx = _ctx(
- build_phase=BuildPhase.COMPOSING,
- turn_intent=TurnIntent(mode=TurnIntentMode.BUILD),
- request_policy=RequestPolicy(),
- )
-
- async def unexpected_update_workflow(*args: object, **kwargs: object) -> object:
- raise AssertionError("composition evidence precheck must run before update_workflow saves")
-
- monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
- monkeypatch.setattr(tools_module, "_update_workflow", unexpected_update_workflow)
- monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
-
- result = await tools_module.update_workflow_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_workflow"),
- json.dumps({"workflow_yaml": workflow_yaml}),
- )
-
- payload = json.loads(result)
- assert payload["ok"] is False
- assert "page-dependent build blocks need observed page evidence" in payload["error"]
- assert "https://example.com/lookup" in payload["error"]
-
-
@pytest.mark.asyncio
async def test_update_and_run_blocks_precheck_uses_proposed_block_observation_refs(monkeypatch) -> None:
from skyvern.forge.sdk.copilot import tools as tools_module
@@ -3308,3 +3173,135 @@ def test_avoidable_ask_with_co_firing_secret_leak_is_not_deferred() -> None:
assert OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION in verdict.reason_codes
assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
assert diagnostics.get("deferred_to_recycle", False) is False
+
+
+_AUTHOR_TIME_KEEP_HARD_REASONS = frozenset(
+ {
+ OutputPolicyReason.RAW_SECRET_LEAK,
+ OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
+ OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
+ }
+)
+
+# Reads as "does not refuse", not "reaches data['findings']". Only schema_incompatibility,
+# code_artifact_metadata_incomplete and synthesized_code_block_not_imposed populate that bucket;
+# demoted OutputPolicyReason members land in the guardrail's steered_reason_codes log line.
+_FINDING = "finding"
+
+# Enumerated by hand on purpose: deriving it from the enum would auto-classify a new refusal
+# source and defeat the ratchet. See architecture/output-policy-disposition.md.
+_AUTHORING_SEAM_REFUSAL_SOURCES: dict[str, str] = {
+ "code_safety": CODE_SAFETY_BLOCK_ID,
+ "credential_scout": CREDENTIAL_SCOUT_BLOCK_ID,
+ "banned_blocks": BANNED_BLOCKS_BLOCK_ID,
+ "raw_secret_leak": CREDENTIAL_SCOUT_BLOCK_ID,
+ "request_policy_clarification_bypass": _FINDING,
+ "unapproved_credential_reference": CREDENTIAL_SCOUT_BLOCK_ID,
+ "credential_scope_broadened": CREDENTIAL_SCOUT_BLOCK_ID,
+ "unbacked_workflow_delivery_claim": _FINDING,
+ "missing_unvalidated_proposal_affordance": _FINDING,
+ "missing_proposal_state": _FINDING,
+ "persistence_state_mismatch": _FINDING,
+ "internal_tool_instruction_leak": _FINDING,
+ "output_policy_context_missing": _FINDING,
+ "internal_block_taxonomy_leak": _FINDING,
+ "internal_classifier_vocab_leak": _FINDING,
+ "self_prescriptive_phrase_leak": _FINDING,
+ "workflow_yaml_in_reply": _FINDING,
+ "avoidable_output_field_confirmation": _FINDING,
+ "actuation_obligation_steer": _FINDING,
+ "actuation_obligation_unmet": _FINDING,
+}
+
+
+def test_every_authoring_seam_refusal_source_is_classified() -> None:
+ """An unclassified refusal source fails here rather than becoming a wall in production."""
+ sources = {reason.value for reason in OutputPolicyReason} | set(AUTHOR_TIME_HARD_BLOCKS)
+
+ assert sources == set(_AUTHORING_SEAM_REFUSAL_SOURCES)
+ for source, disposition in _AUTHORING_SEAM_REFUSAL_SOURCES.items():
+ assert disposition in AUTHOR_TIME_HARD_BLOCKS or disposition == _FINDING, source
+
+
[email protected]("source", sorted(_AUTHORING_SEAM_REFUSAL_SOURCES))
+def test_only_a_hard_block_identity_can_refuse(source: str) -> None:
+ disposition = _AUTHORING_SEAM_REFUSAL_SOURCES[source]
+ if disposition == _FINDING:
+ with pytest.raises(ValueError):
+ AuthorTimeBlock(block_id=source, error="steer")
+ return
+ assert AuthorTimeBlock(block_id=disposition, error="refuse").block_id in AUTHOR_TIME_HARD_BLOCKS
+
+
[email protected]("reason", list(OutputPolicyReason))
+def test_tool_input_disposition_matches_classification_table(reason: OutputPolicyReason) -> None:
+ """Pins every reason code to its reviewed verdict in
+ cloud_docs/workflow-copilot/architecture/output-policy-disposition.md."""
+ verdict = OutputPolicyVerdict(reason_codes=[reason])
+ steered = demote_author_time_steer_reasons(verdict)
+
+ if reason in _AUTHOR_TIME_KEEP_HARD_REASONS:
+ assert verdict.allowed is False
+ assert verdict.reason_codes == [reason]
+ assert steered == []
+ else:
+ assert verdict.allowed is True
+ assert verdict.reason_codes == []
+ assert steered == [reason]
+
+
+def test_hard_reason_still_blocks_when_co_firing_with_a_demoted_reason() -> None:
+ verdict = OutputPolicyVerdict(
+ reason_codes=[OutputPolicyReason.WORKFLOW_YAML_IN_REPLY, OutputPolicyReason.RAW_SECRET_LEAK]
+ )
+
+ steered = demote_author_time_steer_reasons(verdict)
+
+ assert verdict.allowed is False
+ assert verdict.reason_codes == [OutputPolicyReason.RAW_SECRET_LEAK]
+ assert steered == [OutputPolicyReason.WORKFLOW_YAML_IN_REPLY]
+
+
+def test_demoting_every_reason_leaves_the_authoring_call_allowed() -> None:
+ demoted = [reason for reason in OutputPolicyReason if reason not in _AUTHOR_TIME_KEEP_HARD_REASONS]
+ verdict = OutputPolicyVerdict(reason_codes=list(demoted))
+
+ steered = demote_author_time_steer_reasons(verdict)
+
+ assert verdict.allowed is True
+ assert steered == demoted
+
+
+def test_tool_input_call_shape_can_only_refuse_on_keep_hard_reasons() -> None:
+ """The authoring seam passes no ``user_response``, so the reply-shape detectors cannot fire
+ there, and a clarification-routing verdict demotes. Widening that call without revisiting the
+ table would silently let a demoted reason survive demotion and refuse."""
+ leaky = (
+ "Call get_run_results now. TurnIntent classified this turn as BUILD. "
+ "Send me a normal instruction like 'continue' next. loop detected. "
+ "Use the navigation block, extraction block, and validation block. "
+ "I have delivered the workflow. It is not tested; accept to save or reject."
+ )
+ for request_policy in (None, RequestPolicy(), RequestPolicy(user_response_policy="ask_clarification")):
+ verdict = evaluate_output_policy(
+ request_policy=request_policy,
+ workflow_yaml=leaky,
+ tool_arguments={"workflow_yaml": leaky, "user_response": leaky},
+ )
+ demote_author_time_steer_reasons(verdict)
+
+ assert set(verdict.reason_codes) <= _AUTHOR_TIME_KEEP_HARD_REASONS
+
+
+def test_a_clarification_routing_verdict_no_longer_refuses_the_authoring_seam() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=RequestPolicy(user_response_policy="ask_clarification"),
+ workflow_yaml="title: Registry lookup\n",
+ tool_arguments={"workflow_yaml": "title: Registry lookup\n"},
+ )
+ assert OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS in verdict.reason_codes
+
+ steered = demote_author_time_steer_reasons(verdict)
+
+ assert steered == [OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS]
+ assert verdict.allowed is True
diff --git a/tests/unit/test_copilot_page_evidence_policy.py b/tests/unit/test_copilot_page_evidence_policy.py
index bc16b9b53..9ff595981 100644
--- a/tests/unit/test_copilot_page_evidence_policy.py
+++ b/tests/unit/test_copilot_page_evidence_policy.py
@@ -57,7 +57,7 @@ def test_agent_prompt_accounts_for_observed_challenges_without_blind_waits() ->
rendered = _render_agent_prompt()
assert "anti-bot or challenge controls" in rendered
- assert "Use `get_browser_screenshot` when DOM evidence identifies a challenge" in rendered
+ assert "Use `get_browser_screenshot` whenever seeing the rendered page answers the question" in rendered
assert "Prefer explicit challenge handling available to Skyvern over blind waits" in rendered
assert "report the observed blocker instead of retrying the same flow" in rendered
assert "challenge_state.gates_submit_controls=true" in rendered
diff --git a/tests/unit/test_copilot_probable_site_block.py b/tests/unit/test_copilot_probable_site_block.py
index d6d0e7c51..58a38f146 100644
--- a/tests/unit/test_copilot_probable_site_block.py
+++ b/tests/unit/test_copilot_probable_site_block.py
@@ -16,12 +16,8 @@ from skyvern.forge.sdk.copilot.enforcement import (
_repeated_frontier_failure_nudge,
)
from skyvern.forge.sdk.copilot.tools import (
- _challenge_http_request_reject_message,
_detect_probable_site_block_wall,
- _detect_timing_only_challenge_wait_blocks,
_record_run_blocks_result,
- _timing_only_challenge_wait_reject_message,
- _update_workflow,
)
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind
from tests.unit.conftest import make_copilot_context as _fresh_context
@@ -31,48 +27,6 @@ _SCRAPE_WALL_REASON = (
"unexpectedly or become unresponsive during analysis."
)
-_CHALLENGE_WAIT_WORKFLOW = """
-workflow_definition:
- blocks:
- - label: open_page
- block_type: goto_url
- url: https://example.com
- next_block_label: wait_challenge
- - label: wait_challenge
- title: Wait for challenge
- block_type: wait
- wait_sec: 10
-"""
-
-_GENERIC_WAIT_WORKFLOW = """
-workflow_definition:
- blocks:
- - label: wait_for_download
- title: Wait for download
- block_type: wait
- wait_sec: 10
-"""
-
-_CONDITIONAL_ACTION_WORKFLOW = """
-workflow_definition:
- blocks:
- - label: check_for_challenge
- block_type: conditional
- branch_conditions:
- - condition_type: prompt
- condition: If a challenge is visible on the page
- next_block_label: handle_visible_challenge
- next_block_label: extract_data
- - label: handle_visible_challenge
- title: Handle visible challenge
- block_type: navigation
- navigation_goal: Click the visible verification control if present.
- next_block_label: extract_data
- - label: extract_data
- block_type: extraction
- data_extraction_goal: Extract the requested data.
-"""
-
def _scrape_wall_result() -> dict:
return {
@@ -362,103 +316,3 @@ def test_stop_nudge_keeps_configure_proxy_advice_when_proxy_is_none() -> None:
assert exc_info.value.halt.kind == TurnHaltKind.PROBABLE_SITE_BLOCK
assert "configure a proxy" in exc_info.value.halt.blocker_signal.user_facing_reason.lower()
-
-
-def test_detects_challenge_named_wait_block() -> None:
- assert _detect_timing_only_challenge_wait_blocks(_CHALLENGE_WAIT_WORKFLOW) == ["wait_challenge"]
-
-
-def test_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:
- ctx = _fresh_context()
- ctx.last_test_anti_bot = "Cloudflare challenge page detected"
-
- message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)
-
- assert message is not None
- assert "wait_challenge" in message
- assert "timing-only challenge wait" in message
-
-
-async def test_update_workflow_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:
- ctx = _fresh_context()
- ctx.last_test_anti_bot = "Cloudflare challenge page detected"
-
- result = await _update_workflow({"workflow_yaml": _CHALLENGE_WAIT_WORKFLOW}, ctx)
-
- assert result["ok"] is False
- assert "wait_challenge" in str(result["error"])
-
-
-def test_rejects_challenge_wait_after_repeated_scrape_wall() -> None:
- ctx = _fresh_context()
- _record_run_blocks_result(ctx, _scrape_wall_result())
- _record_run_blocks_result(ctx, _scrape_wall_result())
-
- message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)
-
- assert message is not None
- assert "wait_challenge" in message
-
-
-def test_allows_generic_wait_after_block_evidence() -> None:
- ctx = _fresh_context()
- ctx.last_test_anti_bot = "challenge page detected"
-
- assert _timing_only_challenge_wait_reject_message(ctx, _GENERIC_WAIT_WORKFLOW) is None
-
-
-def test_allows_conditional_challenge_action_after_block_evidence() -> None:
- ctx = _fresh_context()
- ctx.last_test_anti_bot = "challenge page detected"
-
- assert _timing_only_challenge_wait_reject_message(ctx, _CONDITIONAL_ACTION_WORKFLOW) is None
-
-
-def test_rejects_new_http_request_after_observed_challenge_evidence() -> None:
- existing_yaml = """
-workflow_definition:
- blocks:
- - label: open_lookup
- block_type: goto_url
- url: https://example.com/registry/search
-"""
- submitted_yaml = """
-workflow_definition:
- blocks:
- - label: open_lookup
- block_type: goto_url
- url: https://example.com/registry/search
- - label: submit_lookup
- block_type: http_request
- method: POST
- url: https://example.com/registry/search?s=1
-"""
- ctx = _fresh_context()
- ctx.composition_page_evidence = {
- "anti_bot_indicators": ["human-verification"],
- "challenge_controls": [{"selector": "#human-verification"}],
- }
- ctx.workflow_yaml = existing_yaml
-
- message = _challenge_http_request_reject_message(ctx, submitted_yaml, ctx.workflow_yaml)
-
- assert message is not None
- assert "submit_lookup" in message
- assert "raw http_request blocks are not allowed" in message
-
-
-def test_allows_existing_http_request_when_challenge_evidence_is_added_later() -> None:
- existing_yaml = """
-workflow_definition:
- blocks:
- - label: submit_lookup
- block_type: http_request
- method: POST
- url: https://example.com/search
-"""
- ctx = _fresh_context()
- ctx.composition_page_evidence = {"anti_bot_indicators": ["human-verification"]}
- ctx.workflow_yaml = existing_yaml
-
- assert _challenge_http_request_reject_message(ctx, existing_yaml, ctx.workflow_yaml) is None
diff --git a/tests/unit/test_copilot_recorded_outcome_grounding.py b/tests/unit/test_copilot_recorded_outcome_grounding.py
deleted file mode 100644
index d8ae244df..000000000
--- a/tests/unit/test_copilot_recorded_outcome_grounding.py
+++ /dev/null
@@ -1,1132 +0,0 @@
-from __future__ import annotations
-
-import json
-from dataclasses import replace
-from types import SimpleNamespace
-
-import pytest
-from structlog.testing import capture_logs
-
-from skyvern.config import settings
-from skyvern.forge.sdk.copilot import tools as tools_module
-from skyvern.forge.sdk.copilot.agent import _recorded_build_test_outcome_prompt
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
-from skyvern.forge.sdk.copilot.build_test_outcome import (
- RecordedBuildTestOutcome,
- RecordedOutcomeBindingConstraint,
- arm_recorded_outcome_grounding_requirement,
- authored_block_signatures_from_workflow,
- maybe_satisfy_recorded_outcome_grounding_requirement,
-)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
- DiagnosisInput,
- DiagnosisRepairContract,
- DiagnosisResult,
- RepairDecision,
- RepairNextAction,
- VerificationResult,
-)
-from skyvern.forge.sdk.copilot.enforcement import MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
-from skyvern.forge.sdk.copilot.result_evidence import (
- ScoutObservationContract,
- mint_scout_observation_contract,
-)
-from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
-from skyvern.forge.sdk.copilot.tools.blockers import _tool_loop_error
-from skyvern.forge.sdk.copilot.tools.run_execution import _update_repair_loop_state
-from skyvern.forge.sdk.copilot.tools.workflow_update import (
- _commit_recorded_outcome_early_terminal,
- _record_author_time_reject_outcome,
- _record_code_authoring_guardrail_reject,
- _recorded_outcome_convergence_reject,
-)
-from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
-from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
-
-_OWNING_BLOCK_WORKFLOW = """
-title: Registry lookup
-workflow_definition:
- blocks:
- - block_type: code
- label: search_records
- code: |
- return {"records": [{"npi": "123"}]}
-"""
-
-_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT = """
-title: Registry lookup
-workflow_definition:
- blocks:
- - block_type: code
- label: search_records
- description: Human-facing note adjusted; steps identical
- code: |
- return {"records": [{"npi": "123"}]}
-"""
-
-_OWNING_BLOCK_WORKFLOW_RENAMED = """
-title: Registry lookup
-workflow_definition:
- blocks:
- - block_type: code
- label: renamed_records
- code: |
- return {"records": [{"npi": "123"}]}
-"""
-
-_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE = """
-title: Registry lookup
-workflow_definition:
- parameters:
- - key: added_param
- parameter_type: workflow
- blocks:
- - block_type: code
- label: search_records
- code: |
- return {"records": [{"npi": "123"}]}
-"""
-
-
[email protected](autouse=True)
-def _disable_author_time_gate_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", False)
-
-
-def _outcome(**updates: object) -> RecordedBuildTestOutcome:
- base = {
- "phase": "persisted_block_run",
- "attempted_tool": "update_and_run_blocks",
- "verdict": "repairable_failure",
- "reason_code": "runtime_block_failure",
- "workflow_run_id": "wr_123",
- "block_labels": ["search_records"],
- "structural_failure_identity": "runtime:timeout_waiting_for_selector:failed",
- "page_evidence_refs": ["origin_present", "results:empty"],
- "observed_evidence_summary": "Timeout waiting for #results.",
- }
- base.update(updates)
- return RecordedBuildTestOutcome(**base) # type: ignore[arg-type]
-
-
-def _ctx(outcome: RecordedBuildTestOutcome | None = None) -> CopilotContext:
- history = []
- if outcome is not None:
- history = [
- {"structural_key": outcome.structural_key, "is_authoritative": True},
- {"structural_key": outcome.structural_key, "is_authoritative": True},
- ]
- return CopilotContext(
- organization_id="o",
- workflow_id="w",
- workflow_permanent_id="wp",
- workflow_yaml="",
- browser_session_id=None,
- stream=SimpleNamespace(), # type: ignore[arg-type]
- user_message="Fix the workflow",
- turn_intent=TurnIntent(
- mode=TurnIntentMode.EDIT,
- user_goal="Fix the workflow",
- authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
- ),
- latest_recorded_build_test_outcome=outcome,
- recorded_build_test_outcome_history=history,
- recorded_outcome_grounding_requirement=None,
- composition_page_evidence=None,
- block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
- completion_verification_result=None,
- verified_criteria_high_water=frozenset(),
- verified_prefix_labels=[],
- verified_prefix_high_water_len=0,
- last_full_workflow_test_ok=False,
- verified_full_pass_consumed=False,
- blocker_signal=None,
- turn_halt=None,
- observed_browser_urls=["https://example.com/results"],
- consecutive_tool_tracker=[],
- tool_activity=[],
- pending_reconciliation_run_id=None,
- pending_reconciliation_requires_user_input=False,
- post_budget_page_inspection_required=False,
- last_failure_category_top=None,
- repeated_action_fingerprint_streak_count=0,
- last_test_non_retriable_nav_error=None,
- )
-
-
-def _contract() -> DiagnosisRepairContract:
- return DiagnosisRepairContract(
- diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
- diagnosis_result=DiagnosisResult(),
- repair_decision=RepairDecision(next_action=RepairNextAction.REPAIR),
- verification_result=VerificationResult(run_status="failed"),
- )
-
-
-def _scout_contract(
- *,
- has_bounded_page_schema: bool = True,
- workflow_run_id: str | None = None,
- observed_after_workflow_run: bool = False,
-) -> ScoutObservationContract:
- page = {
- "current_url": "https://example.com/results",
- "inspection_warnings": [],
- "result_containers_truncated": False,
- "key_value_relations_truncated": False,
- "key_value_relations": [
- {
- "key_text": "NPI",
- "container_selector": ".kv",
- "container_match_count": 1,
- "container_position": 0,
- "value_child_index": 1,
- "direct_child_count": 2,
- "visible": True,
- "value_visible": True,
- }
- ],
- "result_containers": [],
- }
- contract = mint_scout_observation_contract(
- page,
- labels_by_path={"output.npi": ("NPI",)},
- url="https://example.com/results",
- has_bounded_page_schema=has_bounded_page_schema,
- workflow_run_id=workflow_run_id,
- observed_after_workflow_run=observed_after_workflow_run,
- )
- assert contract is not None
- return contract
-
-
-def _bounded_inspect_evidence(**updates: object) -> dict[str, object]:
- evidence: dict[str, object] = {
- "source_tool": "inspect_page_for_composition",
- "current_url": "https://example.com/results",
- "page_title": "Results",
- "forms": [
- {
- "fields": [{"label": "Search", "selector": "#q"}],
- "submit_controls": [{"text": "Search", "selector": "button[type=submit]", "disabled": False}],
- }
- ],
- "navigation_targets": [{"text": "Next", "selector": "a.next", "disabled": False}],
- "result_containers": [{"selector": "#results", "text_excerpt": "No results"}],
- "challenge_controls": [],
- "anti_bot_indicators": [],
- "observed_after_workflow_run": True,
- "workflow_run_id": "wr_123",
- }
- evidence.update(updates)
- return evidence
-
-
-def test_repeated_authoritative_outcome_arms_grounding_before_repair_ceiling() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
-
- _update_repair_loop_state(ctx, _contract())
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.structural_key == outcome.structural_key
- assert requirement.required_tool == "inspect_page_for_composition"
- assert requirement.required_target_url == "current_page"
- assert requirement.workflow_run_id == "wr_123"
- assert ctx.blocker_signal is None
-
-
-def test_log_only_recorded_outcome_grounding_records_without_stashing(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
-
- result = _tool_loop_error(ctx, "update_workflow")
-
- assert result is None
- assert ctx.blocker_signal is None
- assert ctx.latest_tool_blocker_signal is None
- assert ctx.turn_halt is None
- event = ctx.author_time_gate_ablation_events[-1]
- assert event.gate_id == "recorded_outcome_grounding"
- assert event.reason_code == "recorded_outcome_grounding_required"
- assert event.blocked_tool == "update_workflow"
- assert event.fingerprint == outcome.structural_key
- assert event.log_only is True
-
-
-def test_authoritative_persisted_outcome_arms_without_recorded_signature_prefix(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- ctx.recorded_build_test_outcome_history = [{"structural_key": outcome.structural_key, "is_authoritative": True}]
- monkeypatch.setattr(
- run_execution_module,
- "_repair_non_convergence_signature",
- lambda *_: "repair_no_verified_progress",
- )
-
- run_execution_module._update_repair_loop_state(ctx, _contract())
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.structural_key == outcome.structural_key
- assert requirement.workflow_run_id == "wr_123"
- assert requirement.satisfied is False
-
-
-def test_run_backed_author_time_reject_stashes_repair_ceiling_on_first_crossing() -> None:
- outcome = RecordedBuildTestOutcome(
- phase="author_time_reject",
- attempted_tool="update_workflow",
- verdict="authoring_rejected",
- reason_code="metadata_reject",
- block_labels=["search_records"],
- structural_failure_identity="metadata:missing-output",
- )
- ctx = _ctx(outcome)
- key = outcome.structural_key
- assert key is not None
- ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
- ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"
- ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
- contract = _contract()
-
- _update_repair_loop_state(ctx, contract)
-
- assert contract.repair_decision.next_action is RepairNextAction.STOP
- assert contract.repair_loop_state.ceiling_reached is True
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
-
-
-def test_zero_run_author_time_reject_does_not_stash_repair_ceiling_on_first_crossing() -> None:
- outcome = RecordedBuildTestOutcome(
- phase="author_time_reject",
- attempted_tool="update_workflow",
- verdict="authoring_rejected",
- reason_code="metadata_reject",
- block_labels=["search_records"],
- structural_failure_identity="metadata:missing-output",
- )
- ctx = _ctx(outcome)
- key = outcome.structural_key
- assert key is not None
- ctx.last_run_blocks_workflow_run_id = "wr_stale"
- ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"
- ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
- contract = _contract()
-
- _update_repair_loop_state(ctx, contract)
-
- assert contract.repair_loop_state.ceiling_reached is True
- assert contract.repair_decision.next_action is RepairNextAction.REPAIR
- assert ctx.turn_halt is None
- assert ctx.blocker_signal is None
-
-
-def test_progress_observed_outcome_does_not_arm_from_executed_run(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- outcome = _outcome(verdict="progress_observed", reason_code="verified_success")
- ctx = _ctx(outcome)
- ctx.recorded_build_test_outcome_history = [{"structural_key": outcome.structural_key, "is_authoritative": True}]
- monkeypatch.setattr(
- run_execution_module,
- "_repair_non_convergence_signature",
- lambda *_: "repair_no_verified_progress",
- )
-
- run_execution_module._update_repair_loop_state(ctx, _contract())
-
- assert ctx.recorded_outcome_grounding_requirement is None
-
-
-def test_authoritative_outcome_uses_last_executed_run_id_when_outcome_run_id_missing(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- outcome = _outcome(workflow_run_id=None)
- ctx = _ctx(outcome)
- ctx.last_run_blocks_workflow_run_id = "wr_fallback"
- monkeypatch.setattr(
- run_execution_module,
- "_repair_non_convergence_signature",
- lambda *_: "repair_no_verified_progress",
- )
-
- run_execution_module._update_repair_loop_state(ctx, _contract())
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.workflow_run_id == "wr_fallback"
-
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id=None, observed_after_workflow_run=False)
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id="wr_fallback")
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
-
-def test_changed_structural_key_resets_grounding_requirement_to_latest_run_outcome() -> None:
- first = _outcome()
- ctx = _ctx(first)
- arm_recorded_outcome_grounding_requirement(ctx)
- assert ctx.recorded_outcome_grounding_requirement is not None
-
- second = _outcome(structural_failure_identity="runtime:other")
- ctx.latest_recorded_build_test_outcome = second
- ctx.recorded_build_test_outcome_history.append({"structural_key": second.structural_key, "is_authoritative": True})
-
- _update_repair_loop_state(ctx, _contract())
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.structural_key == second.structural_key
-
-
-def test_grounding_blocks_mutation_until_matching_current_page_inspect_evidence() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
-
- error = _tool_loop_error(ctx, "update_and_run_blocks", {"workflow_yaml": "workflow_definition: {blocks: []}"})
-
- assert error is not None
- assert "inspect_page_for_composition" in error
- assert "current_page" in error
- assert ctx.recorded_outcome_grounding_requirement.satisfied is False
-
- ctx.composition_page_evidence = _bounded_inspect_evidence()
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- assert (
- _tool_loop_error(ctx, "update_and_run_blocks", {"workflow_yaml": "workflow_definition: {blocks: []}"}) is None
- )
-
-
- ("evidence_updates", "reject_reason"),
- [
- ({"source_tool": "evaluate"}, "not_inspect_source"),
- ({"workflow_run_id": "wr_old", "observed_after_workflow_run": True}, "run_id_mismatch"),
- ({"observed_after_workflow_run": False}, "run_id_mismatch"),
- ({"current_url": "", "inspected_url": ""}, "no_url"),
- ],
-)
-def test_grounding_rejection_logs_reason_and_run_id_fields(
- evidence_updates: dict[str, object],
- reject_reason: str,
-) -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(**evidence_updates)
-
- with capture_logs() as logs:
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- event = next(log for log in logs if log["event"] == "copilot recorded outcome grounding rejected")
- assert event["reject_reason"] == reject_reason
- assert event["structural_key"] == outcome.structural_key
- assert event["requirement_workflow_run_id"] == "wr_123"
- assert event["evidence_workflow_run_id"] == ctx.composition_page_evidence.get("workflow_run_id")
- assert event["evidence_observed_after_workflow_run"] == ctx.composition_page_evidence.get(
- "observed_after_workflow_run"
- )
- assert event["source_tool"] == ctx.composition_page_evidence.get("source_tool")
- assert event["current_url_present"] is (reject_reason != "no_url")
-
-
-def test_evaluate_grounding_payload_requires_inspect_recovery() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(
- source_tool="evaluate",
- workflow_run_id=None,
- observed_after_workflow_run=False,
- )
-
- with capture_logs() as logs:
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- event = next(log for log in logs if log["event"] == "copilot recorded outcome grounding rejected")
- assert event["reject_reason"] == "not_inspect_source"
- assert event["source_tool"] == "evaluate"
-
- assert _tool_loop_error(ctx, "evaluate", {}) is None
-
-
-def test_no_run_degraded_grounding_satisfies_with_typed_capture_degraded_payload() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(
- workflow_run_id=None,
- observed_after_workflow_run=False,
- forms=[],
- result_containers=[],
- navigation_targets=[],
- challenge_controls=[],
- )
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- payload = requirement.payload
- assert payload is not None
- assert payload.capture_degraded is True
- assert payload.payload_workflow_run_id is None
-
-
-def test_persisted_degraded_empty_grounding_satisfies_and_reaches_prompt() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(
- forms=[],
- result_containers=[],
- navigation_targets=[],
- challenge_controls=[],
- )
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- assert ctx.last_full_workflow_test_ok is False
- assert ctx.completion_verification_result is None
- assert ctx.verified_criteria_high_water == frozenset()
- assert ctx.verified_prefix_high_water_len == 0
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- payload = requirement.payload
- assert payload is not None
- assert payload.observed_empty_page is True
- assert payload.capture_degraded is True
- assert payload.challenge_gated is False
- assert payload.diagnostic_reason == "capture_degraded"
- assert payload.target_url == "current_page"
- assert payload.source_url == "https://example.com/results"
- assert payload.requirement_workflow_run_id == "wr_123"
- assert payload.payload_workflow_run_id == "wr_123"
-
- prompt = _recorded_build_test_outcome_prompt(ctx) # type: ignore[arg-type]
- assert "observed_empty_page: true" in prompt
- assert "capture_degraded: true" in prompt
- assert "diagnostic_reason: capture_degraded" in prompt
- assert "requirement_workflow_run_id: wr_123" in prompt
- assert "payload_workflow_run_id: wr_123" in prompt
-
-
-def test_unsatisfied_grounding_does_not_mask_repair_ceiling_final_blocker() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id="wr_old")
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- _tool_loop_error(ctx, "update_and_run_blocks", {"workflow_yaml": "workflow_definition: {blocks: []}"})
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
-
- repair_ceiling = CopilotToolBlockerSignal(
- blocker_kind="loop_detected",
- agent_steering_text="Stop retrying and report the blocker.",
- user_facing_reason="I couldn't get past the same problem after several attempts.",
- recovery_hint="report_blocker_to_user",
- renders_final_reply=True,
- internal_reason_code="repair_ceiling_reached",
- )
- stash_blocker_signal(ctx, repair_ceiling)
-
- assert ctx.blocker_signal is repair_ceiling
-
-
-def test_same_structural_key_new_workflow_run_rearms_grounding_requirement() -> None:
- first = _outcome(workflow_run_id="wr_123")
- ctx = _ctx(first)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id="wr_123")
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- second = _outcome(workflow_run_id="wr_456")
- ctx.latest_recorded_build_test_outcome = second
- ctx.recorded_build_test_outcome_history.append({"structural_key": second.structural_key, "is_authoritative": True})
-
- requirement = arm_recorded_outcome_grounding_requirement(ctx)
-
- assert requirement is not None
- assert requirement.workflow_run_id == "wr_456"
- assert requirement.satisfied is False
- assert requirement.payload is None
-
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id="wr_123")
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- ctx.composition_page_evidence = _bounded_inspect_evidence(workflow_run_id="wr_456")
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- prompt = _recorded_build_test_outcome_prompt(ctx) # type: ignore[arg-type]
- assert "grounding_workflow_run_id: wr_456" in prompt
- assert "grounding_workflow_run_id: wr_123" not in prompt
-
-
-def test_terminal_blocker_takes_precedence_over_grounding_requirement() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.workflow_verification_evidence = SimpleNamespace(
- active_run_terminal_evidence_detected=True,
- page_title="Done",
- active_run_terminal_evidence_workflow_run_id="wr_terminal",
- )
-
- error = _tool_loop_error(ctx, "update_and_run_blocks", {"workflow_yaml": "workflow_definition: {blocks: []}"})
-
- assert error is not None
- assert ctx.blocker_signal.internal_reason_code == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
- assert "recorded_outcome_grounding_required" not in error
-
-
-def test_pending_reconciliation_takes_precedence_over_grounding_requirement() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.pending_reconciliation_run_id = "wr_pending"
-
- error = _tool_loop_error(ctx, "update_and_run_blocks", {"workflow_yaml": "workflow_definition: {blocks: []}"})
-
- assert error is not None
- assert ctx.blocker_signal.internal_reason_code == "tool_error_pending_reconciliation_no_input"
- assert "recorded_outcome_grounding_required" not in error
-
-
-def test_grounding_abstains_for_non_authoritative_or_missing_current_url_and_records_challenge_payload() -> None:
- non_authoritative = _outcome(structural_failure_identity="", page_evidence_refs=[])
- ctx = _ctx(non_authoritative)
- assert arm_recorded_outcome_grounding_requirement(ctx) is None
-
- outcome = _outcome()
- terminal_ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(terminal_ctx)
- terminal_ctx.composition_page_evidence = _bounded_inspect_evidence(
- challenge_state={"detected": True, "requires_human_verification": True},
- challenge_controls=[{"text": "Verify", "selector": "#captcha"}],
- )
- assert maybe_satisfy_recorded_outcome_grounding_requirement(terminal_ctx) is True
- challenge_payload = terminal_ctx.recorded_outcome_grounding_requirement.payload
- assert challenge_payload is not None
- assert challenge_payload.challenge_gated is True
- assert challenge_payload.diagnostic_reason == "challenge_gated"
-
- no_page_ctx = _ctx(_outcome(phase="scout_evaluate", workflow_run_id=None))
- no_page_ctx.observed_browser_urls = []
- arm_recorded_outcome_grounding_requirement(no_page_ctx)
- assert (
- _tool_loop_error(no_page_ctx, "update_workflow", {"workflow_yaml": "workflow_definition: {blocks: []}"}) is None
- )
-
-
-def test_author_time_churn_without_cached_url_requires_current_page_regrounding() -> None:
- ctx = _ctx(_outcome(phase="author_time_reject", workflow_run_id=None))
- ctx.observed_browser_urls = []
- arm_recorded_outcome_grounding_requirement(ctx)
-
- error = _tool_loop_error(ctx, "update_workflow", {"workflow_yaml": "workflow_definition: {blocks: []}"})
-
- assert error is not None
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
-
-
-async def test_update_and_run_entrypoint_regrounds_before_third_preflight(monkeypatch: pytest.MonkeyPatch) -> None:
- ctx = _ctx(_outcome(phase="author_time_reject", workflow_run_id=None))
- ctx.observed_browser_urls = []
- arm_recorded_outcome_grounding_requirement(ctx)
- monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *_args: False)
- monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(
- tools_module,
- "_metadata_contract_run_preflight_reject",
- lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("third preflight reject")),
- )
-
- result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
- SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
- json.dumps({"workflow_yaml": "workflow_definition: {blocks: []}", "block_labels": []}),
- )
-
- assert json.loads(result)["ok"] is False
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
- assert len(ctx.recorded_build_test_outcome_history) == 2
-
-
-def test_no_run_grounding_requires_fresh_post_arm_inspect_evidence() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
-
- arm_recorded_outcome_grounding_requirement(ctx)
-
- assert ctx.composition_page_evidence is None
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
-
-def test_repeated_no_run_author_time_outcome_arms_through_repair_loop() -> None:
- outcome = _outcome(
- phase="author_time_reject",
- attempted_tool="update_workflow",
- verdict="authoring_rejected",
- reason_code="synthesized_parameter_binding_ambiguous",
- workflow_run_id=None,
- structural_failure_identity="authoring:binding",
- )
- ctx = _ctx(outcome)
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
-
- _update_repair_loop_state(ctx, _contract())
-
- requirement = ctx.recorded_outcome_grounding_requirement
- assert requirement is not None
- assert requirement.structural_key == outcome.structural_key
- assert requirement.workflow_run_id is None
- assert ctx.composition_page_evidence is None
-
-
-def test_author_time_grounding_requires_fresh_post_arm_inspect_evidence() -> None:
- outcome = _outcome(
- phase="author_time_reject",
- attempted_tool="update_workflow",
- verdict="authoring_rejected",
- reason_code="synthesized_parameter_binding_ambiguous",
- workflow_run_id=None,
- structural_failure_identity="authoring:binding",
- )
- ctx = _ctx(outcome)
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
-
- arm_recorded_outcome_grounding_requirement(ctx)
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
-
-def test_grounding_payload_reaches_recorded_outcome_prompt() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(observed_after_workflow_run=False, workflow_run_id=None)
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- prompt = _recorded_build_test_outcome_prompt(ctx) # type: ignore[arg-type]
- assert "RECORDED OUTCOME GROUNDING EVIDENCE:" in prompt
- assert f"repeated_structural_key: {outcome.structural_key}" in prompt
- assert "source_tool: inspect_page_for_composition" in prompt
- assert "observed_after_workflow_run: false" in prompt
-
-
-def _binding_ctx() -> SimpleNamespace:
- outcome = _outcome()
- ctx = _ctx(outcome)
- ctx.workflow_yaml = _OWNING_BLOCK_WORKFLOW
- ctx.code_artifact_metadata = None
- ctx.recorded_outcome_binding_constraint = None
- return ctx
-
-
-def test_satisfy_binds_typed_constraint_with_frontier_facet_and_owning_blocks() -> None:
- ctx = _binding_ctx()
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence()
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- constraint = ctx.recorded_outcome_binding_constraint
- assert isinstance(constraint, RecordedOutcomeBindingConstraint)
- assert constraint.repeated_structural_key == ctx.latest_recorded_build_test_outcome.structural_key
- assert constraint.frontier_facet == "selector_frontier"
- assert constraint.owning_block_labels == ["search_records"]
- assert constraint.diagnostic_reason == "none"
- assert constraint.frontier_uncrossable is False
- assert constraint.recorded_block_signatures == {
- "search_records": authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
- }
-
- prompt = _recorded_build_test_outcome_prompt(ctx) # type: ignore[arg-type]
- assert "RECORDED OUTCOME BINDING CONSTRAINT:" in prompt
- assert "frontier_facet: selector_frontier" in prompt
-
-
-def test_definition_contract_unsatisfied_binds_amend_in_place_frontier() -> None:
- outcome = _outcome(reason_code="definition_contract_unsatisfied")
- ctx = _ctx(outcome)
- ctx.workflow_yaml = _OWNING_BLOCK_WORKFLOW
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence()
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
- constraint = ctx.recorded_outcome_binding_constraint
- assert isinstance(constraint, RecordedOutcomeBindingConstraint)
- assert constraint.frontier_facet == "amend_in_place"
-
-
-def test_degraded_capture_binds_uncrossable_constraint() -> None:
- ctx = _binding_ctx()
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = _bounded_inspect_evidence(
- forms=[], result_containers=[], navigation_targets=[], challenge_controls=[]
- )
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
-
- constraint = ctx.recorded_outcome_binding_constraint
- assert isinstance(constraint, RecordedOutcomeBindingConstraint)
- assert constraint.diagnostic_reason == "capture_degraded"
- assert constraint.frontier_uncrossable is True
-
-
-def test_convergence_reject_only_when_owning_block_frontier_unchanged() -> None:
- outcome = _outcome(authored_structure_signature="different_whole_signature")
- recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
- constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- owning_block_labels=["search_records"],
- recorded_block_signatures={"search_records": recorded_sig},
- )
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=outcome,
- recorded_outcome_binding_constraint=constraint,
- )
-
- unchanged = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
- )
- assert unchanged is not None
- assert unchanged.reason == "frontier_unchanged"
- assert unchanged.commit_early_terminal is False
-
- moved_yaml = _OWNING_BLOCK_WORKFLOW.replace('"123"', '"456"')
- assert _recorded_outcome_convergence_reject(ctx, workflow_yaml=moved_yaml, code_artifact_metadata=None) is None
-
-
-def test_convergence_reject_identical_authored_structure() -> None:
- signature = run_execution_module.authored_structure_signature_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
- outcome = _outcome(authored_structure_signature=signature)
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=outcome,
- recorded_outcome_binding_constraint=None,
- )
-
- decision = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
- )
- assert decision is not None
- assert decision.reason == "identical_authored_structure"
-
-
-def test_convergence_reject_uncrossable_frontier_commits_early_terminal() -> None:
- outcome = _outcome(authored_structure_signature="different_whole_signature")
- recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
- constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- owning_block_labels=["search_records"],
- diagnostic_reason="challenge_gated",
- recorded_block_signatures={"search_records": recorded_sig},
- )
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=outcome,
- recorded_outcome_binding_constraint=constraint,
- blocker_signal=None,
- turn_halt=None,
- consecutive_non_converging_repair_count=2,
- turn_ownership=None,
- blocker_signal_claimant=None,
- gate_precedence_conflict_events=[],
- output_contract_actuation_by_signature={},
- output_contract_actuation_count_by_signature={},
- )
-
- decision = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
- )
- assert decision is not None
- assert decision.reason == "frontier_unchanged"
- assert decision.commit_early_terminal is True
-
- _commit_recorded_outcome_early_terminal(ctx)
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind == TurnHaltKind.REPAIR_CEILING_REACHED
- assert ctx.blocker_signal.internal_reason_code == "repair_ceiling_reached"
- assert ctx.blocker_signal.renders_final_reply is True
- assert ctx.blocker_signal.preserves_workflow_draft is True
- assert ctx.blocker_signal.user_facing_reason.strip()
- assert "can't get past this page" in ctx.blocker_signal.user_facing_reason
- assert ctx.turn_halt.blocker_signal is ctx.blocker_signal
-
-
-def test_block_signature_keys_on_code_not_label_or_description() -> None:
- base = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
- described = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT, None)
- renamed = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW_RENAMED, None)
- code_moved = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW.replace('"123"', '"456"'), None)
-
- assert described["search_records"] == base["search_records"]
- assert renamed["renamed_records"] == base["search_records"]
- assert code_moved["search_records"] != base["search_records"]
-
-
-def test_description_only_edit_reads_as_identical_authored_structure() -> None:
- signature = run_execution_module.authored_structure_signature_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
- outcome = _outcome(authored_structure_signature=signature)
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=outcome,
- recorded_outcome_binding_constraint=None,
- )
-
- decision = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT, code_artifact_metadata=None
- )
- assert decision is not None
- assert decision.reason == "identical_authored_structure"
-
-
-def test_frontier_unchanged_reject_fires_when_whole_signature_moves_but_owning_block_holds() -> None:
- outcome = _outcome(authored_structure_signature="whole_signature_from_prior_attempt")
- recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
- constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- owning_block_labels=["search_records"],
- recorded_block_signatures={"search_records": recorded_sig},
- )
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=outcome,
- recorded_outcome_binding_constraint=constraint,
- )
-
- decision = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE, code_artifact_metadata=None
- )
- assert decision is not None
- assert decision.reason == "frontier_unchanged"
- assert decision.commit_early_terminal is False
-
-
-def test_binding_enforces_across_consecutive_author_time_rejects() -> None:
- run_outcome = _outcome(authored_structure_signature="run_whole_signature")
- recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
- constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=run_outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- owning_block_labels=["search_records"],
- recorded_block_signatures={"search_records": recorded_sig},
- )
- ctx = SimpleNamespace(
- latest_recorded_build_test_outcome=run_outcome,
- recorded_outcome_binding_constraint=constraint,
- )
-
- first = _recorded_outcome_convergence_reject(ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None)
- assert first is not None
- assert first.reason == "frontier_unchanged"
-
- # The first reject re-keys `latest` to an author-time reject outcome whose structural key
- # differs from the bound run outcome; the binding must still enforce on the next
- # different-structure-but-frontier-unchanged submit.
- author_time_latest = _outcome(
- phase="author_time_reject",
- reason_code="unchanged_after_recorded_outcome",
- structural_failure_identity="author_time:rekeyed",
- authored_structure_signature="author_time_reject_whole_signature",
- )
- assert author_time_latest.structural_key != constraint.repeated_structural_key
- ctx.latest_recorded_build_test_outcome = author_time_latest
-
- second = _recorded_outcome_convergence_reject(
- ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE, code_artifact_metadata=None
- )
- assert second is not None
- assert second.reason == "frontier_unchanged"
-
-
-def test_sibling_churn_frontier_unchanged_rejects_reach_honest_churn_stop() -> None:
- ctx = _ctx()
-
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- _record_author_time_reject_outcome(
- ctx,
- reason_code="unchanged_after_recorded_outcome",
- summary="The authored code and output structure are unchanged after the last recorded test outcome.",
- structural_payload={
- "reason_code": "unchanged_after_recorded_outcome",
- "authored_structure_signature": f"moving_whole_signature_{index}",
- "block_labels": ["search_records"],
- },
- authored_structure_signature=f"moving_whole_signature_{index}",
- block_labels=["search_records"],
- )
- _record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
-
- # Sibling churn moves the whole-signature key every turn, so each recorded outcome reads as a
- # non-repeat; the frontier-unchanged flag keeps the churn counter climbing to the honest stop
- # instead of resetting and riding to max_turns.
- assert (
- ctx.recorded_build_test_outcome_history[-1]["structural_key"]
- != ctx.recorded_build_test_outcome_history[-2]["structural_key"]
- )
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
- assert ctx.blocker_signal.internal_reason_code == "code_authoring_guardrail_churn"
-
-
-def test_early_terminal_renders_typed_final_reply_and_preserves_draft() -> None:
- constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key="repeated_key",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- owning_block_labels=["search_records"],
- diagnostic_reason="challenge_gated",
- )
- ctx = SimpleNamespace(
- recorded_outcome_binding_constraint=constraint,
- blocker_signal=None,
- turn_halt=None,
- consecutive_non_converging_repair_count=3,
- turn_ownership=None,
- blocker_signal_claimant=None,
- gate_precedence_conflict_events=[],
- output_contract_actuation_by_signature={},
- output_contract_actuation_count_by_signature={},
- )
-
- _commit_recorded_outcome_early_terminal(ctx)
-
- signal = ctx.blocker_signal
- assert signal.renders_final_reply is True
- assert signal.recovery_hint == "report_blocker_to_user"
- assert signal.blocked_tool == "update_workflow"
- assert signal.cleared_by_tools == frozenset()
- assert signal.preserves_workflow_draft is True
- assert signal.internal_reason_code == "repair_ceiling_reached"
- assert "can't get past this page" in signal.user_facing_reason
- assert signal.internal_reason_code not in signal.user_facing_reason
-
- assert ctx.turn_halt.kind == TurnHaltKind.REPAIR_CEILING_REACHED
- assert ctx.turn_halt.blocker_signal is signal
- assert ctx.turn_halt.draft_state == {"preserves_workflow_draft": True}
- assert ctx.turn_halt.extra["consecutive_identical_repair_count"] == 3
-
-
-def test_update_repair_loop_state_clears_stale_requirement_when_outcome_not_authoritative() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- assert ctx.recorded_outcome_grounding_requirement is not None
- ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
- repeated_structural_key=outcome.structural_key or "",
- phase="persisted_block_run",
- reason_code="runtime_block_failure",
- frontier_facet="selector_frontier",
- )
-
- ctx.latest_recorded_build_test_outcome = None
- ctx.recorded_build_test_outcome_history = []
-
- _update_repair_loop_state(ctx, _contract())
-
- assert ctx.recorded_outcome_grounding_requirement is None
- assert ctx.recorded_outcome_binding_constraint is None
-
-
-def test_scout_contract_grounds_no_run_requirement_after_composition_miss() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = None
- ctx.scout_observation_contract = _scout_contract()
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
- payload = ctx.recorded_outcome_grounding_requirement.payload
- assert payload is not None
- assert payload.source_tool == "evaluate"
- assert payload.capture_degraded is False
- assert "source_tool: evaluate" in _recorded_build_test_outcome_prompt(ctx)
-
-
-def test_degraded_scout_contract_admitted_with_capture_degraded_recorded() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = None
- ctx.scout_observation_contract = _scout_contract(has_bounded_page_schema=False)
-
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
- payload = ctx.recorded_outcome_grounding_requirement.payload
- assert payload is not None
- assert payload.capture_degraded is True
- assert payload.diagnostic_reason == "capture_degraded"
-
-
-def test_arming_no_run_requirement_clears_prior_scout_contract() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- ctx.scout_observation_contract = _scout_contract()
-
- arm_recorded_outcome_grounding_requirement(ctx)
-
- assert ctx.scout_observation_contract is None
- ctx.composition_page_evidence = None
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
-
-def test_run_required_requirement_rejects_unstamped_scout_contract() -> None:
- outcome = _outcome()
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = None
- ctx.scout_observation_contract = _scout_contract()
-
- with capture_logs() as logs:
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- event = next(log for log in logs if log["event"] == "copilot recorded outcome grounding rejected")
- assert event["reject_reason"] == "run_id_mismatch"
-
-
-def test_tampered_scout_contract_rejected_by_grounding_lane() -> None:
- outcome = _outcome(phase="scout_evaluate", workflow_run_id=None, attempted_tool="evaluate")
- ctx = _ctx(outcome)
- arm_recorded_outcome_grounding_requirement(ctx)
- ctx.composition_page_evidence = None
- contract = _scout_contract()
- ctx.scout_observation_contract = replace(contract, source_url="https://evil.example.com/injected")
-
- with capture_logs() as logs:
- assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is False
-
- event = next(log for log in logs if log["event"] == "copilot recorded outcome grounding rejected")
- assert event["reject_reason"] == "not_inspect_source"
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index e230ca912..9a410425b 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -7,7 +7,6 @@ from unittest.mock import AsyncMock, patch
import pytest
-from skyvern.config import settings
from skyvern.forge.sdk.copilot.context import (
CredentialCheck,
StructuredContext,
@@ -446,28 +445,6 @@ def test_credential_prompt_reason_policy_none_is_safe() -> None:
assert credential_prompt_reason(None, "Everything is set, no action needed.") is None
-async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_log_only(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(settings, "ENV", "local")
- monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
-
- policy = await build_request_policy(
- user_message="Use password: Hunter99! to sign in.",
- workflow_yaml="",
- chat_history=[],
- global_llm_context="",
- organization_id="o_test",
- handler=None,
- )
-
- assert policy.raw_secret_detected is True
- assert policy.user_response_policy == "ask_clarification"
- assert policy.allow_update_workflow is False
- assert policy.allow_run_blocks is False
-
-
def _cred(
name: str,
credential_id: str,
diff --git a/tests/unit/test_copilot_scout_captcha_solve.py b/tests/unit/test_copilot_scout_captcha_solve.py
index 4f28e7cf0..a048e07c0 100644
--- a/tests/unit/test_copilot_scout_captcha_solve.py
+++ b/tests/unit/test_copilot_scout_captcha_solve.py
@@ -66,7 +66,6 @@ def _mock_context() -> AgentContext:
"scouted_interactions": [],
"scout_trajectory": [],
"challenge_solve_attempts": {},
- "never_captured_obligation": None,
"last_evaluate_actionable_signature": None,
"last_scout_observation_trajectory_index": None,
}
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index 03ba37d96..a2d115c9e 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -647,86 +647,6 @@ class TestUpdateWorkflowDirect:
assert ctx.workflow_yaml == "original yaml"
- @pytest.mark.asyncio
- async def test_rejects_stale_block_metadata_before_persistence(self) -> None:
- from skyvern.forge.sdk.copilot.tools import _update_workflow
-
- prior_yaml = """
-title: Count example.com topic alpha results
-workflow_definition:
- blocks:
- - block_type: navigation
- label: search_topic_alpha
- title: Search Topic Alpha
- next_block_label: null
- navigation_goal: Search example.com for topic alpha.
-"""
- submitted_yaml = """
-title: Count example.com sample beta results
-workflow_definition:
- blocks:
- - block_type: navigation
- label: search_topic_alpha
- title: Search Topic Alpha
- next_block_label: null
- navigation_goal: Search example.com for sample beta.
-"""
- ctx = _make_ctx(workflow_yaml=prior_yaml)
-
- result = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)
-
- assert result["ok"] is False
- assert "corrected block metadata still appears stale" in result["error"]
- assert "search_topic_alpha" in result["error"]
- assert ctx.workflow_yaml == prior_yaml
-
- @pytest.mark.asyncio
- async def test_stale_check_prefers_in_turn_emission_over_turn_start_yaml(self) -> None:
- # Cross-path flow: an earlier inline REPLACE_WORKFLOW left
- # ctx.last_workflow_yaml at the renamed state. A subsequent
- # update_workflow call submits a regression that re-stales the label
- # — this must compare against ctx.last_workflow_yaml, not the older
- # ctx.workflow_yaml that still points at turn-start.
- from skyvern.forge.sdk.copilot.tools import _update_workflow
-
- turn_start_yaml = """
-title: Alpha workflow
-workflow_definition:
- blocks:
- - block_type: navigation
- label: review_alpha_one
- title: Review Alpha One
- next_block_label: null
- navigation_goal: Review the alpha one records on the directory.
-"""
- in_turn_emission_yaml = """
-title: Beta workflow
-workflow_definition:
- blocks:
- - block_type: navigation
- label: review_beta_two
- title: Review Beta Two
- next_block_label: null
- navigation_goal: Review the beta two records on the directory.
-"""
- regressed_yaml = """
-title: Beta workflow
-workflow_definition:
- blocks:
- - block_type: navigation
- label: review_beta_two
- title: Review Beta Two
- next_block_label: null
- navigation_goal: Review the gamma three records on the directory.
-"""
- ctx = _make_ctx(workflow_yaml=turn_start_yaml)
- ctx.last_workflow_yaml = in_turn_emission_yaml
-
- result = await _update_workflow({"workflow_yaml": regressed_yaml}, ctx)
-
- assert result["ok"] is False
- assert "review_beta_two" in result["error"]
-
class TestWorkflowUpdatePersistence:
def test_record_marks_last_workflow_but_not_legacy_persisted(self) -> None:
diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.py
index 8a97e4634..b85e69385 100644
--- a/tests/unit/test_copilot_tool_loop_error_block_running.py
+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py
@@ -44,9 +44,7 @@ def _ctx(**overrides: Any) -> Any:
"staged_workflow": None,
"staged_workflow_yaml": None,
"has_staged_proposal": False,
- "synthesized_block_reopened_for_output_coverage": False,
"output_contract_actuation_by_signature": {},
- "output_contract_reject_count_by_signature": {},
"output_contract_actuation_count_by_signature": {},
"output_contract_armed_directive_fingerprint_by_signature": {},
"output_contract_dispatch_reopened_by_signature": {},
diff --git a/tests/unit/test_copilot_tools_credential_id_misbinding.py b/tests/unit/test_copilot_tools_credential_id_misbinding.py
index dddc3e1a5..8777cac48 100644
--- a/tests/unit/test_copilot_tools_credential_id_misbinding.py
+++ b/tests/unit/test_copilot_tools_credential_id_misbinding.py
@@ -400,6 +400,12 @@ def _ctx() -> MagicMock:
ctx.organization_id = "o_test"
ctx.allow_untested_workflow_draft = False
ctx.request_policy = None
+ ctx.code_artifact_metadata = {}
+ ctx.code_authoring_guardrail_reject_count = 0
+ ctx.blocker_signal = None
+ ctx.turn_halt = None
+ ctx.build_test_outcomes = []
+ ctx.latest_recorded_build_test_outcome = None
return ctx
diff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.py
index 3bbb26424..69b2e8827 100644
--- a/tests/unit/test_copilot_turn_halt.py
+++ b/tests/unit/test_copilot_turn_halt.py
@@ -6,7 +6,6 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
_OUTPUT_CONTRACT_TERMINAL_REASON_CODES,
- METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
CopilotToolBlockerSignal,
stash_blocker_signal,
)
@@ -82,10 +81,6 @@ def _signal(
TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,
),
(_signal(internal_reason_code="probable_site_block_stop"), TurnHaltKind.PROBABLE_SITE_BLOCK),
- (
- _signal(internal_reason_code=METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE),
- TurnHaltKind.METADATA_REJECT_SAME_KEY,
- ),
],
)
def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expected_kind: TurnHaltKind) -> None:
diff --git a/tests/unit/test_copilot_turn_ownership.py b/tests/unit/test_copilot_turn_ownership.py
index d171d266a..daebb221e 100644
--- a/tests/unit/test_copilot_turn_ownership.py
+++ b/tests/unit/test_copilot_turn_ownership.py
@@ -109,22 +109,22 @@ def test_r31_three_way_contradiction_yields_single_owner() -> None:
_grant_ladder(ctx)
claim_turn(ctx, TurnClaimant.OUTPUT_CONTRACT_ACTUATION)
- rescout = claim_turn(ctx, TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER)
+ credential_churn = claim_turn(ctx, TurnClaimant.CREDENTIAL_PRIORITY_CHURN)
persistence_payload = claim_and_stash_blocker_signal(
ctx,
TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE,
_signal(SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE, blocker_kind="tool_error", renders_final_reply=False),
)
- assert rescout is ClaimOutcome.YIELDED
+ assert credential_churn is ClaimOutcome.YIELDED
assert persistence_payload is None
assert ctx.blocker_signal is None
owner = current_turn_owner(ctx)
assert owner is not None
assert owner.claimant is TurnClaimant.OUTPUT_CONTRACT_ACTUATION
assert sorted(_conflict_fingerprints(ctx)) == [
+ "output_contract_actuation>credential_priority_authoring_churn",
"output_contract_actuation>synthesized_block_persistence_force",
- "output_contract_actuation>uncovered_output_rescout_steer",
]
@@ -134,41 +134,6 @@ def test_precedence_order_covers_every_claimant_once() -> None:
assert set(CLAIMANT_REASON_CODE_FAMILIES) == set(TurnClaimant)
-def test_rescout_outranks_grounding_and_persistence_per_cascade_order() -> None:
- ctx = make_copilot_context()
- assert claim_turn(ctx, TurnClaimant.RECORDED_OUTCOME_GROUNDING) is ClaimOutcome.OWNED
- assert claim_turn(ctx, TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER) is ClaimOutcome.OWNED
-
- ctx2 = make_copilot_context()
- assert claim_and_stash_blocker_signal(
- ctx2,
- TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE,
- _signal(SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE, blocker_kind="tool_error", renders_final_reply=False),
- )
- assert claim_turn(ctx2, TurnClaimant.UNCOVERED_OUTPUT_RESCOUT_STEER) is ClaimOutcome.OWNED
-
-
-def test_grounding_outranks_persistence_per_nested_exception() -> None:
- ctx = make_copilot_context()
- persisted = claim_and_stash_blocker_signal(
- ctx,
- TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE,
- _signal(SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE, blocker_kind="tool_error", renders_final_reply=False),
- )
- assert persisted is not None
- grounding = claim_and_stash_blocker_signal(
- ctx,
- TurnClaimant.RECORDED_OUTCOME_GROUNDING,
- _signal(
- "recorded_outcome_grounding_required", blocker_kind="missing_required_context", renders_final_reply=False
- ),
- )
- assert grounding is not None
- assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == "recorded_outcome_grounding_required"
- assert "recorded_outcome_grounding>synthesized_block_persistence_force" in _conflict_fingerprints(ctx)
-
-
def test_genuinely_terminal_family_is_the_shared_blocker_signal_set() -> None:
assert CLAIMANT_REASON_CODE_FAMILIES[TurnClaimant.GENUINELY_TERMINAL] is GENUINELY_TERMINAL_BLOCKER_REASON_CODES
@@ -345,9 +310,9 @@ def test_unclaimed_signal_fails_open_to_render_while_owner_live() -> None:
assert ctx.gate_precedence_conflict_events == []
-def test_preflight_claim_never_outlives_the_rejecting_call() -> None:
+def test_transient_claim_never_outlives_the_claiming_call() -> None:
ctx = make_copilot_context()
- assert claim_turn(ctx, TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT) is ClaimOutcome.OWNED
+ assert claim_turn(ctx, TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION) is ClaimOutcome.OWNED
assert current_turn_owner(ctx) is None
churn = _churn_signal()
@@ -355,11 +320,11 @@ def test_preflight_claim_never_outlives_the_rejecting_call() -> None:
assert blocker_signal_render_allowed(ctx, churn) is True
-def test_preflight_claim_yields_to_live_ladder_and_records_conflict() -> None:
+def test_transient_claim_yields_to_live_ladder_and_records_conflict() -> None:
ctx = make_copilot_context()
_grant_ladder(ctx)
- assert claim_turn(ctx, TurnClaimant.METADATA_RUN_PREFLIGHT_REJECT) is ClaimOutcome.YIELDED
- assert _conflict_fingerprints(ctx) == ["output_contract_actuation>metadata_run_preflight_reject"]
+ assert claim_turn(ctx, TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION) is ClaimOutcome.YIELDED
+ assert _conflict_fingerprints(ctx) == ["output_contract_actuation>post_run_page_path_interaction"]
def test_carve_out_claimants_are_transient() -> None:
diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.py
index cd44acde9..b69da63d5 100644
--- a/tests/unit/test_copilot_watchdog.py
+++ b/tests/unit/test_copilot_watchdog.py
@@ -467,9 +467,7 @@ def test_reconciliation_guard_message_does_not_say_timed_out() -> None:
repeated_action_fingerprint_streak_count=0,
last_test_non_retriable_nav_error=None,
pending_reconciliation_run_id="wr_guarded",
- synthesized_block_reopened_for_output_coverage=False,
output_contract_actuation_by_signature={},
- output_contract_reject_count_by_signature={},
output_contract_actuation_count_by_signature={},
output_contract_armed_directive_fingerprint_by_signature={},
)
diff --git a/tests/unit/test_file_upload_limits.py b/tests/unit/test_file_upload_limits.py
new file mode 100644
index 000000000..dd29816db
--- /dev/null
+++ b/tests/unit/test_file_upload_limits.py
@@ -0,0 +1,39 @@
+from pathlib import Path
+
+import pytest
+
+from skyvern.constants import AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT, MAX_UPLOAD_FILE_COUNT
+from skyvern.forge.sdk.workflow.models.block import FileUploadBlock
+
+
+ "max_file_count",
+ [MAX_UPLOAD_FILE_COUNT, AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT],
+ ids=["s3", "azure"],
+)
+def test_cloud_storage_block_allows_300_files(tmp_path: Path, max_file_count: int) -> None:
+ for index in range(300):
+ (tmp_path / f"file_{index}.txt").touch()
+
+ files_to_upload = FileUploadBlock.model_construct()._get_files_to_upload_from_download_dir(
+ download_files_path=str(tmp_path),
+ max_file_count=max_file_count,
+ )
+
+ assert len(files_to_upload) == 300
+
+
+ "max_file_count",
+ [MAX_UPLOAD_FILE_COUNT, AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT],
+ ids=["s3", "azure"],
+)
+def test_cloud_storage_block_rejects_more_than_300_files(tmp_path: Path, max_file_count: int) -> None:
+ for index in range(301):
+ (tmp_path / f"file_{index}.txt").touch()
+
+ with pytest.raises(ValueError, match="Max: 300"):
+ FileUploadBlock.model_construct()._get_files_to_upload_from_download_dir(
+ download_files_path=str(tmp_path),
+ max_file_count=max_file_count,
+ )
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 408329b44..a9386178d 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -35,16 +35,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.context import AgentResult, DeliveredUnverifiedPublicOutputs
-from skyvern.forge.sdk.copilot.schema_incompatibility import (
- SchemaIncompatibility,
- render_schema_incompatibility_user_reason,
-)
from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope
-from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
-from skyvern.forge.sdk.copilot.turn_outcome import (
- build_minimal_turn_outcome,
- with_copilot_code_mode_diagnostics,
-)
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.routes.workflow_copilot import (
COPILOT_V2_FLAG_KEY,
@@ -53,7 +44,6 @@ from skyvern.forge.sdk.routes.workflow_copilot import (
workflow_copilot_chat_audio,
workflow_copilot_chat_post,
)
-from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatMessage,
WorkflowCopilotChatRequest,
@@ -2926,74 +2916,3 @@ async def test_persist_state_auto_applicable_without_staged_commit_still_protect
)
app.DATABASE.workflow_params.update_workflow_copilot_chat.assert_not_awaited()
-
-
-def _schema_incompatibility_ctx() -> SimpleNamespace:
- incompat = SchemaIncompatibility(
- block_label="capture_row",
- incompatible_paths=("shoebox",),
- known_output_paths=("order_date", "order_total"),
- )
- return SimpleNamespace(
- latest_schema_incompatibility=incompat,
- turn_halt=SimpleNamespace(kind=TurnHaltKind.SCHEMA_INCOMPATIBILITY),
- code_native_pending_capability=None,
- last_test_ok=None,
- last_failed_workflow_yaml=None,
- )
-
-
-def test_schema_incompatibility_turn_outcome_is_not_repair_ceiling() -> None:
- # SKY-11380: the schema-incompatibility halt is a distinct typed outcome; it must
- # not masquerade as the repair-ceiling diagnostic on the persisted turn.
- ctx = _schema_incompatibility_ctx()
- reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)
- outcome = with_copilot_code_mode_diagnostics(
- build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),
- ctx,
- )
-
- assert outcome.copilot_repair_ceiling_hit is False
- assert outcome.copilot_schema_incompatibility is not None
- assert outcome.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]
- assert outcome.copilot_schema_incompatibility["known_output_paths"] == ["order_date", "order_total"]
-
-
-def test_schema_incompatibility_persists_and_recalls_for_followup_turn() -> None:
- # The follow-up "what was the problem?" turn reads the prior assistant outcome from
- # chat history; the structured incompatibility survives the round-trip so it can be reported.
- ctx = _schema_incompatibility_ctx()
- reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)
- outcome = with_copilot_code_mode_diagnostics(
- build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),
- ctx,
- )
- now = datetime.now(timezone.utc)
- messages = [
- WorkflowCopilotChatMessage(
- workflow_copilot_chat_message_id="m1",
- workflow_copilot_chat_id="c1",
- sender=WorkflowCopilotChatSender.USER,
- content="add a shoebox field to the extraction",
- created_at=now,
- modified_at=now,
- ),
- WorkflowCopilotChatMessage(
- workflow_copilot_chat_message_id="m2",
- workflow_copilot_chat_id="c1",
- sender=WorkflowCopilotChatSender.AI,
- content=reply,
- turn_outcome=outcome,
- created_at=now,
- modified_at=now,
- ),
- ]
-
- recalled = workflow_copilot_route._latest_assistant_turn_outcome(messages)
-
- assert recalled is not None
- assert recalled.terminal_reason == "turn_halt:schema_incompatibility"
- assert recalled.copilot_schema_incompatibility is not None
- assert recalled.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]
- # The persisted reply reports the problem in product language.
- assert "shoebox" in messages[1].content