Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
from __future__ import annotations
import asyncio
import copy
import hashlib
import json
from collections.abc import AsyncIterator, Coroutine
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
from sqlalchemy.ext.asyncio import AsyncEngine
from structlog.testing import capture_logs
from skyvern.forge import app as forge_app
from skyvern.forge.failure_classifier import with_sign_in_form_visible
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import runtime
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import (
_RECORDED_PAGE_TEXT_SECURITY_BOUNDARY,
_build_dynamic_system_prompt,
_build_user_context,
_build_wip_exit_result,
_prior_run_debug_text,
_recorded_build_test_outcome_prompt,
_review_projection_for,
_translate_to_agent_result,
)
from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentence
from skyvern.forge.sdk.copilot.build_test_outcome import (
BLOCK_FACT_SCREEN_NOTICE_MAX_CHARS,
BLOCK_FACT_URL_MAX_CHARS,
OBSERVED_BLOCK_END_URLS_EMPTY,
OBSERVED_BLOCK_END_URLS_UNREPORTABLE,
OBSERVED_BLOCK_END_URLS_WITHHELD,
BuildTestConnectFailure,
BuildTestEvidencePacket,
BuildTestFailedOperation,
BuildTestPacketAiFallbackBlock,
BuildTestPacketDownload,
BuildTestPacketFailure,
BuildTestPacketLocatorObservation,
BuildTestPacketPageState,
BuildTestPacketRegisteredOutput,
BuildTestPacketRequestedOutput,
BuildTestPacketUnfinishedItem,
LoopInputFact,
LoopInputs,
LoopSelectedInput,
PostRunPagePathFailure,
RecordedBuildTestOutcome,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
bind_post_run_page_evidence,
coerce_block_end_urls,
observed_value_extraction_scaffold_lines,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
recorded_outcome_from_authoring_repair_context,
recorded_outcome_from_run_blocks_result,
recorded_outcome_from_scout_act_observe_hollow,
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import AgentResult, CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.enforcement import _summarize_tool_output
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind
from skyvern.forge.sdk.copilot.output_utils import (
_BUILD_TEST_PACKET_MAX_CHARS,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
BUILD_TEST_PACKET_KEY,
_compact_block_fact_maps,
_compact_packet_for_aggregate_limit,
_compacted_newest_labelled,
labelled_url_screen_reasons,
project_build_test_packet_for_llm,
project_direct_test_handoff_packet_for_llm,
sanitize_tool_result_for_llm,
)
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
build_test_page_state_from_evidence,
inject_runtime_authoring_repair_context,
)
from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
from skyvern.forge.sdk.copilot.tools.run_execution import (
COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR,
_authored_literal_locator_selectors,
_carry_prior_attempt_change_identity_into_result,
_failed_block_code,
_failing_code_line,
_newest_failed_result,
_record_run_blocks_result,
_recorded_run_block_result,
_run_blocks_and_collect_debug,
_verify_and_record_run_blocks_result,
build_test_evidence_packet,
finalize_build_test_result,
loop_definition_facts,
loop_selected_input,
project_run_results_page,
with_loop_observations,
)
from skyvern.forge.sdk.copilot.tools.workflow_update import (
_update_workflow,
publish_workflow_candidate,
restore_pending_workflow_proposal,
)
from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations, workflow_to_copilot_yaml
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError
from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from skyvern.forge.sdk.schemas.workflow_copilot import (
COPILOT_PRIVATE_SETTINGS_KEY,
CopilotProposalMetadata,
copilot_proposal_metadata,
)
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import CodeBlock, ForLoopBlock
from skyvern.forge.sdk.workflow.models.google_sheets_blocks import GoogleSheetsWriteBlock
from skyvern.forge.sdk.workflow.models.parameter import (
OutputParameter,
ParameterType,
WorkflowParameter,
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.models.workflow import (
COPILOT_TEST_WORKFLOW_CREATOR,
Workflow,
WorkflowDefinition,
WorkflowRunStatus,
workflow_definition_sha256,
)
from skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatus
from skyvern.schemas.workflows import BlockType
from skyvern.services import workflow_service as workflow_service_module
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import ActionStatus
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from tests.unit.copilot_test_helpers import (
HANDBACK_WORKFLOW_YAML,
HARNESS_RUN_CREATED_AT,
SEARCH_THEN_SELECT_WORKFLOW_YAML,
count_record_and_send,
failed_second_factor_run,
handback_ctx,
historical_code_version,
inert_approval_workflow,
install_get_run_results_harness,
install_historical_run,
install_run_blocks_harness,
make_copilot_ctx,
make_stub_html_artifact,
page_only_failed_block,
passing_run,
run_result_action_row,
run_result_block_row,
same_run_page_evidence,
terminal_extraction_block,
two_page_login_yaml,
)
def test_structural_key_changes_when_page_or_result_structure_changes() -> None:
first = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_first",
block_labels=["search_records"],
structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
page_evidence_refs=["origin_present", "results:empty"],
observed_evidence_summary="No matching records.",
)
second = first.model_copy(
update={
"workflow_run_id": "wr_second",
"page_evidence_refs": ["origin_present", "results:table_rows"],
"observed_evidence_summary": "A table with one result row appeared.",
}
)
assert first.structural_key is not None
assert second.structural_key is not None
assert first.structural_key != second.structural_key
def test_run_blocks_outcome_records_requested_labels_and_shape_hashes() -> None:
outcome = recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": "wr_failed",
"requested_block_labels": ["open", "search", "extract"],
"blocks": [
{"label": "search", "status": "failed", "failure_type": "runtime_error"},
],
"overall_status": "failed",
"failure_type": "runtime_error",
},
},
block_shape_hashes={"open": "h1", "search": "h2", "extract": "h3"},
)
assert outcome is not None
assert outcome.requested_block_labels == ["open", "search", "extract"]
assert outcome.block_shape_hashes == {"open": "h1", "search": "h2", "extract": "h3"}
def test_registered_block_download_reaches_packet_without_parseable_artifact_evidence() -> None:
ctx = make_copilot_ctx()
ctx.registered_artifact_evidence = None
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_1",
"overall_status": "completed",
"requested_block_labels": ["download_statement"],
"executed_block_labels": ["download_statement"],
"blocks": [
{
"label": "download_statement",
"status": "completed",
"extracted_data": {"downloaded_file_artifact_ids": ["a_dl_9"]},
}
],
},
}
packet = build_test_evidence_packet(ctx, result)
assert [(download.artifact_id, download.file_name) for download in packet.downloads] == [("a_dl_9", None)]
def test_packet_projects_every_recorded_block_output_in_row_order_and_scrubs_secrets() -> None:
ctx = _locator_packet_ctx()
ctx.browser_session_id = "pbs_recorded_outputs"
register_secret_scrub_value(ctx, "private-value")
try:
result = {
"ok": False,
"data": {
"workflow_run_id": "wr_recorded_outputs",
"overall_status": "failed",
"blocks": [
{
"label": "produce_rows",
"status": "completed",
"output": {"real_key": "private-value"},
},
{"label": "produce_rows", "status": "completed", "output": []},
{
"label": "use_rows",
"status": "failed",
"failure_reason": "missing output key",
"output": {"available_keys": ["real_key"]},
},
],
},
}
packet = build_test_evidence_packet(ctx, result)
finally:
clear_session_scrub_values("pbs_recorded_outputs")
assert [output.model_dump(mode="json", exclude_none=True) for output in packet.registered_outputs] == [
{
"label": "produce_rows",
"status": "completed",
"output": {"real_key": "[REDACTED_SECRET]"},
"value_complete": True,
},
{
"label": "produce_rows",
"status": "completed",
"output": [],
"value_complete": True,
},
{
"label": "use_rows",
"status": "failed",
"output": {"available_keys": ["real_key"]},
"value_complete": True,
},
]
producer_values = [output.output for output in packet.registered_outputs if output.label == "produce_rows"]
repair_signals = [output.output for output in packet.registered_outputs if output.label == "use_rows"]
assert producer_values == [{"real_key": "[REDACTED_SECRET]"}, []]
assert repair_signals == [{"available_keys": ["real_key"]}]
assert (
packet.omission_notices.count("registered_outputs redacted 1 item(s) containing registered secret values.") == 1
)
@pytest.mark.asyncio
async def test_packet_surfaces_available_keys_from_a_sheets_render_failure() -> None:
write_block = GoogleSheetsWriteBlock(
label="write_row",
output_parameter=OutputParameter(
parameter_type=ParameterType.OUTPUT,
key="write_row_output",
output_parameter_id="op_write_row",
workflow_id="w",
created_at=datetime.now(UTC),
modified_at=datetime.now(UTC),
),
spreadsheet_url="https://docs.google.com/spreadsheets/d/sheet-id/edit",
values='[["{{ extract_failure_rate.output.extracted_information.failure_rate }}"]]',
)
run_context = WorkflowRunContext(
workflow_title="t",
workflow_id="w",
workflow_permanent_id="wpid",
workflow_run_id="wr_sheets_available_keys",
aws_client=MagicMock(),
)
run_context.values["extract_failure_rate"] = {
"status": "completed",
"extracted_information": {"failure_rate": "25.65%"},
"output": {"failure_rate": "25.65%"},
}
with pytest.raises(ValueError) as excinfo:
write_block._render_values_or_raise(run_context)
with (
patch.object(GoogleSheetsWriteBlock, "record_output_parameter_value", AsyncMock()) as record_output,
patch.object(GoogleSheetsWriteBlock, "build_block_result", AsyncMock()),
):
await write_block._template_format_failure_result(
excinfo.value, str(excinfo.value), run_context, "wr_sheets_available_keys", None, None
)
recorded_output = record_output.await_args.args[2]
packet = build_test_evidence_packet(
_locator_packet_ctx(),
{
"ok": False,
"data": {
"workflow_run_id": "wr_sheets_available_keys",
"overall_status": "failed",
"blocks": [{"label": "write_row", "status": "failed", "output": recorded_output}],
},
},
)
assert "extract_failure_rate.output.failure_rate" in packet.registered_outputs[0].output["available_keys"]
def test_projection_keeps_available_keys_when_the_output_exceeds_the_per_output_cap() -> None:
failure_output = {
"failure_reason": (
"Failed to format jinja template: block `write_row` field `values` references a value no "
"upstream block produced: 'dict object' has no attribute 'extracted_information'. Return "
"that key from the producing block, or write an explicit default (e.g. "
"{{ block_label.field | default('') }}) if an empty cell is intended."
),
"available_keys": [
*[f"extract_metrics.output.metric_{index:02d}" for index in range(25)],
*[f"block_{index:02d}" for index in range(10)],
],
}
projected = project_build_test_packet_for_llm(
build_test_evidence_packet(
_locator_packet_ctx(),
{
"ok": False,
"data": {
"workflow_run_id": "wr_available_keys_cap",
"overall_status": "failed",
"blocks": [{"label": "write_row", "status": "failed", "output": failure_output}],
},
},
)
)
registered = projected.registered_outputs[0]
kept = registered.output["available_keys"]
assert "extract_metrics.output.metric_00" in kept
# Top-level names are the only answer when a root is undefined, so trimming must not empty them first.
assert "block_00" in kept
assert 0 < len(kept) < 35
assert registered.value_complete is False
assert any("available_keys on write_row shortened" in notice for notice in projected.omission_notices)
def test_projection_keeps_repair_keys_when_the_failure_reason_alone_fills_the_cap() -> None:
# A generic Jinja failure quotes the whole template, so the reason alone can exceed the cap, and
# build_block_failure_output copies that same text into every errors[].reasoning.
long_reason = "Failed to format Jinja style parameter '" + ("{{ verify.x }} " * 90) + "'."
failure_output = {
"status": "failed",
"failure_reason": long_reason,
"errors": [
{"error_code": "file_parse_failed", "reasoning": long_reason, "confidence_float": 1.0},
{"error_code": "template_failed", "reasoning": long_reason, "confidence_float": 1.0},
],
"available_keys": ["verify.output.status", "verify.output.detail", "verify", "verify_output"],
}
projected = project_build_test_packet_for_llm(
build_test_evidence_packet(
_locator_packet_ctx(),
{
"ok": False,
"data": {
"workflow_run_id": "wr_long_reason",
"overall_status": "failed",
"blocks": [{"label": "write_row", "status": "failed", "output": failure_output}],
},
},
)
)
registered = projected.registered_outputs[0]
assert registered.output["available_keys"], "the repair paths must survive a reason that fills the cap"
assert registered.output["failure_reason"].endswith("...")
assert [error["reasoning"].endswith("...") for error in registered.output["errors"]] == [True, True]
assert registered.value_complete is False
def test_recorded_run_block_projection_keeps_falsey_and_null_outputs() -> None:
def row(output: object) -> SimpleNamespace:
return SimpleNamespace(
label="produce_rows",
block_type=SimpleNamespace(name="CODE"),
status="completed",
workflow_run_block_id="wrb_1",
task_id=None,
failure_reason=None,
error_codes=[],
output=output,
)
assert _recorded_run_block_result(row({}))["output"] == {}
assert _recorded_run_block_result(row(""))["output"] == ""
assert _recorded_run_block_result(row(None))["output"] is None
def test_connect_failure_projects_through_recorded_outcome_and_packet() -> None:
ctx = make_copilot_ctx()
ctx.workflow_yaml = "title: preserved draft"
ctx.staged_workflow_yaml = "title: preserved draft"
ctx.last_workflow_yaml = "title: preserved draft"
failure = BuildTestConnectFailure(
state="cdp_connect_failed",
workflow_run_id="wr_1",
workflow_run_block_id="wrb_1",
task_id="tsk_1",
browser_session_id="pbs_1",
)
result = {
"ok": False,
"error": "Build-test browser acquisition stopped: cdp_connect_failed.",
"data": {
"workflow_run_id": "wr_1",
"overall_status": "setup_failed",
"browser_session_id": "pbs_1",
"requested_block_labels": ["open"],
"executed_block_labels": [],
"blocks": [],
"build_test_connect_failure": failure.model_dump(mode="json"),
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.connect_failure == failure
assert outcome.observed_evidence_summary == build_test_connect_failure_sentence(failure)
packet = build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
assert packet.failure is not None
assert packet.failure.connect_failure == failure
assert packet.canonical_workflow_yaml == "title: preserved draft"
def test_billing_credit_refusal_schema_requires_no_retry_or_new_identity() -> None:
failure = BuildTestConnectFailure(
state="billing_credit_admission_refusal",
retry_action=None,
)
assert failure.model_dump(mode="json") == {
"state": "billing_credit_admission_refusal",
"workflow_run_id": None,
"workflow_run_block_id": None,
"task_id": None,
"browser_session_id": None,
"occupier_run_id": None,
"diagnostic": None,
"retry_action": None,
}
with pytest.raises(ValidationError):
BuildTestConnectFailure(state="billing_credit_admission_refusal")
with pytest.raises(ValidationError):
BuildTestConnectFailure(
state="billing_credit_admission_refusal",
browser_session_id="pbs_must_not_exist",
retry_action=None,
)
with pytest.raises(ValidationError):
BuildTestConnectFailure(state="already_closed", retry_action=None)
def test_billing_refusal_preserves_prior_run_evidence_and_repair_context() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
prior = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_prior",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
observed_evidence_summary="The prior run reached the form before its block failed.",
)
record_build_test_outcome(ctx, prior)
history_before = copy.deepcopy(ctx.recorded_build_test_outcome_history)
billing_result = {
"ok": False,
"data": {
"overall_status": "setup_failed",
"blocks": [],
"build_test_connect_failure": BuildTestConnectFailure(
state="billing_credit_admission_refusal",
retry_action=None,
).model_dump(mode="json"),
},
}
billing_outcome = recorded_outcome_from_run_blocks_result(billing_result)
assert billing_outcome is not None
assert billing_outcome.workflow_run_id is None
assert billing_outcome.executed_block_labels == []
record_build_test_outcome(ctx, billing_outcome)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.workflow_run_id == "wr_prior"
assert latest.observed_evidence_summary == prior.observed_evidence_summary
assert latest.connect_failure is not None
assert latest.connect_failure.state == "billing_credit_admission_refusal"
assert latest.connect_failure.retry_action is None
assert ctx.recorded_build_test_outcome_history == history_before
prompt = _recorded_build_test_outcome_prompt(ctx)
assert "No browser or run started" in prompt
assert "Billing" in prompt
def test_connect_failure_clears_when_a_later_real_run_records_recovery() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
connect_failure = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
workflow_run_id="wr_failed_connect",
structural_failure_identity="build_test_connect:cdp_connect_failed",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_failed_connect",
),
)
record_build_test_outcome(ctx, connect_failure)
record_build_test_outcome(ctx, passing_run("wr_recovered", ["sign_in_and_read"]))
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.workflow_run_id == "wr_recovered"
assert ctx.latest_recorded_build_test_outcome.connect_failure is None
assert ctx.recorded_build_test_outcome_history[-1]["connect_failure"] is None
def test_post_run_connect_failure_keeps_the_runs_repairable_block_failure() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
workflow_run_block_id="wrb_1",
block_label="collect_credentials",
failing_line=12,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
failed_operation=failed_operation,
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
workflow_run_id="wr_1",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.failed_operation == failed_operation
assert latest.verdict == "repairable_failure"
assert latest.reason_code == "runtime_block_failure"
assert latest.attempted_block_label == "collect_credentials"
assert latest.failed_block_labels == ["collect_credentials"]
assert latest.connect_failure is not None
assert latest.connect_failure.state == "cdp_connect_failed"
def test_acquisition_failure_without_a_run_id_still_leaves_the_earlier_failure_findable() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
block_label="collect_credentials",
failing_line=12,
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
# An acquisition failure that never reached a run names none, so it merges into the failure it
# follows and keeps that run's identity.
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.workflow_run_id == "wr_1"
assert latest.attempted_block_label == "collect_credentials"
assert latest.connect_failure is not None
assert ctx.recorded_build_test_outcome_history[-1]["workflow_run_id"] == "wr_1"
def test_a_post_run_connect_failure_keeps_the_block_failures_diagnostic_evidence() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
observed_evidence_summary="select_option timed out waiting for the state dropdown",
structural_failure_identity="locator-timeout-identity",
page_evidence_refs=["pe_1"],
evidence_refs=["ev_1"],
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
block_label="collect_credentials",
failing_line=12,
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
workflow_run_id="wr_1",
structural_failure_identity="build_test_connect:cdp_connect_failed",
observed_evidence_summary="Build-test browser acquisition stopped: cdp_connect_failed.",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
# The diagnosis the model repairs from, not just the label naming what failed.
assert "select_option timed out" in latest.observed_evidence_summary
assert latest.page_evidence_refs == ["pe_1"]
assert latest.evidence_refs == ["ev_1"]
assert latest.structural_failure_identity == "locator-timeout-identity"
# The acquisition fact is additional, not a replacement. It stays typed rather than folded into
# the summary, which the repair prompt clips to 160 characters.
assert latest.connect_failure is not None
assert latest.connect_failure.state == "cdp_connect_failed"
assert "cdp_connect_failed" not in latest.observed_evidence_summary
def test_a_block_that_died_on_a_plain_exception_is_preserved_like_a_browser_failure() -> None:
"""``failed_operation`` is populated only for browser-operation failures, so keying the merge on
it would leave an ordinary Python exception with none of its diagnosis."""
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
observed_evidence_summary="KeyError: 'state' while building the payload",
page_evidence_refs=["pe_1"],
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
workflow_run_id="wr_1",
observed_evidence_summary="Build-test browser acquisition stopped: cdp_connect_failed.",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.failed_operation is None
assert latest.verdict == "repairable_failure"
assert latest.reason_code == "runtime_block_failure"
assert latest.attempted_block_label == "collect_credentials"
assert "KeyError" in latest.observed_evidence_summary
assert latest.page_evidence_refs == ["pe_1"]
assert latest.connect_failure is not None
def test_merging_an_acquisition_failure_drops_the_stale_page_continuation() -> None:
"""The continuation binds the model to act on the prior run's page, in the browser this same
record says could not be acquired."""
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
page_path_failure=PostRunPagePathFailure(
kind="login",
workflow_run_id="wr_1",
current_url="https://example.test/login",
continuation_targets=(),
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
workflow_run_id="wr_1",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.page_path_failure is None
assert latest.reason_code == "runtime_block_failure"
assert latest.connect_failure is not None
def test_a_terminal_challenge_keeps_its_own_reason_over_an_earlier_block_failure() -> None:
"""A terminal challenge outranks a failed block to suppress code repair. Restating it as that
failure would send the model to edit code that was never the obstacle."""
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
block_label="collect_credentials",
failing_line=12,
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="terminal_challenge_blocker",
workflow_run_id="wr_1",
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.reason_code == "terminal_challenge_blocker"
assert latest.verdict == "not_authoritative"
assert latest.attempted_block_label == ""
assert ctx.recorded_build_test_outcome_history[-1]["reason_code"] == "terminal_challenge_blocker"
def test_a_block_failure_with_no_run_id_is_never_merged_into_an_acquisition_failure() -> None:
"""The merged record must carry a real run id: the prior-attempt change identity names the
earlier attempt by it."""
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
newest = ctx.recorded_build_test_outcome_history[-1]
assert not (newest["reason_code"] == "runtime_block_failure" and not newest["workflow_run_id"])
def test_a_different_runs_blocker_is_not_restated_as_the_earlier_runs_block_failure() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
block_label="collect_credentials",
failing_line=12,
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="terminal_challenge_blocker",
workflow_run_id="wr_2",
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.workflow_run_id == "wr_2"
assert latest.verdict == "not_authoritative"
assert latest.reason_code == "terminal_challenge_blocker"
assert latest.attempted_block_label == ""
assert latest.failed_block_labels == []
# The earlier run's failure is still carried: wr_2's own entry must not be stamped as a runtime
# block failure under wr_2's id.
assert latest.failed_operation is not None
assert latest.failed_operation.workflow_run_id == "wr_1"
assert ctx.recorded_build_test_outcome_history[-1]["reason_code"] == "terminal_challenge_blocker"
assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == ["wr_1", "wr_2"]
def test_later_run_with_its_own_failed_block_replaces_the_earlier_attribution() -> None:
ctx = _run_history_ctx(two_page_login_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_1",
attempted_block_label="collect_credentials",
failed_block_labels=["collect_credentials"],
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_1",
block_label="collect_credentials",
failing_line=12,
),
),
)
second_failure = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_2",
block_label="sign_in_and_read",
failing_line=3,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_2",
attempted_block_label="sign_in_and_read",
failed_block_labels=["sign_in_and_read"],
failed_operation=second_failure,
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.failed_operation == second_failure
assert latest.attempted_block_label == "sign_in_and_read"
assert latest.failed_block_labels == ["sign_in_and_read"]
def _failed_run_result_with_categories(categories: list[dict]) -> dict:
return {
"ok": False,
"data": {
"workflow_run_id": "wr_failed",
"overall_status": "failed",
"failure_categories": categories,
"blocks": [
{"label": "search", "status": "failed", "failure_reason": "Timeout waiting for #results"},
],
},
}
def test_structural_identity_ignores_keyword_only_anti_bot_categories() -> None:
element_only = recorded_outcome_from_run_blocks_result(
_failed_run_result_with_categories([{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8}])
)
with_keyword_stamp = recorded_outcome_from_run_blocks_result(
_failed_run_result_with_categories(
[
{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8},
{"category": "ANTI_BOT_DETECTION", "confidence_float": 0.7, "evidence_source": "keyword_only"},
]
)
)
with_carrier = recorded_outcome_from_run_blocks_result(
_failed_run_result_with_categories(
[
{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8},
{"category": "ANTI_BOT_DETECTION", "confidence_float": 0.9, "evidence_source": "challenge_state"},
]
)
)
assert element_only is not None and with_keyword_stamp is not None and with_carrier is not None
assert with_keyword_stamp.structural_failure_identity == element_only.structural_failure_identity
assert with_carrier.structural_failure_identity != element_only.structural_failure_identity
def test_structural_key_ignores_display_prose_and_workflow_run_id() -> None:
first = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_first",
block_labels=["search_records"],
structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
page_evidence_refs=["origin_present", "form:search"],
observed_evidence_summary="Timeout waiting for #results.",
display_text="The page did not show results.",
)
second = first.model_copy(
update={
"workflow_run_id": "wr_second",
"observed_evidence_summary": "Different explanation with the same structural observation.",
"display_text": "Another user-facing sentence.",
}
)
assert first.structural_key == second.structural_key
def test_structural_key_ignores_authored_signature_but_retains_it() -> None:
first = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="no_meaningful_output",
structural_failure_identity="completion:typed-outcome",
page_evidence_refs=["origin:https://example.com", "result:#results rows=0"],
authored_structure_signature="authored:first",
)
second = first.model_copy(update={"authored_structure_signature": "authored:second"})
assert first.structural_key is not None
assert first.structural_key == second.structural_key
assert first.authored_structure_signature == "authored:first"
assert second.authored_structure_signature == "authored:second"
def test_scout_act_observe_hollow_outcome_is_structural_and_privacy_bounded() -> None:
outcome = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#search",
current_url="https://example.com/customers/acme-inc/results?token=secret",
source_url="https://example.com/accounts/claim-123/search?name=customer",
page_evidence={
"page_title": "Private Account Search",
"forms": [],
"navigation_targets": [],
"result_containers": [],
"clickable_controls": [],
"visible_text": "Customer name should not persist",
"body": "<main></main>",
"schema_empty_page": True,
},
recapture_attempted=True,
recapture_result="timeout",
)
key_payload = outcome.structural_key_payload
assert outcome.reason_code == "scout_act_observe_hollow_after_interaction"
assert outcome.structural_key is not None
assert outcome.is_authoritative is True
assert "recapture_attempted:true" in outcome.page_evidence_refs
assert "recapture_result:timeout" in outcome.page_evidence_refs
for sensitive in ("token=secret", "name=customer", "acme-inc", "claim-123", "Customer name", "Private Account"):
assert sensitive not in str(key_payload)
def test_hollow_outcome_carries_observed_value_excerpt_off_the_structural_key() -> None:
def _outcome(visible_text: str) -> RecordedBuildTestOutcome:
return recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#submit",
current_url="https://example.com/confirmation",
source_url="https://example.com/form",
page_evidence={
"page_title": "Confirmation",
"forms": [],
"visible_text_excerpt": visible_text,
},
recapture_attempted=True,
recapture_result="hollow",
)
confirmation = _outcome("Request WTR-1842-DEMO for account 100245 confirmed")
other = _outcome("A completely different confirmation body")
assert "WTR-1842-DEMO" in confirmation.observed_page_value_excerpt
assert "100245" in confirmation.observed_page_value_excerpt
assert confirmation.structural_key == other.structural_key
assert "WTR-1842-DEMO" not in str(confirmation.structural_key_payload)
def test_hollow_outcome_reason_code_unchanged_with_value_carrying_relation() -> None:
outcome = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#view-statement",
current_url="https://portal.example.com/statement",
source_url="https://portal.example.com/statement",
page_evidence={
"page_title": "Statement",
"forms": [],
"key_value_relations": [
{
"key_text": "March 2026 statement",
"value_text": "Amount due: $3,927.75",
"container_selector": "#result",
"value_child_index": 1,
"direct_child_count": 3,
"visible": True,
"value_visible": True,
}
],
"key_value_relations_truncated": False,
},
recapture_attempted=True,
recapture_result="hollow",
)
assert outcome.reason_code == "scout_act_observe_hollow_after_interaction"
assert outcome.is_authoritative is True
assert "$3,927.75" not in str(outcome.structural_key_payload)
def test_hollow_outcome_value_excerpt_falls_back_to_legacy_text_keys() -> None:
from_visible_text = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#submit",
current_url="https://example.com/confirmation",
source_url=None,
page_evidence={"visible_text": "Legacy visible text body"},
recapture_attempted=False,
recapture_result="not_attempted_no_budget",
)
from_body_text = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#submit",
current_url="https://example.com/confirmation",
source_url=None,
page_evidence={"bodyText": "Legacy body text body"},
recapture_attempted=False,
recapture_result="not_attempted_no_budget",
)
assert from_visible_text.observed_page_value_excerpt == "Legacy visible text body"
assert from_body_text.observed_page_value_excerpt == "Legacy body text body"
def test_hollow_outcome_value_excerpt_is_bounded_and_key_independent() -> None:
long_text = "X" * 5000
outcome = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#submit",
current_url="https://example.com/confirmation",
source_url=None,
page_evidence={"visible_text_excerpt": long_text},
recapture_attempted=True,
recapture_result="hollow",
)
baseline = recorded_outcome_from_scout_act_observe_hollow(
interaction_tool="click",
selector="#submit",
current_url="https://example.com/confirmation",
source_url=None,
page_evidence={"visible_text_excerpt": ""},
recapture_attempted=True,
recapture_result="hollow",
)
assert 0 < len(outcome.observed_page_value_excerpt) <= 700
assert baseline.observed_page_value_excerpt == ""
assert outcome.structural_key == baseline.structural_key
def test_author_time_reject_carries_value_excerpt_off_the_convergence_key() -> None:
carried = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={"version": "metadata_reject_output_contract:v1", "signature": "abc"},
observed_page_value_excerpt=" Request WTR-1842-DEMO for account 100245 " + "detail " * 400,
missing_requested_output_facts=[{"output_path": "output.confirmation_number"}],
)
baseline = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={"version": "metadata_reject_output_contract:v1", "signature": "abc"},
missing_requested_output_facts=[{"output_path": "output.confirmation_number"}],
)
assert "WTR-1842-DEMO" in carried.observed_page_value_excerpt
assert 0 < len(carried.observed_page_value_excerpt) <= 700
assert "WTR-1842-DEMO" not in str(carried.structural_key_payload)
assert carried.structural_key == baseline.structural_key
def test_observed_value_extraction_scaffold_binds_output_paths() -> None:
scaffold = observed_value_extraction_scaffold_lines(
"Request WTR-1842-DEMO for account 100245",
["output.confirmation_number", "output.account_number", "output.confirmation_number"],
)
assert scaffold[0].startswith("OBSERVED PAGE VALUES CONTRACT")
assert "observed_values: Request WTR-1842-DEMO for account 100245" in scaffold
assert "bind_output_paths:" in scaffold
assert "- output.confirmation_number: <observed value>" in scaffold
assert "- output.account_number: <observed value>" in scaffold
assert sum(1 for line in scaffold if line.startswith("- output.confirmation_number")) == 1
def test_observed_value_extraction_scaffold_without_paths_surfaces_values_only() -> None:
assert observed_value_extraction_scaffold_lines("Confirmed WTR-1842-DEMO", []) == [
"observed_page_values: Confirmed WTR-1842-DEMO"
]
assert observed_value_extraction_scaffold_lines(" ", ["output.x"]) == []
def test_prose_or_label_only_typed_outcome_is_not_authoritative() -> None:
outcome = RecordedBuildTestOutcome(
phase="author_time_reject",
attempted_tool="update_workflow",
attempted_block_label="search_records",
verdict="authoring_rejected",
reason_code="code_safety_reject",
block_labels=["search_records"],
observed_evidence_summary="This sounds like the same failure.",
display_text="Use the prior failure reason.",
)
assert outcome.structural_key is None
assert outcome.is_authoritative is False
def test_record_none_clears_stale_latest_outcome() -> None:
ctx = SimpleNamespace(
latest_recorded_build_test_outcome=None,
recorded_build_test_outcome_history=[],
recorded_persisted_block_run_workflow_run_id=None,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
structural_failure_identity="typed-runtime-identity",
),
)
assert ctx.latest_recorded_build_test_outcome is not None
record_build_test_outcome(ctx, None)
assert ctx.latest_recorded_build_test_outcome is None
def test_record_authoritative_persisted_run_latches_run_backed_evidence() -> None:
ctx = SimpleNamespace(
latest_recorded_build_test_outcome=None,
recorded_build_test_outcome_history=[],
recorded_persisted_block_run_workflow_run_id=None,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_recorded",
structural_failure_identity="runtime:failed",
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="author_time_reject",
attempted_tool="update_workflow",
verdict="authoring_rejected",
reason_code="metadata_reject",
structural_failure_identity="metadata:missing",
),
)
assert ctx.recorded_persisted_block_run_workflow_run_id == "wr_recorded"
assert ctx.recorded_persisted_block_run_workflow_run_id == "wr_recorded"
def test_non_authoritative_persisted_run_does_not_latch_run_backed_evidence() -> None:
ctx = SimpleNamespace(
latest_recorded_build_test_outcome=None,
recorded_build_test_outcome_history=[],
recorded_persisted_block_run_workflow_run_id=None,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_hollow",
),
)
assert ctx.recorded_persisted_block_run_workflow_run_id is None
assert ctx.recorded_persisted_block_run_workflow_run_id is None
def test_authored_structure_signature_is_stable_and_excludes_raw_code_or_prose() -> None:
workflow_yaml = """
title: Registry lookup
workflow_definition:
parameters:
- parameter_type: workflow
workflow_parameter_type: string
key: provider_query
blocks:
- block_type: code
label: search_registry
parameter_keys:
- provider_query
code: |
await page.goto("https://example.com/search")
return {"records": [{"npi": "123"}]}
"""
metadata = [
{
"block_label": "search_registry",
"declared_goal": "Find the exact provider row from the page prose.",
"claimed_outcomes": [
{
"id": "claim:provider",
"text": "The provider was found in the directory.",
"goal_value_paths": ["records[].npi"],
"extraction_schema": {
"type": "object",
"properties": {"records": {"type": "array", "items": {"type": "object"}}},
},
}
],
}
]
prose_changed_metadata = [
{
**metadata[0],
"declared_goal": "Different page prose for the same structure.",
"claimed_outcomes": [{**metadata[0]["claimed_outcomes"][0], "text": "Different prose."}],
}
]
signature = authored_structure_signature_from_workflow(workflow_yaml, metadata)
same_structure = authored_structure_signature_from_workflow(workflow_yaml, prose_changed_metadata)
dumped = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="no_meaningful_output",
structural_failure_identity="completion:typed",
authored_structure_signature=signature,
).model_dump(mode="json")
assert signature is not None
assert signature == same_structure
assert "page.goto" not in str(dumped)
assert "Find the exact provider" not in str(dumped)
def test_authored_block_signatures_ignore_cosmetic_block_fields() -> None:
base = """
title: Registry lookup
workflow_definition:
blocks:
- block_type: code
label: search_registry
parameter_keys:
- provider_query
code: |
return {"records": [{"npi": "123"}]}
"""
described = base.replace(
" label: search_registry\n",
" label: search_registry\n description: look up the provider by name\n",
)
continue_on_failure = base.replace(
" label: search_registry\n",
" label: search_registry\n continue_on_failure: true\n",
)
renamed = base.replace("search_registry", "lookup_registry")
baseline = authored_block_signatures_from_workflow(base, None)
assert authored_block_signatures_from_workflow(described, None) == baseline
assert authored_block_signatures_from_workflow(continue_on_failure, None) == baseline
renamed_signatures = authored_block_signatures_from_workflow(renamed, None)
assert set(renamed_signatures) == {"lookup_registry"}
assert renamed_signatures["lookup_registry"] == baseline["search_registry"]
def test_authored_structure_signature_changes_on_code_parameter_or_output_structure() -> None:
base = """
title: Registry lookup
workflow_definition:
blocks:
- block_type: code
label: search_registry
parameter_keys:
- provider_query
code: |
return {"records": [{"npi": "123"}]}
"""
changed_code = base.replace('"123"', '"456"')
changed_parameter = base.replace("provider_query", "provider_name")
metadata = [{"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].npi"]}]}]
changed_metadata = [
{"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].license"]}]}
]
signature = authored_structure_signature_from_workflow(base, metadata)
assert signature is not None
assert authored_structure_signature_from_workflow(changed_code, metadata) != signature
assert authored_structure_signature_from_workflow(changed_parameter, metadata) != signature
assert authored_structure_signature_from_workflow(base, changed_metadata) != signature
def test_no_meaningful_output_keeps_authoritative_unsatisfied_criteria_identity() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_partial",
"overall_status": "completed",
"blocks": [
{
"label": "lookup_provider_and_extract_credentials",
"status": "completed",
"extracted_data": {"npi": "", "evidence_text": "address and statuses appear in page text"},
}
],
},
}
verification = CompletionVerificationResult(
status="evaluated",
criterion_ids=["npi", "locations", "statuses", "overall"],
verdicts=[
CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence", output_path="npi"),
CriterionVerdict(
criterion_id="locations",
state="unsatisfied",
reason_code="no_evidence",
output_path="locations",
),
CriterionVerdict(
criterion_id="statuses",
state="unsatisfied",
reason_code="no_evidence",
output_path="credentialing_statuses",
),
CriterionVerdict(
criterion_id="overall",
state="unsatisfied",
reason_code="no_evidence",
output_path="overall_credentialing_status",
),
],
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="no_meaningful_output",
workflow_run_id="wr_partial",
),
completion_verification=verification,
authored_structure_signature="authored:partial-location-only",
)
assert outcome is not None
assert outcome.phase == "persisted_block_run"
assert outcome.reason_code == "no_meaningful_output"
assert outcome.workflow_run_id == "wr_partial"
assert outcome.is_authoritative is True
assert outcome.structural_key is not None
assert outcome.authored_structure_signature == "authored:partial-location-only"
assert outcome.key_provenance["structural_failure_identity"] == "CompletionVerificationResult verdict structure"
assert outcome.missing_requested_output_facts == [
{
"criterion_id": "statuses",
"output_path": "credentialing_statuses",
"output_root": "credentialing_statuses",
"reason_code": "no_evidence",
"value_status": "no_typed_value",
"partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
},
{
"criterion_id": "locations",
"output_path": "locations",
"output_root": "locations",
"reason_code": "no_evidence",
"value_status": "no_typed_value",
"partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
},
{
"criterion_id": "npi",
"output_path": "npi",
"output_root": "npi",
"reason_code": "no_evidence",
"value_status": "empty_typed_value",
},
{
"criterion_id": "overall",
"output_path": "overall_credentialing_status",
"output_root": "overall_credentialing_status",
"reason_code": "no_evidence",
"value_status": "no_typed_value",
"partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
},
]
payload_text = str(outcome.structural_key_payload)
assert "evidence_text" not in payload_text
assert "address and statuses" not in payload_text
def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_structural",
"blocks": [
{
"label": "publish_result",
"status": "completed",
"extracted_data": {"document_name": "Resale Demand Package"},
}
],
},
}
verification = CompletionVerificationResult(
status="evaluated",
criterion_ids=["c0"],
verdicts=[
CriterionVerdict(
criterion_id="c0",
state="unsatisfied",
reason_code="structurally_abstained",
output_path="output.document_name",
)
],
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_structural"),
completion_verification=verification,
)
assert outcome is not None
assert outcome.verdict == "not_authoritative"
assert outcome.reason_code == "run_completed_unevaluated"
assert outcome.is_authoritative is False
def test_completed_run_with_registered_outputs_is_not_classified_as_failed_run() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_completed",
"overall_status": "completed",
"blocks": [{"label": "collect_top_entry", "status": "completed"}],
"registered_output_parameter_values": [
{"output_parameter_id": "op_1", "value": {"output": {"top_entry": "First listed entry"}}}
],
},
}
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_completed"),
)
assert outcome is not None
assert outcome.reason_code != "failed_run"
assert outcome.reason_code == "run_completed_unevaluated"
def test_failed_run_classification_preserved_for_not_ok_run() -> None:
result = {
"ok": False,
"data": {
"workflow_run_id": "wr_not_ok",
"overall_status": "failed",
"failure_type": "block_failure",
"blocks": [{"label": "collect_top_entry", "status": "completed"}],
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.reason_code == "runtime_block_failure"
assert outcome.verdict == "repairable_failure"
def test_recorded_failed_run_preserves_runtime_block_identity() -> None:
for run_ok in (False, True):
result = {
"ok": run_ok,
"data": {
"workflow_run_id": "wr_not_ok",
"overall_status": "failed",
"failure_type": "block_failure",
"blocks": [
{
"label": "collect_top_entry",
"status": "failed",
"failure_reason": (
"TimeoutError: Locator.click: Timeout 30000ms exceeded while waiting for "
'locator("#cta-stale-regression-probe")'
),
}
],
},
}
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
workflow_run_id="wr_not_ok",
),
)
assert outcome is not None
assert outcome.reason_code == "runtime_block_failure"
assert outcome.verdict == "repairable_failure"
assert outcome.attempted_block_label == "collect_top_entry"
def test_failed_run_classification_preserved_for_failed_block() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_failed_block",
"overall_status": "completed",
"failure_type": "block_failure",
"blocks": [{"label": "collect_top_entry", "status": "failed", "failure_reason": "selector not found"}],
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.reason_code == "runtime_block_failure"
def test_judge_evaluated_non_satisfaction_keeps_failure_classification() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_judged",
"overall_status": "completed",
"blocks": [{"label": "collect_top_entry", "status": "completed"}],
},
}
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="no_meaningful_output",
workflow_run_id="wr_judged",
),
)
assert outcome is not None
assert outcome.reason_code == "no_meaningful_output"
def test_no_meaningful_output_does_not_mark_presence_only_abstention_as_missing_output() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_top_post",
"overall_status": "completed",
"blocks": [
{
"label": "extract_top_hn_post",
"status": "completed",
"extracted_data": {"output": {"top_post": "Claude Sonnet 5"}},
}
],
},
}
verification = CompletionVerificationResult(
status="evaluated",
criterion_ids=["top_post"],
verdicts=[
CriterionVerdict(
criterion_id="top_post",
state="unsatisfied",
reason_code="structurally_abstained",
output_path="output.top_post",
grounding_mode="missing",
evidence_ref="block_outputs:extract_top_hn_post.output.top_post",
)
],
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="no_meaningful_output",
workflow_run_id="wr_top_post",
),
completion_verification=verification,
)
assert outcome is not None
assert outcome.reason_code == "no_meaningful_output"
assert outcome.is_authoritative is True
assert outcome.missing_requested_output_facts == []
def test_no_meaningful_output_keeps_missing_fact_for_absent_requested_output() -> None:
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_no_top_post",
"overall_status": "completed",
"blocks": [
{
"label": "extract_top_hn_post",
"status": "completed",
"extracted_data": {"output": {}},
}
],
},
}
verification = CompletionVerificationResult(
status="evaluated",
criterion_ids=["top_post"],
verdicts=[
CriterionVerdict(
criterion_id="top_post",
state="unsatisfied",
reason_code="no_evidence",
output_path="output.top_post",
grounding_mode="missing",
)
],
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="no_meaningful_output",
workflow_run_id="wr_no_top_post",
),
completion_verification=verification,
)
assert outcome is not None
assert outcome.reason_code == "no_meaningful_output"
assert outcome.missing_requested_output_facts == [
{
"criterion_id": "top_post",
"output_path": "output.top_post",
"output_root": "output",
"reason_code": "no_evidence",
"value_status": "no_typed_value",
"grounding_mode": "missing",
}
]
assert outcome.structural_key_payload is not None
assert "output.top_post" in str(outcome.structural_key_payload)
def test_synthesized_parameter_repair_context_produces_structural_recorded_outcome() -> None:
context = CodeAuthoringRepairContext(
block_label="search_records",
reason_code="synthesized_parameter_binding_ambiguous",
unresolved_names=["confirmation_number", "row_text"],
parameter_keys=["confirmation_number"],
available_parameter_keys=["confirmation_number"],
binding_candidates=["confirmation_number", "row_text"],
)
outcome = recorded_outcome_from_authoring_repair_context(context)
assert outcome.phase == "author_time_reject"
assert outcome.reason_code == "synthesized_parameter_binding_ambiguous"
assert outcome.structural_key is not None
assert (
outcome.structural_key
== recorded_outcome_from_authoring_repair_context(
context.model_copy(update={"unresolved_names": ["row_text", "confirmation_number"]})
).structural_key
)
def test_authoring_repair_context_missing_output_fields_affect_structural_outcome() -> None:
context = CodeAuthoringRepairContext(
block_label="read_resource_table",
reason_code="runtime_missing_output_dependency",
missing_output_key="create_resource_output",
available_output_keys=["search_output"],
current_block_parameter_keys=["create_resource_output"],
output_dependency_failure_class="missing_prior_block_output",
)
outcome = recorded_outcome_from_authoring_repair_context(context)
assert outcome.reason_code == "runtime_missing_output_dependency"
assert (
outcome.structural_key
!= recorded_outcome_from_authoring_repair_context(
context.model_copy(update={"missing_output_key": "verify_resource_output"})
).structural_key
)
assert (
outcome.structural_key
!= recorded_outcome_from_authoring_repair_context(
context.model_copy(update={"available_output_keys": ["search_output", "verify_resource_output"]})
).structural_key
)
def test_author_time_reject_structural_payloads_make_distinct_keys() -> None:
first = recorded_outcome_from_author_time_reject(
reason_code="schema_incompatibility",
structural_payload={
"block_label": "extract_record",
"incompatible_paths": ["records[].expiration_date"],
"known_output_paths": ["records[].name"],
},
)
second = recorded_outcome_from_author_time_reject(
reason_code="schema_incompatibility",
structural_payload={
"block_label": "extract_record",
"incompatible_paths": ["records[].license_number"],
"known_output_paths": ["records[].name"],
},
)
assert first.structural_key is not None
assert second.structural_key is not None
assert first.structural_key != second.structural_key
def test_metadata_reject_preserves_missing_requested_output_facts() -> None:
outcome = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "recorded_outcome_missing_output_coverage",
"missing_output_roots": ["address", "credentialing_status"],
"block_labels": ["lookup_provider_and_extract_credentials"],
},
missing_requested_output_facts=[
{
"output_path": "address",
"output_root": "address",
"reason_code": "recorded_outcome_missing_output_coverage",
"value_status": "no_typed_value",
},
{
"output_path": "credentialing_status",
"output_root": "credentialing_status",
"reason_code": "recorded_outcome_missing_output_coverage",
"value_status": "no_typed_value",
},
],
)
assert outcome.reason_code == "metadata_reject"
assert outcome.is_authoritative is True
assert outcome.missing_requested_output_facts == [
{
"output_path": "address",
"output_root": "address",
"reason_code": "recorded_outcome_missing_output_coverage",
"value_status": "no_typed_value",
},
{
"output_path": "credentialing_status",
"output_root": "credentialing_status",
"reason_code": "recorded_outcome_missing_output_coverage",
"value_status": "no_typed_value",
},
]
assert outcome.structural_key_payload is not None
assert "address" in str(outcome.structural_key_payload)
def test_author_time_reject_without_structural_payload_is_not_authoritative() -> None:
outcome = recorded_outcome_from_author_time_reject(
reason_code="code_safety_reject",
observed_evidence_summary="Rewrite the code without unsafe behavior.",
)
assert outcome.is_authoritative is False
assert outcome.structural_key is None
def test_metadata_reject_key_uses_typed_fields_not_wording() -> None:
first = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "metadata_reject",
"offending_labels": ["search_registry"],
"required_fields": ["claimed_outcomes", "completion_criteria"],
"missing_fields_by_label": {"search_registry": ["claimed_outcomes"]},
"violation_categories": ["missing_required_list"],
},
observed_evidence_summary="Metadata requires non-empty claimed_outcomes.",
)
same_structure = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "metadata_reject",
"offending_labels": ["search_registry"],
"required_fields": ["claimed_outcomes", "completion_criteria"],
"missing_fields_by_label": {"search_registry": ["claimed_outcomes"]},
"violation_categories": ["missing_required_list"],
},
observed_evidence_summary="Different wording for the same typed metadata failure.",
)
changed_label = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "metadata_reject",
"offending_labels": ["extract_registry"],
"required_fields": ["claimed_outcomes", "completion_criteria"],
"missing_fields_by_label": {"extract_registry": ["claimed_outcomes"]},
"violation_categories": ["missing_required_list"],
},
)
changed_required_field = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "metadata_reject",
"offending_labels": ["search_registry"],
"required_fields": ["terminal_verifier_expectations"],
"missing_fields_by_label": {"search_registry": ["terminal_verifier_expectations"]},
"violation_categories": ["missing_required_list"],
},
)
changed_missing_field = recorded_outcome_from_author_time_reject(
reason_code="metadata_reject",
structural_payload={
"reason_code": "metadata_reject",
"offending_labels": ["search_registry"],
"required_fields": ["claimed_outcomes", "completion_criteria"],
"missing_fields_by_label": {"search_registry": ["completion_criteria"]},
"violation_categories": ["missing_required_list"],
},
)
assert first.structural_key == same_structure.structural_key
assert changed_label.structural_key != first.structural_key
assert changed_required_field.structural_key != first.structural_key
assert changed_missing_field.structural_key != first.structural_key
def test_output_policy_reject_key_uses_stable_trace_payload() -> None:
payload = {
"surface": "tool_body",
"tool_name": "update_workflow",
"allowed": False,
"output_kind": "workflow_update_proposal",
"reason_codes": ["raw_secret_leak"],
}
first = recorded_outcome_from_author_time_reject(
reason_code="output_policy_reject",
structural_payload=payload,
observed_evidence_summary="Output policy blocked this Copilot output before persistence.",
)
same_structure = recorded_outcome_from_author_time_reject(
reason_code="output_policy_reject",
structural_payload=dict(payload),
observed_evidence_summary="Different wording for the same output-policy reject.",
)
changed_reason = recorded_outcome_from_author_time_reject(
reason_code="output_policy_reject",
structural_payload={**payload, "reason_codes": ["unapproved_credential_reference"]},
)
changed_surface = recorded_outcome_from_author_time_reject(
reason_code="output_policy_reject",
structural_payload={**payload, "surface": "final_response"},
)
assert first.reason_code == "output_policy_reject"
assert first.is_authoritative is True
assert first.structural_key == same_structure.structural_key
assert changed_reason.structural_key != first.structural_key
assert changed_surface.structural_key != first.structural_key
def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:
result = {
"ok": False,
"error": "Timeout waiting for results.",
"data": {
"workflow_run_id": "wr_failed",
"overall_status": "failed",
"blocks": [
{
"label": "search_records",
"status": "failed",
"failure_reason": "Timeout waiting for results.",
}
],
},
}
page_evidence = {
"workflow_run_id": "wr_failed",
"observed_after_workflow_run": True,
"source_tool": "inspect_page_for_composition",
"current_url": "https://example.test/search?secret=redacted",
"page_title": "Search",
"forms": [{"fields": [{"label": "Search", "selector": "#search"}]}],
"result_containers": [{"selector": "#results", "text_excerpt": "No matching records"}],
}
outcome = recorded_outcome_from_run_blocks_result(result, page_evidence=page_evidence)
assert outcome.workflow_run_id == "wr_failed"
assert outcome.phase == "persisted_block_run"
assert outcome.reason_code == "runtime_block_failure"
assert outcome.structural_key is not None
assert "form:Search #search" in outcome.page_evidence_refs
assert "result:#results rows=unknown" in outcome.page_evidence_refs
@pytest.mark.asyncio
async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _locator_packet_ctx()
ctx.proposal_workflow_run_id = "wr_failed_complete_packet"
now = datetime(2026, 8, 28, tzinfo=UTC)
output_parameter = OutputParameter(
output_parameter_id="out_records",
workflow_id="wf_run_snapshot",
key="records",
description="Collected records",
created_at=now,
modified_at=now,
)
run_workflow = SimpleNamespace(
workflow_id="wf_run_snapshot",
workflow_permanent_id=ctx.workflow_permanent_id,
version=1,
created_by=None,
modified_at=datetime(2026, 4, 21, 12, 0),
organization_id=ctx.organization_id,
workflow_definition=SimpleNamespace(
parameters=[output_parameter],
blocks=[
CodeBlock(
label="collect_records",
output_parameter=output_parameter,
code='return {"records": [{"name": "bounded result"}]}',
)
],
),
)
run = SimpleNamespace(
workflow_run_id="wr_failed_complete_packet",
workflow_id="wf_run_snapshot",
workflow_permanent_id=ctx.workflow_permanent_id,
browser_session_id="pbs_failed_complete_packet",
status="failed",
failure_reason="Code block failed.",
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
workflow_definition_sha256=None,
)
block = SimpleNamespace(
workflow_run_block_id="wrb_failed_complete_packet",
parent_workflow_run_block_id=None,
task_id=None,
label="collect_records",
block_type=SimpleNamespace(name="CODE"),
status="failed",
failure_reason="NameError at generated line 7",
error_codes=["user_code_error"],
output=[{"name": "bounded result"}],
final_url="https://example.test/results",
)
artifact = make_stub_html_artifact("art_failed_terminal", ArtifactType.HTML_ACTION)
html = (
b"<html><body><main><h1>Results</h1><table><tbody>"
b"<tr><td>One bounded result</td></tr></tbody></table></main></body></html>"
)
fake_app = SimpleNamespace(
DATABASE=SimpleNamespace(
workflow_runs=SimpleNamespace(
get_workflow_run=AsyncMock(return_value=run),
get_workflow_run_output_parameters=AsyncMock(
return_value=[
SimpleNamespace(output_parameter_id="out_snapshot", value=[{"name": "bounded result"}])
]
),
),
observer=SimpleNamespace(get_workflow_run_blocks=AsyncMock(return_value=[block])),
workflows=SimpleNamespace(get_workflow_for_workflow_run=AsyncMock(return_value=run_workflow)),
artifacts=SimpleNamespace(get_artifacts_for_run=AsyncMock(return_value=[artifact])),
),
AGENT_FUNCTION=SimpleNamespace(should_dispatch_copilot_block_run_to_worker=AsyncMock(return_value=True)),
ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=AsyncMock(return_value=html)),
)
monkeypatch.setattr(run_execution_module, "app", fake_app)
async def attach_trace(
blocks: object, results: list[dict[str, object]], organization_id: str, include_completed: bool = False
) -> None:
results[0]["action_trace"] = [{"code_line": 7, "action": "evaluate"}]
monkeypatch.setattr(run_execution_module, "_attach_action_traces", attach_trace)
monkeypatch.setattr(run_execution_module, "_attach_failed_block_screenshots", AsyncMock())
produced_outcomes: list[RecordedBuildTestOutcome] = []
transported_outcomes: list[RecordedBuildTestOutcome | None] = []
real_producer = run_execution_module.recorded_outcome_from_run_blocks_result
real_packet_builder = run_execution_module.build_test_evidence_packet
def capture_producer(*args: object, **kwargs: object) -> RecordedBuildTestOutcome:
produced = real_producer(*args, **kwargs)
produced_outcomes.append(produced)
return produced
def capture_packet(
context: CopilotContext,
result: object,
*,
recorded_outcome: RecordedBuildTestOutcome | None = None,
) -> BuildTestEvidencePacket:
transported_outcomes.append(recorded_outcome)
return real_packet_builder(context, result, recorded_outcome=recorded_outcome)
monkeypatch.setattr(run_execution_module, "recorded_outcome_from_run_blocks_result", capture_producer)
monkeypatch.setattr(run_execution_module, "build_test_evidence_packet", capture_packet)
hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr_failed_complete_packet")
system_input = str(
_build_dynamic_system_prompt("", CopilotConfig(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER))(
SimpleNamespace(context=ctx), None
)
)
ordinary_input = (
system_input
+ "\n"
+ _build_user_context(
workflow_yaml=ctx.workflow_yaml,
chat_history_text="",
global_llm_context="",
debug_run_info_text=_prior_run_debug_text(hydrated),
user_message="Repair the recorded run.",
)
)
assert hydrated is not None
assert len(produced_outcomes) == 1
assert produced_outcomes[0].workflow_run_id == "wr_failed_complete_packet"
assert transported_outcomes == [produced_outcomes[0]]
assert ctx.latest_recorded_build_test_outcome is None
assert hydrated["registered_outputs"] == [
{
"label": "collect_records",
"status": "failed",
"output": [{"name": "bounded result"}],
"value_complete": True,
}
]
assert hydrated["run"]["execution_source"] == {
"source_kind": "staged",
"workflow_id": "wf_run_snapshot",
"block_code_sha256": {
"collect_records": hashlib.sha256(b'return {"records": [{"name": "bounded result"}]}').hexdigest()
},
}
assert '"status": "failed"' in ordinary_input
assert '"workflow_run_id": "wr_failed_complete_packet"' in ordinary_input
assert '"output_parameter_id": "out_records"' in ordinary_input
assert '"output_parameter_key": "records"' in ordinary_input
assert '"block_label": "collect_records"' in ordinary_input
assert '"block_status": "failed"' in ordinary_input
assert '"error_codes"' in ordinary_input
assert '"user_code_error"' in ordinary_input
assert '"failing_line": 7' in ordinary_input
assert '"name": "bounded result"' in ordinary_input
assert '"page_state"' in ordinary_input
assert "One bounded result" in ordinary_input
assert "RECORDED BUILD-TEST OUTCOME" not in system_input
assert "repairable_failure" not in system_input
assert "wr_failed_complete_packet" not in system_input
nonassociated_ctx = _locator_packet_ctx()
nonassociated_ctx.proposal_workflow_run_id = "wr_different_proposal"
nonassociated = await run_execution_module.hydrate_prior_run_packet(
nonassociated_ctx,
workflow_run_id="wr_failed_complete_packet",
)
assert nonassociated is not None
assert nonassociated["run"]["execution_source"]["source_kind"] == "run_version"
assert nonassociated["run"]["execution_source"]["workflow_id"] == "wf_run_snapshot"
def test_unavailable_registered_output_rows_do_not_become_false_missing_output_facts() -> None:
ctx = _locator_packet_ctx()
result = {
"ok": False,
"data": {
"workflow_run_id": "wr_registry_unavailable",
"overall_status": "failed",
"failure_type": "user_code_error",
"blocks": [
{
"label": "collect_records",
"status": "failed",
"failure_reason": "failed",
"error_codes": ["user_code_error"],
}
],
"requested_output_parameter_definitions": [
{
"workflow_run_id": "wr_registry_unavailable",
"output_parameter_id": "out_records",
"output_parameter_key": "records",
}
],
"registered_output_values_omission": "persisted registered output values were unavailable",
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.missing_requested_output_facts == []
record_build_test_outcome(ctx, outcome)
packet = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
assert any("persisted registered output values were unavailable" in notice for notice in packet.omission_notices)
def _code_exception_result(failure_reason: str) -> dict[str, object]:
return {
"ok": False,
"data": {
"workflow_run_id": "wr_code_exception",
"overall_status": "failed",
"blocks": [
{
"label": "count_items",
"block_type": "CODE",
"status": "failed",
"failure_reason": failure_reason,
"error_codes": ["user_code_error"],
}
],
},
}
def test_a_classified_code_exception_keys_on_its_class_instead_of_the_runner_reason() -> None:
name_error = _code_exception_result("CodeBlock failed with NameError at line 48: name 'count' is not defined.")
unbound = _code_exception_result(
"CodeBlock failed with UnboundLocalError at line 47: cannot access local variable 'count' where it is not "
"associated with a value."
)
unclassified = [recorded_outcome_from_run_blocks_result(result) for result in (name_error, unbound)]
classified = [
recorded_outcome_from_run_blocks_result(result, runtime_failure_class="wrapper_scope_name_resolution")
for result in (name_error, unbound)
]
assert all(outcome is not None and outcome.structural_key for outcome in [*unclassified, *classified])
assert unclassified[0].structural_key != unclassified[1].structural_key
assert classified[0].structural_key == classified[1].structural_key
assert classified[0].structural_key != unclassified[0].structural_key
def test_runtime_block_failure_outcome_keys_playwright_hidden_locator_structure() -> None:
table_result = {
"ok": False,
"data": {
"workflow_run_id": "wr_failed",
"overall_status": "failed",
"blocks": [
{
"label": "extract_records",
"status": "failed",
"failure_reason": (
"Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "
'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table").first '
"to be visible\n - locator resolved to hidden <table>...</table>\n"
),
}
],
},
}
row_result = {
"ok": False,
"data": {
"workflow_run_id": "wr_failed_again",
"overall_status": "failed",
"blocks": [
{
"label": "extract_records",
"status": "failed",
"failure_reason": (
"Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "
'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table tbody tr").first '
"to be visible\n - locator resolved to hidden <tr>...</tr>\n"
),
}
],
},
}
table_outcome = recorded_outcome_from_run_blocks_result(table_result)
row_outcome = recorded_outcome_from_run_blocks_result(row_result)
assert table_outcome is not None
assert row_outcome is not None
assert table_outcome.is_authoritative is True
assert row_outcome.is_authoritative is True
assert table_outcome.structural_key != row_outcome.structural_key
assert table_outcome.key_provenance["structural_failure_identity"] == "typed runtime failure structure"
def test_persisted_run_prose_only_failure_is_not_authoritative() -> None:
result = {
"ok": False,
"error": "The registry form failed after waiting for the same selector.",
"data": {
"workflow_run_id": "wr_failed",
"overall_status": "failed",
"failure_reason": "Timeout waiting for selector #results on the registry form.",
"blocks": [{"label": "search_records", "status": "failed"}],
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is None or outcome.is_authoritative is False
assert outcome is None or outcome.structural_key is None
def _run_history_ctx(workflow_yaml: str) -> SimpleNamespace:
return SimpleNamespace(
workflow_yaml=workflow_yaml,
# The delivered candidate defaults to what the turn is working on; a test that cares about a
# draft diverging from the saved workflow sets them apart explicitly.
persisted_workflow_yaml=workflow_yaml,
staged_workflow_yaml=None,
latest_recorded_build_test_outcome=None,
recorded_build_test_outcome_history=[],
recorded_persisted_block_run_workflow_run_id=None,
)
def _raising_code_workflow_yaml(code: str) -> str:
return (
"workflow_definition:\n"
" parameters: []\n"
" blocks:\n"
" - block_type: code\n"
" label: book_trip\n"
" code: |\n" + "".join(f" {line}\n" for line in code.splitlines())
)
def _generated_code_exception_result(
workflow_run_id: str,
failure_reason: str,
downloaded_file_artifact_ids: list[str] | None = None,
) -> dict[str, object]:
block: dict[str, object] = {
"label": "book_trip",
"block_type": "CODE",
"status": "failed",
"workflow_run_block_id": "wrb_trip",
"failure_reason": failure_reason,
"error_codes": ["user_code_error"],
}
if downloaded_file_artifact_ids:
block["extracted_data"] = {"downloaded_file_artifact_ids": downloaded_file_artifact_ids}
return {
"ok": False,
"data": {
"workflow_run_id": workflow_run_id,
"browser_session_id": "pbs_trip",
"overall_status": "failed",
"requested_block_labels": ["book_trip"],
"executed_block_labels": ["book_trip"],
# The run reached the page but retained no usable evidence of it, which is the shape
# that used to erase the whole outcome.
"post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
"blocks": [block],
},
}
def test_generated_code_exception_reaches_repair() -> None:
"""A raised exception is the only evidence: no page state, no outputs, no failure categories."""
raising_code = "total = passenger_count * fare\nawait page.locator('#pay-now').click()\n"
failure_reason = "CodeBlock failed with NameError at line 1: name 'passenger_count' is not defined."
other_line_reason = "CodeBlock failed with NameError at line 2: name 'passenger_count' is not defined."
ctx = CopilotContext(
organization_id="org",
workflow_id="w",
workflow_permanent_id="wpid",
workflow_yaml=_raising_code_workflow_yaml(raising_code),
persisted_workflow_yaml=_raising_code_workflow_yaml(raising_code),
browser_session_id=None,
stream=None, # type: ignore[arg-type]
api_key=None,
block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
)
outcome = recorded_outcome_from_run_blocks_result(
_generated_code_exception_result("wr_trip", failure_reason),
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
display_reason=failure_reason,
workflow_run_id="wr_trip",
run_completed=False,
),
)
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code == "runtime_block_failure"
assert outcome.attempted_block_label == "book_trip"
assert outcome.is_authoritative is True
assert failure_reason in outcome.observed_evidence_summary
assert outcome.page_capture is not None and outcome.page_capture.omission == "page_capture_unavailable"
# Two exceptions in the same block are different failures, not one repeated one.
other_line_outcome = recorded_outcome_from_run_blocks_result(
_generated_code_exception_result("wr_trip_again", other_line_reason)
)
assert other_line_outcome is not None
assert other_line_outcome.structural_key != outcome.structural_key
# The exception's own message is written by the failing code, so it cannot decide identity.
# These two ran out of the same unlocatable raise; only the value the message quotes differs.
unlocated = "CodeBlock failed with NameError: name 'passenger_count' is not defined."
unlocated_quoting_a_line = "CodeBlock failed with NameError: name 'passenger_count' is not defined at line 9."
assert (
recorded_outcome_from_run_blocks_result(
_generated_code_exception_result("wr_trip_third", unlocated_quoting_a_line)
).structural_key # type: ignore[union-attr]
== recorded_outcome_from_run_blocks_result(
_generated_code_exception_result("wr_trip_fourth", unlocated)
).structural_key # type: ignore[union-attr]
)
record_build_test_outcome(ctx, outcome)
rendered = _recorded_build_test_outcome_prompt(ctx)
assert failure_reason in rendered
assert "page_capture: status=unavailable; omission=page_capture_unavailable" in rendered
def test_a_code_failure_with_no_typed_identity_is_not_admitted() -> None:
"""Nothing names the failure, so no later edit could ever be shown to have addressed it."""
assert (
recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": "wr_untyped",
"overall_status": "failed",
"requested_block_labels": ["book_trip"],
"blocks": [
{
"label": "book_trip",
"block_type": "CODE",
"status": "failed",
"failure_reason": "the block did not produce the requested output",
}
],
},
}
)
is None
)
def test_a_bare_code_failure_is_admitted_and_native_blocks_stay_out() -> None:
"""A non-builtin exception in a code block reaches persistence as the runner's generic phrase and
is still recorded as an authored-code failure; a native block's failure is not."""
# Byte-for-byte what the runner persists when the raised class is not a builtin, e.g. a
# Playwright timeout: the exception's own words are stripped on the way out.
generic_reason = "CodeBlock failed while running user code."
admitted = recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": "wr_browser_op",
"overall_status": "failed",
"requested_block_labels": ["book_trip"],
"blocks": [
{
"label": "book_trip",
"block_type": "CODE",
"status": "failed",
"failure_reason": generic_reason,
"error_codes": ["user_code_error"],
}
],
},
}
)
assert admitted is not None
assert admitted.reason_code == "runtime_block_failure"
assert admitted.attempted_block_label == "book_trip"
# A failure no rewrite could have prevented is not recorded as an authored-code failure: the
# same block can pass unchanged once the sandbox is back, the browser reconnects, or the run is
# not cancelled.
for code in ("runner_unavailable", "browser_disconnected", "cancelled"):
assert (
recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": f"wr_{code}",
"overall_status": "failed",
"requested_block_labels": ["book_trip"],
"blocks": [
{
"label": "book_trip",
"block_type": "CODE",
"status": "failed",
"failure_reason": generic_reason,
"error_codes": [code],
}
],
},
}
)
is None
), code
native_outcome = recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": "wr_native",
"overall_status": "failed",
"requested_block_labels": ["inspect_record"],
"blocks": [
{
"label": "inspect_record",
"block_type": "TASK",
"status": "failed",
"failure_reason": "Reached the maximum steps (30)",
"error_codes": ["max_steps_exceeded"],
}
],
},
}
)
assert native_outcome is None
def _templated_selector_yaml(*, templated: bool = True) -> str:
call = 'f"#submit-{kind}"' if templated else '"#submit-btn"'
return f"""
title: Read the metric
workflow_definition:
blocks:
- block_type: code
label: sign_in_and_read
code: |
if await page.locator("#token").count():
await page.locator({call}).click()
"""
def test_merging_an_acquisition_failure_does_not_resnapshot_the_failure_against_edited_code() -> None:
"""History snapshots are taken from the current draft, so re-appending the carried failure after
an edit would record it against code the run never executed."""
ctx = _run_history_ctx(_templated_selector_yaml())
ctx.runner_code_block_associations_by_label = {}
record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
entries_before = len(ctx.recorded_build_test_outcome_history)
signature_at_failure = ctx.recorded_build_test_outcome_history[-1]["attempted_block_signature"]
assert signature_at_failure
# The model repairs the block, then the next dispatch dies before a run exists.
edited = _templated_selector_yaml(templated=False)
ctx.workflow_yaml = edited
ctx.persisted_workflow_yaml = edited
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="unrecoverable_tool_error",
connect_failure=BuildTestConnectFailure(
state="cdp_connect_failed",
browser_session_id="pbs_stale",
),
),
)
assert len(ctx.recorded_build_test_outcome_history) == entries_before
assert ctx.recorded_build_test_outcome_history[-1]["attempted_block_signature"] == signature_at_failure
# The diagnosis still reaches the repair prompt, which reads only the newest outcome.
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.reason_code == "runtime_block_failure"
assert latest.connect_failure is not None
def _secure_runner_failure_result(error_code: str, category: str | None) -> dict[str, object]:
data: dict[str, object] = {
"workflow_run_id": "wr_runner",
"overall_status": "failed",
"blocks": [
{
"label": "run_code",
"block_type": "CODE",
"status": "failed",
"failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",
"error_codes": [error_code],
}
],
}
if category is not None:
data["failure_categories"] = [{"category": category, "confidence_float": 1.0, "reasoning": "sandbox"}]
return {"ok": False, "data": data}
def test_unrecoverable_tool_error_run_is_not_repairable_and_not_authoritative() -> None:
outcome = recorded_outcome_from_run_blocks_result(
_secure_runner_failure_result("runner_unavailable", "UNRECOVERABLE_TOOL_ERROR")
)
assert outcome is not None
assert outcome.verdict != "repairable_failure"
assert outcome.verdict == "not_authoritative"
assert outcome.reason_code == "unrecoverable_tool_error"
assert outcome.is_authoritative is False
assert outcome.structural_failure_identity == ""
def test_user_code_error_run_stays_repairable() -> None:
outcome = recorded_outcome_from_run_blocks_result(
_secure_runner_failure_result("user_code_error", "CODE_BLOCK_FAILURE")
)
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code == "runtime_block_failure"
def test_runner_timeout_run_stays_repairable() -> None:
outcome = recorded_outcome_from_run_blocks_result(_secure_runner_failure_result("timeout", "CODE_BLOCK_FAILURE"))
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code == "runtime_block_failure"
def _failed_run_result(observations: object) -> dict[str, object]:
data: dict[str, object] = {
"workflow_run_id": "wr_1",
"overall_status": "failed",
"requested_block_labels": ["read_value"],
"executed_block_labels": ["read_value"],
"blocks": [{"label": "read_value", "status": "failed", "failure_reason": "CodeBlock execution timed out."}],
}
if observations is not None:
data["authored_locator_observations"] = observations
return {"ok": False, "data": data}
def _locator_packet_ctx() -> CopilotContext:
return CopilotContext(
organization_id="org",
workflow_id="w",
workflow_permanent_id="wpid",
workflow_yaml="workflow_definition:\n blocks: []\n",
persisted_workflow_yaml="workflow_definition:\n blocks: []\n",
browser_session_id=None,
stream=None, # type: ignore[arg-type]
api_key=None,
)
def _oversized_packet(packet: BuildTestEvidencePacket) -> BuildTestEvidencePacket:
filler = "f" * 200
return packet.model_copy(
update={
"canonical_workflow_yaml": "w" * 30_000,
"attempted_block_labels": [f"{filler}{index}" for index in range(30)],
"executed_block_labels": [f"{filler}{index}" for index in range(30)],
"registered_outputs": [
BuildTestPacketRegisteredOutput(
label=filler,
status=filler,
output="v" * 1_000,
)
for _ in range(15)
],
"requested_outputs": [
BuildTestPacketRequestedOutput(
workflow_run_id="wr_failed_complete_packet",
output_parameter_id=f"requested_{index}",
output_parameter_key=f"requested_{index}",
)
for index in range(10)
],
"downloads": [BuildTestPacketDownload(artifact_id=filler, file_name=filler) for _ in range(15)],
"unfinished_items": [
BuildTestPacketUnfinishedItem(
kind="unverified_block",
label=filler,
output_path=filler,
reason_code=filler,
)
for _ in range(30)
],
}
)
def test_a_page_whose_only_fact_is_a_label_value_pair_still_yields_a_packet_page_state() -> None:
evidence = {
"workflow_run_id": "wr_bindings",
"observed_after_workflow_run": True,
"source_tool": "inspect_page_for_composition",
"key_value_relations": [
{
"key_text": "Sessions started",
"value_text": "72.51k",
"container_selector": "div.query-value__container",
"visible": True,
"value_visible": True,
}
],
}
from_evidence = build_test_page_state_from_evidence(evidence, workflow_run_id="wr_bindings", omission_notices=[])
assert from_evidence is not None
assert from_evidence.value_bindings == ["Sessions started=72.51k"]
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
result: dict[str, object] = {
"ok": False,
"error": "Run failed.",
"data": {
"workflow_run_id": "wr_bindings",
"overall_status": "failed",
"blocks": [{"label": "read_sessions", "status": "failed", "failure_reason": "RuntimeError"}],
"authoring_repair_context": {
"workflow_run_id": "wr_bindings",
"observed_after_workflow_run": True,
"page_value_bindings": ["Sessions started=72.51k"],
},
},
}
_record_run_blocks_result(ctx, result)
packet = build_test_evidence_packet(ctx, result)
projected = project_build_test_packet_for_llm(packet)
assert packet.failure is not None and packet.failure.page_state is not None
assert packet.failure.page_state.value_bindings == ["Sessions started=72.51k"]
assert projected.failure is not None and projected.failure.page_state is not None
assert projected.failure.page_state.value_bindings == ["Sessions started=72.51k"]
_BLOCKING_LAYER = {
"kind": "interaction_blocking_layer",
"text": "We use cookies",
"intercepts_outside_control": True,
"visible_controls": [{"tag": "button", "text": "Accept all"}],
"visible_controls_omitted": 1,
}
def test_a_blocking_layer_reaches_the_model_and_a_malformed_one_is_announced() -> None:
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
result: dict[str, object] = {
"ok": False,
"error": "Run failed.",
"data": {
"workflow_run_id": "wr_obstructed",
"overall_status": "failed",
"blocks": [{"label": "search", "status": "failed", "failure_reason": "RuntimeError"}],
"post_run_page_evidence": {
"workflow_run_id": "wr_obstructed",
"observed_after_workflow_run": True,
"page_obstructions": [_BLOCKING_LAYER, {**_BLOCKING_LAYER, "undeclared_field": "x"}],
},
},
}
_record_run_blocks_result(ctx, result)
data = result["data"]
assert isinstance(data, dict)
data["build_test_packet"] = build_test_evidence_packet(ctx, result).model_dump(mode="json", exclude_none=True)
model_packet = json.loads(json.dumps(sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)))["data"][
"build_test_packet"
]
assert "failure.page_state.obstructions omitted: 1 malformed item(s)." in model_packet["omission_notices"]
assert [
(item["text"], item["intercepts_outside_control"], item["visible_controls_omitted"])
for item in model_packet["failure"]["page_state"]["obstructions"]
] == [("We use cookies", True, 1)]
def test_a_standalone_clickable_control_reaches_the_packet_page_state_and_the_llm_projection() -> None:
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
result: dict[str, object] = {
"ok": False,
"error": "Run failed.",
"data": {
"workflow_run_id": "wr_failed",
"overall_status": "failed",
"blocks": [
{
"label": "open_statements",
"status": "failed",
"failure_reason": 'Timeout waiting for locator("#continue-btn-x9")',
}
],
},
}
_record_run_blocks_result(ctx, result)
ctx.composition_page_evidence = {
"workflow_run_id": "wr_failed",
"observed_after_workflow_run": True,
"source_tool": "inspect_page_for_composition",
"current_url": "https://example.test/app/statements",
"navigation_targets": [{"text": f"Section {index}", "selector": f"a.s{index}"} for index in range(6)],
"clickable_controls": [
{"text": "Continue to statements", "selector": "#continue-btn-x9", "tag": "button", "disabled": True}
],
}
inject_runtime_authoring_repair_context(ctx, result)
finalized = ctx.last_code_authoring_repair_context
assert isinstance(finalized, CodeAuthoringRepairContext)
packet = build_test_evidence_packet(ctx, result)
projected = project_build_test_packet_for_llm(packet)
assert packet.failure is not None and packet.failure.page_state is not None
assert packet.run.workflow_run_id == "wr_failed"
assert packet.failure.page_state.action_summaries == finalized.page_action_summaries
assert projected.failure is not None and projected.failure.page_state is not None
assert projected.failure.page_state.action_summaries == [
"Continue to statements disabled",
"Section 0",
"Section 1",
"Section 2",
"Section 3",
]
assert all("#continue-btn-x9" not in summary for summary in projected.failure.page_state.action_summaries)
def test_screenshot_ablation_replays_preserve_page_facts_without_minting_a_decision() -> None:
"""A missing frame changes only capture facts; the recorded run's failure remains a run fact."""
def rendered_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
def replay(*, screenshot_present: bool) -> tuple[dict[str, object], dict[str, object], str]:
ctx = _locator_packet_ctx()
page_evidence: dict[str, object] = {
"workflow_run_id": "wr_capture_failure",
"source_browser_session_id": "pbs_capture_failure",
"run_browser_session_id": "pbs_capture_failure",
"observed_after_workflow_run": True,
"source_tool": "inspect_page_for_composition",
"current_url": "https://example.test/checkout/payment",
"forms": [{"submit_controls": [{"text": "Place order", "disabled": False}]}],
"clickable_controls": [{"text": "Continue to payment", "disabled": False}],
**({} if screenshot_present else {"visual_capture_omissions": ["screenshot_capture_failed"]}),
}
result: dict[str, object] = {
"ok": False,
"data": {
"workflow_run_id": "wr_capture_failure",
"browser_session_id": "pbs_capture_failure",
"overall_status": "failed",
"requested_block_labels": ["continue_to_payment"],
"executed_block_labels": ["continue_to_payment"],
"blocks": [
{
"label": "continue_to_payment",
"status": "failed",
"failure_reason": "Click did not reach payment options.",
}
],
"post_run_page_evidence": page_evidence,
**({"screenshot_base64": "c2NyZWVuc2hvdA=="} if screenshot_present else {}),
},
}
stored, preserved = store_post_run_page_evidence(
ctx,
page_evidence,
run_id="wr_capture_failure",
current_url="https://example.test/checkout/payment",
source_browser_session_id="pbs_capture_failure",
run_browser_session_id="pbs_capture_failure",
)
assert preserved is False
assert stored["observed_after_workflow_run"] is True
result["data"]["post_run_page_evidence"] = run_execution_module._same_run_page_evidence_for_result(
ctx, "wr_capture_failure"
)
outcome = recorded_outcome_from_run_blocks_result(
result,
page_evidence=result["data"]["post_run_page_evidence"],
)
assert outcome is not None
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
ordinary_input = _build_user_context(
workflow_yaml=ctx.workflow_yaml,
chat_history_text="",
global_llm_context="",
debug_run_info_text=_prior_run_debug_text(packet),
user_message="Repair the recorded run.",
)
return outcome.model_dump(mode="json"), packet, ordinary_input
with_screenshot = [replay(screenshot_present=True) for _ in range(3)]
without_screenshot = [replay(screenshot_present=False) for _ in range(3)]
assert len({rendered_bytes(replay_outcome) for replay_outcome, _, _ in with_screenshot}) == 1
assert len({rendered_bytes(replay_outcome) for replay_outcome, _, _ in without_screenshot}) == 1
assert len({rendered_bytes(packet) for _, packet, _ in with_screenshot}) == 1
assert len({rendered_bytes(packet) for _, packet, _ in without_screenshot}) == 1
baseline_outcome, baseline_packet, _ = with_screenshot[0]
missing_outcome, missing_packet, ordinary_input = without_screenshot[0]
assert {key: value for key, value in baseline_outcome.items() if key != "page_capture"} == {
key: value for key, value in missing_outcome.items() if key != "page_capture"
}
assert missing_packet["page_capture"] == {"status": "captured", "omission": "screenshot_capture_failed"}
assert baseline_packet["screenshot"]["present"] is True
assert missing_packet["screenshot"]["present"] is False
assert {
key: value
for key, value in baseline_packet.items()
if key not in {"page_capture", "screenshot", "omission_notices"}
} == {
key: value
for key, value in missing_packet.items()
if key not in {"page_capture", "screenshot", "omission_notices"}
}
assert not {"success_verdict", "terminal", "repair_decision"}.intersection(missing_packet)
decision_surface = {
key: value
for key, value in missing_packet.items()
if key not in {"page_capture", "screenshot", "omission_notices"}
}
assert b"success_verdict" not in rendered_bytes(decision_surface)
assert b"terminal" not in rendered_bytes(decision_surface)
assert b"repair_decision" not in rendered_bytes(decision_surface)
assert '"workflow_run_id": "wr_capture_failure"' in ordinary_input
assert '"browser_session_id": "pbs_capture_failure"' in ordinary_input
assert "Place order" in ordinary_input
assert "Continue to payment" in ordinary_input
assert '"screenshot_capture_failed"' in ordinary_input
def test_unavailable_page_capture_is_a_typed_omission_without_invented_page_state() -> None:
result: dict[str, object] = {
"ok": False,
"data": {
"workflow_run_id": "wr_page_unavailable",
"overall_status": "failed",
"failure_type": "runtime_error",
"requested_block_labels": ["continue_to_payment"],
"executed_block_labels": ["continue_to_payment"],
"blocks": [
{
"label": "continue_to_payment",
"status": "failed",
"failure_reason": "Click did not reach payment options.",
}
],
"post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
},
}
replayed_outcomes_and_packets = []
for _ in range(3):
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
replayed_outcomes_and_packets.append((outcome, packet))
assert len({outcome.model_dump_json() for outcome, _ in replayed_outcomes_and_packets}) == 1
assert len({packet.model_dump_json() for _, packet in replayed_outcomes_and_packets}) == 1
outcome, packet = replayed_outcomes_and_packets[0]
assert outcome.page_capture is not None
assert outcome.page_capture.status == "unavailable"
assert outcome.page_capture.omission == "page_capture_unavailable"
assert packet.page_capture == outcome.page_capture
assert packet.failure is not None and packet.failure.page_state is None
assert packet.page_state is None
assert "success_verdict" not in packet.model_dump_json()
assert "repair_decision" not in packet.model_dump_json()
assert "terminal" not in packet.model_dump_json()
def test_stale_page_evidence_cannot_infer_a_capture_for_the_current_run() -> None:
result: dict[str, object] = {
"ok": False,
"data": {
"workflow_run_id": "wr_current",
"browser_session_id": "pbs_current",
"overall_status": "failed",
"failure_type": "runtime_error",
"blocks": [{"label": "continue", "status": "failed", "failure_reason": "Click failed."}],
},
}
stale_scout_packet = {
"source_browser_session_id": "pbs_current",
"observed_after_workflow_run": False,
"forms": [{"submit_controls": [{"text": "Place order", "disabled": False}]}],
}
outcome = recorded_outcome_from_run_blocks_result(result, page_evidence=stale_scout_packet)
assert outcome is not None
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
assert outcome.page_capture is None
assert outcome.page_evidence_refs == []
assert packet.page_capture is None
def test_required_input_unbound_outcome_preserves_the_typed_capture_fact() -> None:
result: dict[str, object] = {
"ok": False,
"data": {
"workflow_run_id": "wr_unbound",
"overall_status": "failed",
"blocks": [{"label": "continue", "status": "failed", "failure_reason": "Input was not bound."}],
"post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
},
}
outcome = recorded_outcome_from_run_blocks_result(
result,
unbound_required_parameter_keys=["checkout_id"],
block_parameter_keys={"continue": ["checkout_id"]},
)
assert outcome is not None
assert outcome.reason_code == "required_input_unbound"
assert outcome.page_capture is not None
assert outcome.page_capture.model_dump() == {"status": "unavailable", "omission": "page_capture_unavailable"}
@pytest.mark.parametrize(
"return_path",
[
"terminal_challenge",
"demonstrated",
"not_evaluated",
"no_structural_identity",
"degraded_floor",
"recorded_failed_block",
"run_failed_block",
],
)
def test_every_run_outcome_return_path_preserves_the_typed_capture_fact(return_path: str) -> None:
result: dict[str, object] = {
"ok": False,
"data": {
"workflow_run_id": "wr_capture_paths",
"overall_status": "failed",
"failure_type": "runtime_error",
"blocks": [{"label": "continue", "status": "failed", "failure_reason": "Click failed."}],
"post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
},
}
recorded_run_outcome: RecordedRunOutcome | None = None
completion_verification: CompletionVerificationResult | None = None
if return_path == "terminal_challenge":
recorded_run_outcome = RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="terminal_challenge_blocker",
workflow_run_id="wr_capture_paths",
)
elif return_path == "demonstrated":
result = {**result, "ok": True, "data": {**result["data"], "blocks": []}}
recorded_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_capture_paths")
elif return_path == "not_evaluated":
result = {**result, "ok": True, "data": {**result["data"], "blocks": []}}
recorded_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_capture_paths")
elif return_path == "no_structural_identity":
result = {**result, "ok": True, "data": {**result["data"], "blocks": [], "failure_type": None}}
recorded_run_outcome = RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
workflow_run_id="wr_capture_paths",
)
elif return_path == "degraded_floor":
result = {**result, "ok": True, "data": {**result["data"], "blocks": [], "failure_type": None}}
recorded_run_outcome = RecordedRunOutcome(verdict="not_demonstrated", workflow_run_id="wr_capture_paths")
completion_verification = CompletionVerificationResult(
status="evaluated",
criterion_ids=["c0"],
verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="no_evidence")],
degraded_criterion_ids=["c0"],
)
elif return_path == "recorded_failed_block":
recorded_run_outcome = RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
workflow_run_id="wr_capture_paths",
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=recorded_run_outcome,
completion_verification=completion_verification,
)
assert outcome is not None
assert outcome.page_capture is not None
assert outcome.page_capture.model_dump() == {"status": "unavailable", "omission": "page_capture_unavailable"}
def test_a_not_evaluated_run_keeps_the_page_values_it_captured() -> None:
"""A completed-but-unevaluated run is when the model most needs what the page showed."""
result: dict[str, object] = {
"ok": True,
"data": {
"workflow_run_id": "wr_not_evaluated_values",
"overall_status": "completed",
"blocks": [
{
"label": "read_failure_rate",
"block_type": "code",
"status": "completed",
"extracted_data": {"failure_rate": ""},
}
],
},
}
page_evidence = {
"observed_after_workflow_run": True,
"workflow_run_id": "wr_not_evaluated_values",
"current_url": "https://dashboard.example.com/health",
"visible_text_excerpt": "Service health Failure rate 26.05 %",
}
outcome = recorded_outcome_from_run_blocks_result(
result,
page_evidence=page_evidence,
recorded_run_outcome=RecordedRunOutcome(
run_completed=True, verdict="not_evaluated", workflow_run_id="wr_not_evaluated_values"
),
declared_goal_path_omissions=[{"block_label": "read_failure_rate", "output_path": "failure_rate"}],
)
assert outcome is not None
assert outcome.reason_code == "run_completed_unevaluated"
assert "26.05 %" in outcome.observed_page_value_excerpt
# Declared-path omissions stay facts on this path; carrying them here would make the run graded.
assert outcome.missing_requested_output_facts == []
# The values must arrive without minting authority: page_evidence_refs feed structural_key, so
# an unevaluated run carrying them would report itself authoritative and be latched as
# run-backed evidence. Scope is this constructor; a run that later reaches
# bind_post_run_page_evidence has its refs overlaid there, which predates this pin.
assert outcome.verdict == "not_authoritative"
assert outcome.page_evidence_refs == []
assert outcome.is_authoritative is False
def test_page_capture_rejects_an_unavailable_page_without_its_typed_omission() -> None:
with pytest.raises(ValidationError, match="page_capture_unavailable"):
BuildTestEvidencePacket.model_validate(
{
"canonical_workflow_source": "unavailable",
"run": {},
"screenshot": {"present": False},
"page_capture": {"status": "unavailable"},
}
)
def test_locator_observations_reach_the_failure_packet() -> None:
result = _failed_run_result([{"authored_selector": "button.old", "match_count": 0}])
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
observation = packet.failure.locator_observations[0]
assert (observation.authored_selector, observation.match_count) == ("button.old", 0)
assert observation.match_index is None
assert observation.observed_candidates is None
assert observation.unobserved_reason is None
def test_an_unattempted_observation_is_disclosed_rather_than_reported_as_no_locators() -> None:
# A dispatched run, a non-code block, or a policy without inline execution never attempts the
# observation. That is a different fact from "this block names no locator", and it is also not
# evidence the page was unreachable.
packet = build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result(None))
assert packet.failure is not None
assert packet.failure.locator_observations == []
assert any("no post-action locator observation was attempted" in notice for notice in packet.omission_notices)
def test_a_block_with_no_literal_locator_is_disclosed_as_such() -> None:
packet = build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result([]))
assert packet.failure is not None
assert any("names no literal locator" in notice for notice in packet.omission_notices)
def test_locators_the_run_could_not_be_asked_about_remain_typed_rows() -> None:
result = _failed_run_result(
[
{"authored_selector": "#rate", "unobserved_reason": "run_page_unavailable"},
{"authored_selector": "text=Submit", "unobserved_reason": "run_page_unavailable"},
]
)
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
assert [row.authored_selector for row in packet.failure.locator_observations] == ["#rate", "text=Submit"]
assert {row.unobserved_reason for row in packet.failure.locator_observations} == {"run_page_unavailable"}
@pytest.mark.parametrize(
"kwargs",
[
{"match_count": None},
{"match_count": False},
{"match_count": 0, "match_index": 0},
{"match_count": 0, "observed_candidates": ["button"]},
{"match_count": 2, "match_index": 0},
{"match_count": 2, "observed_candidates": ["button"]},
{"match_count": 2, "match_index": 0, "observed_candidates": [""]},
{
"match_count": 2,
"match_index": 0,
"observed_candidates": ["button"],
"unobserved_reason": "identity_read_failed",
},
{"unobserved_reason": "run_page_unavailable", "observed_candidates": ["button"]},
],
)
def test_locator_observation_schema_rejects_mixed_or_incomplete_states(kwargs: dict[str, object]) -> None:
with pytest.raises(ValidationError):
BuildTestPacketLocatorObservation.model_validate({"authored_selector": "button", **kwargs})
def test_positive_locator_observation_requires_index_zero_and_an_identity() -> None:
row = BuildTestPacketLocatorObservation(
authored_selector="button",
match_count=2,
match_index=0,
observed_candidates=["button#save"],
)
assert row.match_count == 2
assert row.match_index == 0
assert row.observed_candidates == ["button#save"]
def test_a_malformed_observation_is_dropped_and_counted() -> None:
result = _failed_run_result([{"authored_selector": "button.old", "match_count": "many"}, {"no_selector": True}])
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
assert packet.failure.locator_observations == []
assert any("omitted 2 malformed item(s)" in notice for notice in packet.omission_notices)
def test_failed_block_code_reads_the_definition_not_the_run_rows() -> None:
# run_blocks works with workflow_run_block rows, which carry status and output but no code, so
# a producer that looked for code on them would silently never observe anything.
workflow = SimpleNamespace(
workflow_definition={
"blocks": [
{"label": "ok_block", "block_type": "code", "code": "pass"},
{"label": "read_value", "block_type": "code", "code": 'page.locator("button.old")'},
]
}
)
run_rows = [{"label": "ok_block", "status": "completed"}, {"label": "read_value", "status": "failed"}]
assert _failed_block_code(workflow, _newest_failed_result(run_rows)) == 'page.locator("button.old")'
def test_failed_block_code_uses_failed_result_row_order_not_definition_order() -> None:
workflow = SimpleNamespace(
workflow_definition={
"blocks": [
{"label": "main", "block_type": "code", "code": 'page.locator("#main")'},
{"label": "finally", "block_type": "code", "code": 'page.locator("#finally")'},
]
}
)
assert (
_failed_block_code(
workflow,
_newest_failed_result([{"label": "main", "status": "failed"}, {"label": "finally", "status": "failed"}]),
)
== 'page.locator("#finally")'
)
def test_failed_block_code_does_not_skip_a_nullable_label_to_an_earlier_failed_row() -> None:
workflow = SimpleNamespace(
workflow_definition={
"blocks": [
{"label": "main", "block_type": "code", "code": 'page.locator("#main")'},
{"label": "finally", "block_type": "code", "code": 'page.locator("#finally")'},
]
}
)
failed_rows = [{"label": "finally", "status": "failed"}, {"label": None, "status": "failed"}]
selected = _newest_failed_result(failed_rows)
assert selected is failed_rows[-1]
assert _failed_block_code(workflow, selected) is None
def test_nullable_label_sequence_keeps_packet_metadata_and_locator_evidence_on_selected_row() -> None:
workflow = SimpleNamespace(
workflow_definition={
"blocks": [
{"label": "finally", "block_type": "code", "code": 'page.locator("#finally")'},
]
}
)
failed_rows = [
{"label": "finally", "status": "failed", "failure_reason": "finally failed"},
{"label": None, "status": "failed", "failure_reason": "unlabeled failed"},
]
selected = _newest_failed_result(failed_rows)
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["blocks"] = failed_rows
data["action_trace_summary"] = ["unlabeled line 3 failed"]
data["failing_code_line"] = 3
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert _failed_block_code(workflow, selected) is None
assert packet.failure is not None
assert packet.failure.block_label is None
assert packet.failure.reason == "unlabeled failed"
assert packet.failure.action_trace == ["unlabeled line 3 failed"]
assert packet.failure.failing_line == 3
assert packet.failure.locator_observations == []
def test_failure_label_trace_and_locators_share_the_newest_failed_result_row() -> None:
result = _failed_run_result(
[
{
"authored_selector": "#finally",
"match_count": 1,
"match_index": 0,
"observed_candidates": ["button#finally"],
}
]
)
data = result["data"]
assert isinstance(data, dict)
data["blocks"] = [
{"label": "main", "status": "failed", "failure_reason": "main failed"},
{"label": "finally", "status": "failed", "failure_reason": "finally failed"},
]
data["action_trace_summary"] = ["finally line 7 failed"]
data["failing_code_line"] = 7
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
assert packet.failure.block_label == "finally"
assert packet.failure.action_trace == ["finally line 7 failed"]
assert packet.failure.failing_line == 7
assert [row.authored_selector for row in packet.failure.locator_observations] == ["#finally"]
def test_failed_block_code_is_none_when_nothing_failed() -> None:
workflow = SimpleNamespace(workflow_definition={"blocks": [{"label": "a", "block_type": "code", "code": "pass"}]})
results = [{"label": "a", "status": "completed"}]
assert _failed_block_code(workflow, _newest_failed_result(results)) is None
def test_authored_literal_selectors_are_ordered_by_source_position() -> None:
# ast.walk is breadth-first, so a nested locator would surface after a later shallow one.
code = 'await wrapper(page.locator("first"))\npage.locator("second")\n'
assert _authored_literal_locator_selectors(code) == ["first", "second"]
def test_a_run_outside_the_chats_browser_says_so() -> None:
# A carried resume browser is detached without being freshly minted, so the fact is derived
# from detachment: keying it on minting would tell the model it shared the chat's browser.
result = _failed_run_result(None)
result["data"]["run_detached_from_chat"] = True
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.run.browser is not None
assert packet.run.browser.ran_outside_this_chats_browser is True
assert "other than" in packet.run.browser.note
def test_a_run_sharing_the_chat_browser_says_that_instead() -> None:
result = _failed_run_result(None)
result["data"]["run_detached_from_chat"] = False
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.run.browser is not None
assert packet.run.browser.ran_outside_this_chats_browser is False
assert "target=" not in packet.run.browser.note
def test_an_unrecorded_browser_relationship_is_absent_rather_than_guessed() -> None:
packet = build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result(None))
assert packet.run.browser is None
def test_the_runners_typed_error_and_failing_line_reach_the_repair_turn() -> None:
# The runtime records which error code fired and which line raised. Rendering them into an
# action-summary sentence tells the repair turn the block failed in English; these are the
# fields it would act on.
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["blocks"] = [
{
"label": "extract_failure_rate",
"status": "failed",
"failure_reason": "code error at line 6",
"error_codes": ["user_code_error"],
}
]
data["failing_code_line"] = 6
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
assert packet.failure.error_codes == ["user_code_error"]
assert packet.failure.failing_line == 6
def test_browser_operation_failure_projects_same_row_run_and_block_identity() -> None:
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["workflow_run_id"] = "wr_browser_operation"
data["blocks"] = [
{
"workflow_run_block_id": "wrb_capture_failure",
"label": "collect_failure_rate",
"status": "failed",
"failure_reason": "browser operation failed",
"error_codes": ["browser_operation_failed"],
}
]
data["failing_code_line"] = 11
outcome = recorded_outcome_from_run_blocks_result(result)
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
assert outcome is not None
assert outcome.failed_operation is not None
assert outcome.failed_operation.model_dump() == {
"kind": "browser_operation_failed",
"workflow_run_id": "wr_browser_operation",
"workflow_run_block_id": "wrb_capture_failure",
"block_label": "collect_failure_rate",
"failing_line": 11,
}
assert packet.failure is not None
assert packet.failure.failed_operation == outcome.failed_operation
projected = project_build_test_packet_for_llm(packet)
assert projected.failure is not None
assert projected.failure.failed_operation == outcome.failed_operation
def test_a_run_with_two_failed_blocks_reports_the_newest_failure() -> None:
"""``data['blocks']`` is chronological, so on a continue-on-failure run the packet names the
failure the run stopped on rather than an earlier one it survived."""
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["workflow_run_id"] = "wr_two_failures"
data["blocks"] = [
{
"workflow_run_block_id": "wrb_search",
"label": "run_search",
"status": "failed",
"failure_reason": "search submit never navigated",
"error_codes": ["browser_operation_failed"],
},
{
"workflow_run_block_id": "wrb_select",
"label": "select_first_result",
"status": "failed",
"failure_reason": "result locator never appeared",
"error_codes": ["browser_operation_failed"],
},
]
outcome = recorded_outcome_from_run_blocks_result(result)
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
assert outcome is not None
assert outcome.failed_operation is not None
assert outcome.failed_operation.workflow_run_block_id == "wrb_select"
assert outcome.failed_operation.block_label == "select_first_result"
assert packet.failure is not None
assert packet.failure.block_label == "select_first_result"
assert packet.failure.reason == "result locator never appeared"
def test_browser_operation_packet_uses_same_run_recorded_outcome_as_its_authority() -> None:
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["workflow_run_id"] = "wr_recorded_authority"
data["blocks"] = [
{
"workflow_run_block_id": "wrb_recorded_authority",
"label": "collect_failure_rate",
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
]
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.failed_operation is not None
blocks = data["blocks"]
assert isinstance(blocks, list)
assert isinstance(blocks[0], dict)
blocks[0]["workflow_run_block_id"] = "wrb_raw_result_changed_after_recording"
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
assert packet.failure is not None
assert packet.failure.failed_operation == outcome.failed_operation
assert packet.failure.failed_operation.workflow_run_block_id == "wrb_recorded_authority"
def test_browser_operation_producer_redacts_registered_block_label_secret() -> None:
session_id = "pbs_build_test_operation_redaction"
secret = "build-test-label-secret-value"
ctx = SimpleNamespace(browser_session_id=session_id, secret_scrub_values=[])
register_secret_scrub_value(ctx, secret)
try:
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["blocks"] = [
{
"label": f"collect_{secret}_rate",
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
]
outcome = recorded_outcome_from_run_blocks_result(result)
finally:
clear_session_scrub_values(session_id)
assert outcome is not None
assert outcome.failed_operation is not None
assert secret not in outcome.model_dump_json()
assert outcome.failed_operation.block_label == "collect_[REDACTED_SECRET]_rate"
def test_browser_operation_structural_key_ignores_transient_run_and_block_ids() -> None:
first = _failed_run_result(None)
first_data = first["data"]
assert isinstance(first_data, dict)
first_data["workflow_run_id"] = "wr_first"
first_data["blocks"] = [
{
"workflow_run_block_id": "wrb_first",
"label": "collect_failure_rate",
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
]
first_data["failing_code_line"] = 11
second = copy.deepcopy(first)
second_data = second["data"]
assert isinstance(second_data, dict)
second_data["workflow_run_id"] = "wr_second"
blocks = second_data["blocks"]
assert isinstance(blocks, list)
assert isinstance(blocks[0], dict)
blocks[0]["workflow_run_block_id"] = "wrb_second"
first_outcome = recorded_outcome_from_run_blocks_result(first)
second_outcome = recorded_outcome_from_run_blocks_result(second)
assert first_outcome is not None
assert second_outcome is not None
assert first_outcome.failed_operation is not None
assert second_outcome.failed_operation is not None
assert first_outcome.failed_operation.workflow_run_id != second_outcome.failed_operation.workflow_run_id
assert first_outcome.structural_key == second_outcome.structural_key
def test_browser_operation_failure_survives_non_clearing_outcomes_until_the_block_completes() -> None:
failed_workflow = """title: retain browser failure
workflow_definition:
parameters: []
blocks:
- block_type: code
label: collect_failure_rate
code: |
return {"failure_rate": await page.locator("canvas.failure-rate").inner_text()}
"""
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
ctx.runner_code_block_associations_by_label = {"collect_failure_rate": "cba_collect_failure_rate"}
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_failed",
workflow_run_block_id="wrb_failed",
block_label="collect_failure_rate",
failing_line=1,
block_association="cba_collect_failure_rate",
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_failed",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="scout_evaluate",
attempted_tool="inspect_page_for_composition",
verdict="progress_observed",
reason_code="verified_success",
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="progress_observed",
reason_code="verified_success",
workflow_run_id="wr_unchanged",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
verified_progress_marker="run-completed",
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_block_failed_again",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
executed_block_labels=["collect_failure_rate"],
executed_block_associations=("cba_collect_failure_rate",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="not_authoritative",
reason_code="run_completed_unevaluated",
workflow_run_id="wr_block_completed",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
executed_block_labels=["collect_failure_rate"],
executed_block_associations=("cba_collect_failure_rate",),
completed_block_associations=("cba_collect_failure_rate",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_unevaluated_later_run_that_completed_the_failed_block_clears_the_stop() -> None:
failed_workflow = """workflow_definition:
parameters: []
blocks:
- block_type: code
label: submit
code: |
await page.locator("#submit").click(timeout=1000)
"""
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
ctx.runner_code_block_associations_by_label = {"submit": "cba_submit"}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_failed",
structural_failure_identity="browser-operation",
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_failed",
block_label="submit",
failing_line=1,
block_association="cba_submit",
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="run_completed_unevaluated",
workflow_run_id="wr_retested",
executed_block_associations=("cba_submit",),
completed_block_associations=("cba_submit",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_browser_operation_failure_clears_when_a_full_replacement_completes_the_block() -> None:
failed_workflow = """workflow_definition:
parameters: []
blocks:
- block_type: code
label: collect_failure_rate
copilot_block_association: cba_original
code: |
return await page.locator("canvas.failure-rate").inner_text()
- block_type: code
label: summarize
code: |
return {"summary": "ready"}
"""
replacement_workflow = """workflow_definition:
parameters: []
blocks:
- block_type: code
label: summarize
code: |
return {"summary": "ready"}
- block_type: code
label: collect_failure_rate
copilot_block_association: cba_original
code: |
return {"unrelated": "replacement"}
"""
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.staged_workflow_yaml = failed_workflow
ctx.runner_code_block_associations_by_label = {"collect_failure_rate": "cba_original"}
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_failed",
block_label="collect_failure_rate",
failing_line=1,
block_association="cba_original",
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_failed",
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
ctx.workflow_yaml = replacement_workflow
ctx.staged_workflow_yaml = replacement_workflow
ctx.runner_code_block_associations_by_label = runner_code_block_associations(
replacement_workflow,
prior_associations=ctx.runner_code_block_associations_by_label,
preserve_existing=False,
)
replacement_association = ctx.runner_code_block_associations_by_label["collect_failure_rate"]
assert replacement_association != "cba_original"
# Running the replacement without completing it resolves nothing.
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="runtime_block_failure",
workflow_run_id="wr_replacement_failed",
executed_block_labels=["collect_failure_rate"],
executed_block_associations=(replacement_association,),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="progress_observed",
reason_code="verified_success",
workflow_run_id="wr_replacement",
executed_block_labels=["collect_failure_rate"],
executed_block_associations=(replacement_association,),
completed_block_associations=(replacement_association,),
verified_progress_marker="run-completed",
),
)
# The replacement carries a fresh identity, so the recorded association can never run again;
# the label's current identity completing is the only route by which this repair can clear.
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_code_block_association_covers_conditional_blocks_without_entering_packet_projection() -> None:
workflow_yaml = """workflow_definition:
parameters: []
blocks:
- block_type: conditional
label: inspect_state
branch_conditions:
- condition: "True"
blocks:
- block_type: code
label: read_failure_rate
code: |
return await page.locator("canvas.failure-rate").inner_text()
"""
associations = runner_code_block_associations(workflow_yaml)
assert set(associations) == {"read_failure_rate"}
assert "copilot_block_association" not in workflow_yaml
outcome = recorded_outcome_from_run_blocks_result(
{
"ok": False,
"data": {
"workflow_run_id": "wr_conditional_failure",
"overall_status": "failed",
"failing_code_line": 1,
"blocks": [
{
"label": "read_failure_rate",
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
],
},
},
block_associations_by_label=associations,
)
assert outcome is not None
assert outcome.failed_operation is not None
assert outcome.failed_operation.block_association == associations["read_failure_rate"]
assert "copilot_block_association" not in outcome.model_dump_json()
def test_recorded_outcome_execution_receipts_come_from_block_status_not_requested_labels() -> None:
result = {
"ok": False,
"data": {
"workflow_run_id": "wr_receipts",
"overall_status": "failed",
"requested_block_labels": ["requested_only", "executed"],
"executed_block_labels": ["requested_only", "executed"],
"blocks": [
{"label": "requested_only", "status": "skipped"},
{
"label": "executed",
"status": "failed",
"failure_reason": "browser operation failed",
"error_codes": ["browser_operation_failed"],
},
],
},
}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.requested_block_labels == ["requested_only", "executed"]
assert outcome.executed_block_labels == ["executed"]
def test_browser_operation_failure_without_a_block_association_is_never_cleared() -> None:
failed_workflow = """title: retain browser failure
workflow_definition:
parameters: []
blocks:
- block_type: code
label: collect_failure_rate
copilot_block_association: cba_collect_failure_rate
code: |
value = await page.locator("canvas.failure-rate").inner_text()
return {"failure_rate": value}
"""
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_failed",
block_label="collect_failure_rate",
failing_line=1,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_failed",
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
verdict="not_authoritative",
reason_code="run_completed_unevaluated",
workflow_run_id="wr_later_completed_run",
requested_block_labels=["collect_failure_rate"],
executed_block_labels=["collect_failure_rate"],
executed_block_associations=("cba_collect_failure_rate",),
completed_block_associations=("cba_collect_failure_rate",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
def test_browser_operation_failure_is_not_cleared_by_an_untested_persisted_edit() -> None:
failed_workflow = """title: retain browser failure
workflow_definition:
parameters: []
blocks:
- block_type: code
label: collect_failure_rate
code: |
return {"failure_rate": await page.locator("canvas.failure-rate").inner_text()}
"""
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_failed",
block_label="collect_failure_rate",
failing_line=1,
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_failed",
requested_block_labels=["collect_failure_rate"],
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
ctx.workflow_yaml = failed_workflow.replace("canvas.failure-rate", "[data-testid='failure-rate']")
ctx.persisted_workflow_yaml = ctx.workflow_yaml
record_build_test_outcome(ctx, None)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
def test_failure_in_the_latest_run_keeps_the_stop_and_names_its_block() -> None:
ctx = _locator_packet_ctx()
ctx.runner_code_block_associations_by_label = {"continue_to_payment": "cba_continue_to_payment"}
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_latest",
workflow_run_block_id="wrb_latest",
block_label="continue_to_payment",
failing_line=3,
block_association="cba_continue_to_payment",
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_latest",
attempted_block_label="continue_to_payment",
block_labels=["search_tomorrow_trip", "continue_to_payment"],
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="not_authoritative",
reason_code="blocker_reported",
workflow_run_id="wr_latest",
block_labels=["search_tomorrow_trip", "continue_to_payment"],
),
)
latest = ctx.latest_recorded_build_test_outcome
assert latest is not None
assert latest.failed_operation == failed_operation
assert latest.completed_block_associations == ()
assert latest.failed_operation.kind == "browser_operation_failed"
assert latest.failed_operation.block_label == "continue_to_payment"
def test_stop_clears_when_the_failed_block_completed_even_though_a_later_block_failed() -> None:
ctx = _locator_packet_ctx()
ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in", "collect_titles": "cba_collect_titles"}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_login_failed",
block_labels=["log_in", "collect_titles"],
structural_failure_identity="browser-operation",
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_login_failed",
block_label="log_in",
failing_line=2,
block_association="cba_log_in",
),
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_collect_failed",
block_labels=["log_in", "collect_titles"],
executed_block_labels=["log_in", "collect_titles"],
executed_block_associations=("cba_log_in", "cba_collect_titles"),
completed_block_associations=("cba_log_in",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_post_run_enrichment_does_not_resurrect_a_released_stop() -> None:
ctx = _locator_packet_ctx()
ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_login_failed",
block_labels=["log_in"],
structural_failure_identity="browser-operation",
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_login_failed",
block_label="log_in",
failing_line=2,
block_association="cba_log_in",
),
),
)
released = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="not_authoritative",
reason_code="run_completed_unevaluated",
workflow_run_id="wr_login_completed",
block_labels=["log_in"],
executed_block_labels=["log_in"],
executed_block_associations=("cba_log_in",),
completed_block_associations=("cba_log_in",),
)
record_build_test_outcome(ctx, released)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
assert bind_post_run_page_evidence(
ctx,
{"workflow_run_id": "wr_login_completed"},
None,
regraded=released,
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_stop_holds_when_the_failed_block_only_terminated_in_the_later_run() -> None:
ctx = _locator_packet_ctx()
ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}
failed_operation = BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_login_failed",
block_label="log_in",
failing_line=2,
block_association="cba_log_in",
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_login_failed",
block_labels=["log_in"],
structural_failure_identity="browser-operation",
failed_operation=failed_operation,
),
)
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="not_authoritative",
reason_code="terminal_challenge_blocker",
workflow_run_id="wr_login_terminated",
block_labels=["log_in"],
executed_block_labels=["log_in"],
executed_block_associations=("cba_log_in",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
@pytest.mark.asyncio
async def test_browser_operation_outcome_clears_after_server_composed_verified_retest(
monkeypatch: pytest.MonkeyPatch,
) -> None:
failed_workflow = """title: read the failure rate
workflow_definition:
parameters: []
blocks:
- block_type: code
label: collect_failure_rate
copilot_block_association: cba_collect_failure_rate
code: |
value = await page.locator("canvas.failure-rate").inner_text()
return {"failure_rate": value}
"""
repaired_workflow = failed_workflow.replace(
'page.locator("canvas.failure-rate")',
"page.locator(\"[data-testid='failure-rate']\")",
)
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
ctx.runner_code_block_associations_by_label = {"collect_failure_rate": "cba_collect_failure_rate"}
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="runtime_block_failure",
workflow_run_id="wr_browser_operation",
block_labels=["collect_failure_rate"],
structural_failure_identity="browser-operation",
failed_operation=BuildTestFailedOperation(
kind="browser_operation_failed",
workflow_run_id="wr_browser_operation",
workflow_run_block_id="wrb_browser_operation",
block_label="collect_failure_rate",
failing_line=1,
block_association="cba_collect_failure_rate",
),
),
)
prompt = _recorded_build_test_outcome_prompt(ctx)
persisted_attempts = [failed_workflow]
tested_attempts: list[str] = []
async def persist_workflow(
payload: dict[str, object], copilot_ctx: CopilotContext, **_kwargs: object
) -> dict[str, object]:
candidate = payload["workflow_yaml"]
assert isinstance(candidate, str)
assert payload["_preserve_code_block_associations"] is True
persisted_attempts.append(candidate)
copilot_ctx.workflow_yaml = candidate
copilot_ctx.persisted_workflow_yaml = candidate
code = candidate.split("code: |", 1)[1]
workflow = SimpleNamespace(
proxy_location=None,
workflow_definition=SimpleNamespace(
blocks=[
SimpleNamespace(
label="collect_failure_rate",
block_type=SimpleNamespace(value="code"),
code=code,
)
]
),
)
return {"ok": True, "data": {"block_count": 1}, "_workflow": workflow}
async def test_persisted_workflow(
_params: dict[str, object], copilot_ctx: CopilotContext, **_kwargs: object
) -> dict[str, object]:
persisted = copilot_ctx.persisted_workflow_yaml
assert isinstance(persisted, str)
tested_attempts.append(persisted)
repaired_selector_present = "[data-testid='failure-rate']" in persisted
return {
"ok": repaired_selector_present,
"data": {
"workflow_run_id": "wr_retested_repair",
"overall_status": "completed" if repaired_selector_present else "failed",
"requested_block_labels": ["collect_failure_rate"],
"executed_block_labels": ["collect_failure_rate"],
"blocks": [
{
"label": "collect_failure_rate",
"status": "completed" if repaired_selector_present else "failed",
"extracted_data": {"failure_rate": "16.67%"} if repaired_selector_present else None,
}
],
},
}
async def observe_test_result(
copilot_ctx: CopilotContext, result: dict[str, object], _handler_start: float
) -> object:
record_build_test_outcome(
copilot_ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="edit_block_and_run",
verdict="progress_observed",
reason_code="verified_success",
workflow_run_id="wr_retested_repair",
executed_block_associations=("cba_collect_failure_rate",),
completed_block_associations=("cba_collect_failure_rate",),
verified_progress_marker="run-completed",
),
)
return copilot_ctx.latest_recorded_build_test_outcome
monkeypatch.setattr(tools_module, "_update_and_run_requires_skipped_run", lambda *args: False)
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
monkeypatch.setattr(tools_module, "_frontier_runtime_page_url", AsyncMock(return_value=None))
monkeypatch.setattr(
tools_module,
"_plan_frontier",
lambda *args: (["collect_failure_rate"], {}, "collect_failure_rate", "initial"),
)
monkeypatch.setattr(tools_module, "_update_workflow", persist_workflow)
monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", test_persisted_workflow)
monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", observe_test_result)
monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
tool_result = await tools_module.edit_block_and_run_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="edit_block_and_run"),
json.dumps(
{
"label": "collect_failure_rate",
"expected_code": 'value = await page.locator("canvas.failure-rate").inner_text()\nreturn {"failure_rate": value}',
"replacement_code": (
"value = await page.locator(\"[data-testid='failure-rate']\").inner_text()\n"
'return {"failure_rate": value}'
),
"block_labels": ["collect_failure_rate"],
"parameters": {},
}
),
)
parsed_result = json.loads(tool_result)
assert "browser_operation_failed" in prompt
assert "wrb_browser_operation" in prompt
assert "collect_failure_rate" in prompt
assert "failing_line=1" in prompt
assert persisted_attempts == [failed_workflow, repaired_workflow]
assert "canvas.failure-rate" in persisted_attempts[0]
assert "canvas.failure-rate" not in persisted_attempts[1]
assert tested_attempts == [repaired_workflow]
assert parsed_result["ok"] is True
assert parsed_result["data"]["workflow_run_id"] == "wr_retested_repair"
assert parsed_result["data"]["build_test_packet"]["run"] == {
"workflow_run_id": "wr_retested_repair",
"status": "completed",
}
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.workflow_run_id == "wr_retested_repair"
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_browser_operation_identity_is_bounded_by_the_shared_packet_projection() -> None:
long_identity = "x" * 200
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["workflow_run_id"] = long_identity
data["blocks"] = [
{
"workflow_run_block_id": long_identity,
"label": long_identity,
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
]
projected = project_build_test_packet_for_llm(build_test_evidence_packet(_locator_packet_ctx(), result))
assert projected.failure is not None
assert projected.failure.failed_operation is not None
operation = projected.failure.failed_operation
assert len(operation.workflow_run_id or "") == 160
assert len(operation.workflow_run_block_id or "") == 160
assert len(operation.block_label or "") == 160
assert all(
value.endswith("...")
for value in (operation.workflow_run_id, operation.workflow_run_block_id, operation.block_label)
if value
)
assert any("failure.failed_operation.workflow_run_id shortened" in notice for notice in projected.omission_notices)
assert any(
"failure.failed_operation.workflow_run_block_id shortened" in notice for notice in projected.omission_notices
)
assert any("failure.failed_operation.block_label shortened" in notice for notice in projected.omission_notices)
def test_connect_failure_identity_is_bounded_by_the_shared_packet_projection() -> None:
long_identity = "x" * 200
result = {
"ok": False,
"data": {
"overall_status": "setup_failed",
"blocks": [],
"build_test_connect_failure": BuildTestConnectFailure(
state="cdp_connect_failed",
workflow_run_id=long_identity,
workflow_run_block_id=long_identity,
task_id=long_identity,
browser_session_id=long_identity,
).model_dump(mode="json"),
},
}
projected = project_build_test_packet_for_llm(build_test_evidence_packet(_locator_packet_ctx(), result))
assert projected.failure is not None
assert projected.failure.connect_failure is not None
failure = projected.failure.connect_failure
identities = (
failure.workflow_run_id,
failure.workflow_run_block_id,
failure.task_id,
failure.browser_session_id,
)
assert all(len(identity or "") == 160 for identity in identities)
assert all((identity or "").endswith("...") for identity in identities)
assert sum("failure.connect_failure" in notice for notice in projected.omission_notices) == 4
def test_browser_operation_packet_names_each_unavailable_operation_identity() -> None:
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data.pop("workflow_run_id", None)
data["blocks"] = [
{
"label": None,
"status": "failed",
"error_codes": ["browser_operation_failed"],
}
]
data.pop("failing_code_line", None)
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert packet.failure is not None
assert packet.failure.failed_operation is not None
notices = packet.omission_notices
assert any("failure.failed_operation.workflow_run_id omitted" in notice for notice in notices)
assert any("failure.failed_operation.workflow_run_block_id omitted" in notice for notice in notices)
assert any("failure.failed_operation.block_label omitted" in notice for notice in notices)
assert any("failure.failed_operation.failing_line omitted" in notice for notice in notices)
def test_a_failure_the_runner_did_not_type_carries_neither_field() -> None:
packet = build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result(None))
assert packet.failure is not None
assert packet.failure.error_codes == []
assert packet.failure.failing_line is None
def test_native_failed_block_identity_survives_packet_projection_and_aggregate_compaction() -> None:
result = _failed_run_result(None)
data = result["data"]
assert isinstance(data, dict)
data["action_observations"] = ["click completed"]
data["blocks"] = [
{
"workflow_run_block_id": "wrb_native",
"task_id": "tsk_native",
"step_id": "stp_native",
"label": "native_task",
"block_type": "TASK",
"status": "failed",
"failure_reason": "Reached the maximum steps (30)",
}
]
outcome = recorded_outcome_from_run_blocks_result(result)
packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
projected = project_build_test_packet_for_llm(packet)
compacted = project_build_test_packet_for_llm(_oversized_packet(packet))
assert packet.failure is not None
assert projected.failure is not None
assert compacted.failure is not None
expected = ("wrb_native", "tsk_native", "stp_native", "TASK")
for failure in (packet.failure, projected.failure, compacted.failure):
assert (
failure.workflow_run_block_id,
failure.task_id,
failure.step_id,
failure.block_type,
) == expected
assert [notice for notice in projected.omission_notices if notice.startswith("failure.page_state omitted:")] == [
"failure.page_state omitted: no bounded same-run page state was recorded."
]
def test_historical_failure_packet_without_native_identities_remains_valid() -> None:
failure = BuildTestPacketFailure(block_label="code", block_status="failed", reason="boom")
assert failure.workflow_run_block_id is None
assert failure.task_id is None
assert failure.step_id is None
assert failure.block_type is None
def test_recorded_run_block_result_keeps_native_machine_identities() -> None:
row = SimpleNamespace(
workflow_run_block_id="wrb_native",
task_id="tsk_native",
label="native_task",
block_type=SimpleNamespace(name="TASK"),
status="failed",
failure_reason="Reached the maximum steps (30)",
error_codes=["max_steps_exceeded"],
output=None,
)
result = _recorded_run_block_result(row)
assert result == {
"workflow_run_block_id": "wrb_native",
"task_id": "tsk_native",
"label": "native_task",
"block_type": "TASK",
"status": "failed",
"failure_reason": "Reached the maximum steps (30)",
"error_codes": ["max_steps_exceeded"],
"output": None,
}
@pytest.fixture
def synthetic_native_actions_newest_first() -> list[SimpleNamespace]:
"""Synthetic rows exercise ordering and privacy without claiming production custody."""
return [
SimpleNamespace(
task_id="tsk_synthetic",
step_id="stp_newest",
action_type="click",
status="completed",
reasoning="private reasoning newest",
element_id="private-element-newest",
output=None,
response=None,
),
*[
SimpleNamespace(
task_id="tsk_synthetic",
step_id=f"stp_{number}",
action_type=action_type,
status="completed",
reasoning=f"private reasoning {number}",
element_id=f"private-element-{number}",
output=None,
response=None,
)
for number, action_type in [
(5, "scroll"),
(4, "wait"),
(3, "hover"),
(2, "select_option"),
(1, "input_text"),
(0, "goto_url"),
]
],
]
@pytest.mark.asyncio
async def test_native_actions_use_newest_step_and_keep_newest_six_chronological(
monkeypatch: pytest.MonkeyPatch,
synthetic_native_actions_newest_first: list[SimpleNamespace],
) -> None:
block = SimpleNamespace(task_id="tsk_synthetic")
result = {"status": "failed"}
fake_app = SimpleNamespace(
DATABASE=SimpleNamespace(
tasks=SimpleNamespace(
get_recent_actions_for_tasks=AsyncMock(return_value=synthetic_native_actions_newest_first)
)
)
)
monkeypatch.setattr(run_execution_module, "app", fake_app)
await run_execution_module._attach_action_traces([block], [result], "org_native")
assert result["step_id"] == "stp_newest"
assert run_execution_module._retained_action_observations([result]) == [
"input_text completed",
"select_option completed",
"hover completed",
"wait completed",
"scroll completed",
"click completed",
]
summary = run_execution_module._failure_action_trace_summary(result)
assert summary == run_execution_module._retained_action_observations([result])
assert "private" not in str(summary)
assert "goto_url" not in str(summary)
def test_native_actions_bound_the_global_newest_slice_before_chronological_rendering() -> None:
results = [
{
"action_trace": [
{"action": "select_option", "status": "completed"},
{"action": "input_text", "status": "completed"},
{"action": "goto_url", "status": "completed"},
{"action": "reload_page", "status": "completed"},
]
},
{
"action_trace": [
{"action": "click", "status": "completed"},
{"action": "scroll", "status": "completed"},
{"action": "wait", "status": "completed"},
{"action": "hover", "status": "completed"},
]
},
]
assert run_execution_module._retained_action_observations(results) == [
"input_text completed",
"select_option completed",
"hover completed",
"wait completed",
"scroll completed",
"click completed",
]
def test_the_failing_line_is_read_from_the_recorders_own_stamp() -> None:
# The cold-repair path derives the line from the attached action trace rather than parsing a
# rendered sentence. Only the recorder's integer stamp counts: personalize_action writes user
# data to other action fields.
trace = [
{"action": "NULL_ACTION", "status": "failed", "code_line": 6},
{"action": "CLICK", "status": "completed"},
]
assert _failing_code_line(trace) == 6
def test_a_trace_without_a_line_stamp_yields_none_rather_than_a_guess() -> None:
assert _failing_code_line([{"action": "CLICK", "status": "failed"}]) is None
assert _failing_code_line([{"action": "CLICK", "status": "failed", "code_line": "6"}]) is None
assert _failing_code_line(None) is None
def test_the_projection_hands_every_consumer_an_already_redacted_packet() -> None:
"""This projection is the only redaction on the tool-result path, so a test that exercises the
redactor rather than the projection would let a simplification delete it in silence."""
from skyvern.forge.sdk.copilot.output_utils import project_build_test_packet_for_llm
result = {
"ok": False,
"data": {
"workflow_run_id": "wr_secret",
"overall_status": "failed",
"blocks": [
{
"label": "extract",
"status": "failed",
"failure_reason": "login rejected with password=hunter2",
"error_codes": ["user_code_error"],
}
],
},
}
projected = project_build_test_packet_for_llm(build_test_evidence_packet(_locator_packet_ctx(), result))
rendered = projected.model_dump_json()
assert "hunter2" not in rendered
assert "[REDACTED_SECRET]" in rendered
assert projected.run.workflow_run_id == "wr_secret"
assert projected.failure is not None
assert projected.failure.block_label == "extract"
assert projected.failure.error_codes == ["user_code_error"]
_REPEAT_FAILURE_REASON = "CodeBlock failed with NameError at line 1: name 'passenger_count' is not defined."
_REPEAT_RAISING_CODE = "total = passenger_count * fare\n"
_REPEAT_CHANGED_CODE = "total = int(await page.locator('#pax').input_value()) * fare\n"
def _record_failing_attempt(ctx: SimpleNamespace, code: str, workflow_run_id: str) -> dict:
"""Run one failing attempt of the same block against `code`, returning that run's own result.
The failing block registers a download, because a failing hand-back is not a minimal payload."""
ctx.workflow_yaml = _raising_code_workflow_yaml(code)
ctx.persisted_workflow_yaml = ctx.workflow_yaml
result = _generated_code_exception_result(
workflow_run_id,
_REPEAT_FAILURE_REASON,
downloaded_file_artifact_ids=[f"a_dl_{workflow_run_id}"],
)
record_build_test_outcome(
ctx,
recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
display_reason=_REPEAT_FAILURE_REASON,
workflow_run_id=workflow_run_id,
run_completed=False,
),
),
)
return result
def test_a_repeat_failing_run_reports_whether_the_code_it_ran_changed() -> None:
"""Nothing else answers it: the rendered source binding compares the latest attempt to the current
definition, which on a repeat failure reads as a match and says nothing about the prior attempt."""
ctx = _run_history_ctx(_raising_code_workflow_yaml(_REPEAT_RAISING_CODE))
first = _record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_1")
_carry_prior_attempt_change_identity_into_result(ctx, first, "edit_block_and_run")
assert "prior_attempt_change_identity" not in first["data"]
resubmitted = _record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_2")
_carry_prior_attempt_change_identity_into_result(ctx, resubmitted, "edit_block_and_run")
assert resubmitted["data"]["prior_attempt_change_identity"] == {
"prior_workflow_run_id": "wr_1",
"block_label": "book_trip",
"changed": False,
"basis": "code_hash",
}
repaired = _record_failing_attempt(ctx, _REPEAT_CHANGED_CODE, "wr_3")
_carry_prior_attempt_change_identity_into_result(ctx, repaired, "edit_block_and_run")
assert repaired["data"]["prior_attempt_change_identity"] == {
"prior_workflow_run_id": "wr_2",
"block_label": "book_trip",
"changed": True,
"basis": "code_hash",
}
assert repaired["data"]["blocks"][0]["extracted_data"] == {"downloaded_file_artifact_ids": ["a_dl_wr_3"]}
def test_legs_that_re_ran_no_code_carry_no_change_identity() -> None:
"""The validation and update legs return before any run, and a passing run after two failures is
handed back exactly as the run produced it."""
ctx = _run_history_ctx(_raising_code_workflow_yaml(_REPEAT_RAISING_CODE))
_record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_1")
_record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_2")
rejected = {"ok": False, "error": "block_labels must include the edited block 'book_trip'."}
_carry_prior_attempt_change_identity_into_result(ctx, rejected, "edit_block_and_run")
assert rejected == {"ok": False, "error": "block_labels must include the edited block 'book_trip'."}
passing = {"ok": True, "data": {"workflow_run_id": "wr_3", "overall_status": "completed"}}
_carry_prior_attempt_change_identity_into_result(ctx, passing, "edit_block_and_run")
assert passing == {"ok": True, "data": {"workflow_run_id": "wr_3", "overall_status": "completed"}}
def test_the_change_identity_survives_sanitization_and_compaction() -> None:
"""The model may reach the repair several turns later, after the output has been compacted."""
ctx = _run_history_ctx(_raising_code_workflow_yaml(_REPEAT_RAISING_CODE))
_record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_1")
resubmitted = _record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_2")
_carry_prior_attempt_change_identity_into_result(ctx, resubmitted, "edit_block_and_run")
carried = resubmitted["data"]["prior_attempt_change_identity"]
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", resubmitted)
assert sanitized["data"]["prior_attempt_change_identity"] == carried
compacted = json.loads(_summarize_tool_output(json.dumps(resubmitted)))
assert compacted["prior_attempt_change_identity"] == carried
assert compacted["workflow_run_id"] == "wr_2"
assert compacted["overall_status"] == "failed"
assert compacted["blocks"][0]["failure_reason"] == _REPEAT_FAILURE_REASON
def test_an_attempt_recorded_before_this_field_existed_reports_unavailable_not_unchanged() -> None:
"""A turn already in flight when this ships has a prior entry with no code hash, and reporting
that silence as `changed: False` would accuse the model of resubmitting code nobody compared."""
ctx = _run_history_ctx(_raising_code_workflow_yaml(_REPEAT_RAISING_CODE))
_record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_1")
ctx.recorded_build_test_outcome_history[0].pop("attempted_block_code_hash")
resubmitted = _record_failing_attempt(ctx, _REPEAT_RAISING_CODE, "wr_2")
_carry_prior_attempt_change_identity_into_result(ctx, resubmitted, "edit_block_and_run")
assert resubmitted["data"]["prior_attempt_change_identity"] == {
"prior_workflow_run_id": "wr_1",
"block_label": "book_trip",
"changed": None,
"basis": "unavailable",
}
def _completed_output_run_result(retained_value: object, *, register_row: bool = True) -> dict[str, object]:
run_id = "wr_requested_output"
registered_row = {
"workflow_run_id": run_id,
"output_parameter_id": "op_payment_options",
"output_parameter_key": "collect_options_output",
"block_label": "collect_options",
"block_type": "code",
"value": retained_value,
}
return {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": ["collect_options"],
"executed_block_labels": ["collect_options"],
"blocks": [
{
"label": "collect_options",
"block_type": "code",
"status": "completed",
"output": retained_value,
"extracted_data": {"collect_options_output": retained_value},
}
],
"requested_output_parameter_definitions": [
{
"workflow_run_id": run_id,
"output_parameter_id": "op_payment_options",
"output_parameter_key": "collect_options_output",
"block_label": "collect_options",
"block_type": "code",
}
],
"registered_output_parameter_values": [registered_row] if register_row else [],
},
}
@pytest.mark.parametrize(
("case", "retained_value", "register_row"),
[
("no_row", {"payment_options": ["Visa"]}, False),
("null_value", None, True),
# An empty collection is a value the code returned. Whether "no options were offered" is
# the right answer is the model's call; reporting it absent would invite invented data.
("empty_object", {}, True),
("empty_list", [], True),
("run_owned_value", {"payment_options": ["Visa", "PayPal"]}, True),
],
)
def test_a_completed_run_keeps_registered_output_facts_model_owned(
case: str,
retained_value: object,
register_row: bool,
) -> None:
result = _completed_output_run_result(retained_value, register_row=register_row)
data = result["data"]
assert isinstance(data, dict)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_evaluated",
workflow_run_id="wr_requested_output",
run_completed=True,
),
registered_output_parameter_payloads=data["registered_output_parameter_values"],
)
assert outcome is not None
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
ordinary_input = _build_user_context(
workflow_yaml="",
chat_history_text="",
global_llm_context="",
debug_run_info_text=_prior_run_debug_text(packet),
user_message="Repair the recorded run.",
)
assert outcome.missing_requested_output_facts == []
assert outcome.verdict == "not_authoritative"
assert packet["unfinished_items"] == []
if register_row:
assert "collect_options" in ordinary_input
if case == "run_owned_value":
assert '"payment_options"' in ordinary_input
def test_snapshot_registered_row_with_regenerated_id_satisfies_requested_output_by_key() -> None:
result = _completed_output_run_result({"payment_options": ["Visa"]})
data = result["data"]
assert isinstance(data, dict)
registered = data["registered_output_parameter_values"]
assert isinstance(registered, list)
registered[0]["output_parameter_id"] = "op_snapshot_regenerated"
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(
verdict="not_evaluated",
workflow_run_id="wr_requested_output",
run_completed=True,
),
registered_output_parameter_payloads=registered,
)
assert outcome is not None
assert outcome.missing_requested_output_facts == []
assert outcome.verdict == "not_authoritative"
def test_row_only_registered_loop_output_reaches_the_model_visible_packet() -> None:
loop_value = [[{"record_number": "123"}], [{"record_number": "456"}]]
result = _completed_output_run_result(loop_value)
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0].pop("output")
blocks[0].pop("extracted_data")
packet = project_build_test_packet_for_llm(build_test_evidence_packet(_locator_packet_ctx(), result)).model_dump(
mode="json", exclude_none=True
)
ordinary_input = _build_user_context(
workflow_yaml="",
chat_history_text="",
global_llm_context="",
debug_run_info_text=_prior_run_debug_text(packet),
user_message="Repair the recorded run.",
)
assert packet["registered_outputs"] == [
{
"label": "collect_options",
"status": "completed",
"output": {"collect_options_output": loop_value},
"value_complete": True,
}
]
assert "123" in ordinary_input
assert "456" in ordinary_input
assert any("output-parameter rows" in notice for notice in packet["omission_notices"])
def test_partial_loop_fallback_precedes_twelve_recorded_outputs_and_has_no_fabricated_status() -> None:
loop_value = [[{"record_number": "123"}]]
result = _completed_output_run_result(loop_value)
data = result["data"]
assert isinstance(data, dict)
data["blocks"] = [
{"label": f"inner_{index}", "status": "completed", "output": {"index": index}} for index in range(12)
]
packet = project_build_test_packet_for_llm(build_test_evidence_packet(_locator_packet_ctx(), result)).model_dump(
mode="json", exclude_none=True
)
assert len(packet["registered_outputs"]) == 12
assert packet["registered_outputs"][0] == {
"label": "collect_options",
"output": {"collect_options_output": loop_value},
"value_complete": True,
}
assert any("registered_outputs shortened" in notice for notice in packet["omission_notices"])
def test_explicit_null_block_output_rejects_stale_registered_row_fallback() -> None:
result = _completed_output_run_result({"stale": True})
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = None
packet = build_test_evidence_packet(_locator_packet_ctx(), result).model_dump(mode="json", exclude_none=True)
assert packet["registered_outputs"] == []
assert any("explicit null workflow run block output" in notice for notice in packet["omission_notices"])
def test_a_declared_goal_path_the_run_left_empty_remains_a_fact_not_an_automatic_grade() -> None:
run_id = "wr_goal_paths"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
"collect_options": {
"claimed_outcomes": [
{"id": "options", "goal_value_paths": ["$.payment_options", "$.cart_line_item"]},
]
}
}
retained_value = {"url": "https://example.test/cart", "actions": ["click", "click"], "payment_options": ["Visa"]}
result: dict[str, object] = {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": ["collect_options"],
"executed_block_labels": ["collect_options"],
"blocks": [
{
"label": "collect_options",
"block_type": "code",
"status": "completed",
"extracted_data": retained_value,
}
],
},
}
_anti_bot, empty_data_blocks, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(
result, ctx
)
run_execution_module._record_build_test_outcome(
ctx,
result,
RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
goal_path_omissions,
)
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
assert empty_data_blocks is False
assert goal_path_omissions == [{"block_label": "collect_options", "output_path": "cart_line_item"}]
assert outcome.verdict == "not_authoritative"
assert outcome.reason_code == "run_completed_unevaluated"
assert outcome.missing_requested_output_facts == []
expected_observation = {
"kind": "requested_output_observation",
"label": "collect_options",
"output_path": "cart_line_item",
}
assert expected_observation in packet["unfinished_items"]
direct_packet = project_direct_test_handoff_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
assert expected_observation in direct_packet["unfinished_items"]
satisfied_result = json.loads(json.dumps(result))
satisfied_result["data"]["blocks"][0]["extracted_data"]["cart_line_item"] = {"qty": 1}
_anti_bot, satisfied_empty, _categories, satisfied_omissions = run_execution_module._analyze_run_blocks(
satisfied_result, ctx
)
assert satisfied_empty is False
assert satisfied_omissions == []
def test_a_top_level_array_goal_path_is_not_dropped_for_having_no_root() -> None:
"""``$[*].number`` normalizes to ``[].number``, whose root is empty; the path still binds."""
run_id = "wr_array_goal_path"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
"collect_rows": {"claimed_outcomes": [{"id": "rows", "goal_value_paths": ["$[*].number"]}]}
}
result: dict[str, object] = {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": ["collect_rows"],
"executed_block_labels": ["collect_rows"],
"blocks": [
{
"label": "collect_rows",
"block_type": "code",
"status": "completed",
"extracted_data": {"url": "https://example.test/rows"},
}
],
},
}
_anti_bot, empty_data_blocks, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(
result, ctx
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
declared_goal_path_omissions=goal_path_omissions,
)
assert outcome is not None
assert empty_data_blocks is False
assert outcome.verdict == "not_authoritative"
assert goal_path_omissions == [{"block_label": "collect_rows", "output_path": "[].number"}]
assert outcome.missing_requested_output_facts == []
def test_declared_path_omissions_do_not_create_repair_instructions() -> None:
run_id = "wr_shared_goal_path"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
label: {"claimed_outcomes": [{"id": label, "goal_value_paths": ["$.status"]}]}
for label in ("collect_first", "collect_second")
}
result: dict[str, object] = {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": ["collect_first", "collect_second"],
"executed_block_labels": ["collect_first", "collect_second"],
"blocks": [
{
"label": label,
"block_type": "code",
"status": "completed",
"extracted_data": {"url": f"https://example.test/{label}"},
}
for label in ("collect_first", "collect_second")
],
},
}
_anti_bot, _empty, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(result, ctx)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
declared_goal_path_omissions=goal_path_omissions,
)
assert outcome is not None
assert outcome.verdict == "not_authoritative"
assert outcome.missing_requested_output_facts == []
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
assert packet["unfinished_items"] == []
def test_a_long_declared_goal_path_reaches_the_factual_packet_uncut() -> None:
"""A clipped observation path names nothing for the acting model to interpret."""
run_id = "wr_long_goal_path"
long_path = "$.checkout.summary." + ".".join(f"level_{index:02d}" for index in range(15)) + ".amount_due"
normalized = long_path.removeprefix("$.")
# Wider than every ceiling the path crosses on its way to the model, and inside the one the
# facts themselves carry, so a survivor proves the projections and not a shorter fixture.
assert 160 < len(normalized) <= 180, "the fixture must exceed every downstream bound"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
"collect_total": {"claimed_outcomes": [{"id": "total", "goal_value_paths": [long_path]}]}
}
result: dict[str, object] = {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": ["collect_total"],
"executed_block_labels": ["collect_total"],
"blocks": [
{
"label": "collect_total",
"block_type": "code",
"status": "completed",
"extracted_data": {"url": "https://example.test/checkout"},
}
],
},
}
_anti_bot, _empty, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(result, ctx)
run_execution_module._record_build_test_outcome(
ctx,
result,
RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
goal_path_omissions,
)
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert goal_path_omissions == [{"block_label": "collect_total", "output_path": normalized}]
assert outcome.missing_requested_output_facts == []
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
expected_observation = {
"kind": "requested_output_observation",
"label": "collect_total",
"output_path": normalized,
}
assert expected_observation in packet["unfinished_items"]
direct_packet = project_direct_test_handoff_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
assert expected_observation in direct_packet["unfinished_items"]
def _goal_path_run_result(run_id: str, blocks: list[dict[str, object]]) -> dict[str, object]:
labels = [str(block["label"]) for block in blocks]
return {
"ok": True,
"data": {
"workflow_run_id": run_id,
"overall_status": "completed",
"requested_block_labels": labels,
"executed_block_labels": labels,
"blocks": blocks,
},
}
def test_a_sibling_blocks_complete_record_does_not_clear_another_blocks_omission() -> None:
"""A run must not be credited for a path one block omitted because a different block proved its own."""
run_id = "wr_sibling_record"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
"collect_options": {
"claimed_outcomes": [{"id": "options", "goal_value_paths": ["$.payment_options", "$.cart_line_item"]}]
}
}
result = _goal_path_run_result(
run_id,
[
{
"label": "collect_options",
"block_type": "code",
"status": "completed",
"extracted_data": {
"url": "https://example.test/cart",
"actions": ["click", "click"],
"payment_options": ["Visa"],
},
},
{
"label": "lookup_record",
"block_type": "code",
"status": "completed",
"extracted_data": {
"lookup_record_output": {
"entity_found": True,
"entity_name": "Jordan Example",
"record_number": "1234567890",
"items": [{"item_label": "Sample Practice", "status": "Active"}],
"overall_status": "Active",
}
},
},
],
)
_anti_bot, empty_data_blocks, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(
result, ctx
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
declared_goal_path_omissions=goal_path_omissions,
)
assert outcome is not None
assert goal_path_omissions == [{"block_label": "collect_options", "output_path": "cart_line_item"}]
assert empty_data_blocks is False
assert outcome.verdict == "not_authoritative"
def test_a_completed_block_that_retained_nothing_at_all_reports_every_declared_path() -> None:
"""Retaining no output is the strongest omission there is; it must not skip the arm that records it."""
run_id = "wr_null_extracted"
ctx = _locator_packet_ctx()
ctx.code_artifact_metadata = {
"collect_options": {
"claimed_outcomes": [{"id": "options", "goal_value_paths": ["$.payment_options", "$.cart_line_item"]}]
}
}
result = _goal_path_run_result(
run_id,
[{"label": "collect_options", "block_type": "code", "status": "completed", "extracted_data": None}],
)
_anti_bot, empty_data_blocks, _categories, goal_path_omissions = run_execution_module._analyze_run_blocks(
result, ctx
)
outcome = recorded_outcome_from_run_blocks_result(
result,
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id=run_id, run_completed=True),
declared_goal_path_omissions=goal_path_omissions,
)
assert outcome is not None
assert empty_data_blocks is True
assert [omission["output_path"] for omission in goal_path_omissions] == ["payment_options", "cart_line_item"]
assert outcome.verdict == "not_authoritative"
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
).model_dump(mode="json", exclude_none=True)
assert packet["unfinished_items"] == []
class _CheckoutPage:
"""The focused local fixture the repaired block reads; no browser, no network."""
_TEXT = {
"#cart-line-item": "Running Jacket / Size M / Qty 1 / 88.00",
"#payment-methods": "Card, Wallet, Pay in 4",
"#selected-product": "Running Jacket",
}
def __init__(self) -> None:
self.context = MagicMock()
async def inner_text(self, selector: str) -> str:
return self._TEXT[selector]
_UNREPAIRED_CODE = """
url = "https://example.test/checkout"
actions = ["goto", "click", "click"]
"""
_REPAIRED_CODE = """
url = "https://example.test/checkout"
actions = ["goto", "click", "click"]
line_item_text = await page.inner_text("#cart-line-item")
name, size, quantity, price = [part.strip() for part in line_item_text.split("/")]
cart_line_item = {"name": name, "size": size, "quantity": quantity, "price": price}
payment_options = [option.strip() for option in (await page.inner_text("#payment-methods")).split(",")]
selection_output = {"product": await page.inner_text("#selected-product"), "price": price}
"""
async def _execute_code_block(monkeypatch: pytest.MonkeyPatch, code: str) -> dict[str, object]:
"""Run the block through the real CodeBlock executor and return the output it produced."""
class FakeBrowserState:
def __init__(self) -> None:
self.browser_artifacts = BrowserArtifacts()
async def get_working_page(self) -> object:
return _CheckoutPage()
class FakeWorkflowRunContext:
values: dict[str, object] = {}
secrets: dict[str, object] = {}
include_secrets_in_templates = False
organization_id = None
workflow_title = "Checkout"
workflow_id = "w_checkout"
workflow_permanent_id = "wpid_checkout"
workflow_run_id = "wr_executor_witness"
browser_session_id = None
workflow_run_outputs: list[object] = []
workflow = None
def get_block_metadata(self, label: str | None) -> dict[str, object]:
return {}
def build_workflow_run_summary(self) -> str:
return ""
def mask_secrets_in_data(self, data: object, mask: str = "*****") -> object:
return data
async def noop(*args: object, **kwargs: object) -> None:
return None
async def browser_state(*args: object, **kwargs: object) -> FakeBrowserState:
return FakeBrowserState()
monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block", noop)
monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", browser_state)
monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda *args: FakeWorkflowRunContext())
monkeypatch.setattr(CodeBlock, "record_output_parameter_value", noop)
now = datetime.now(UTC)
block = CodeBlock(
label="collect_payment_options",
code=code,
output_parameter=OutputParameter(
parameter_type=ParameterType.OUTPUT,
key="collect_payment_options_output",
description="checkout facts",
output_parameter_id="op_checkout",
workflow_id="w_checkout",
created_at=now,
modified_at=now,
),
)
result = await block.execute(workflow_run_id="wr_executor_witness", workflow_run_block_id="")
assert result.success is True, "the fixture block must execute"
assert isinstance(result.output_parameter_value, dict)
return result.output_parameter_value
def _executor_witness_ctx() -> CopilotContext:
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.code_artifact_metadata = {
"collect_payment_options": {
"claimed_outcomes": [
{
"id": "checkout",
"goal_value_paths": ["$.cart_line_item", "$.payment_options", "$.selection_output"],
}
]
}
}
return ctx
@pytest.mark.asyncio
async def test_the_repaired_block_executes_and_the_run_owns_the_outputs_it_was_asked_for(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The whole chain on executed code: a completed run that owns nothing names the paths it owes,
and the repaired block's own execution output clears them."""
before_output = await _execute_code_block(monkeypatch, _UNREPAIRED_CODE)
after_output = await _execute_code_block(monkeypatch, _REPAIRED_CODE)
# The outputs under test came out of the executor, not out of a fixture file.
assert set(before_output) == {"url", "actions"}
assert after_output["payment_options"] == ["Card", "Wallet", "Pay in 4"]
assert after_output["cart_line_item"] == {
"name": "Running Jacket",
"size": "Size M",
"quantity": "Qty 1",
"price": "88.00",
}
def recorded(output: dict[str, object]) -> tuple[CopilotContext, object]:
ctx = _executor_witness_ctx()
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_executor_witness",
"overall_status": "completed",
"requested_block_labels": ["collect_payment_options"],
"executed_block_labels": ["collect_payment_options"],
"blocks": [
{
"label": "collect_payment_options",
"block_type": "code",
"status": "completed",
"output": output,
"extracted_data": output,
}
],
# The value the executor produced, on the surface the runtime registers it on.
"registered_output_parameter_values": [
{
"workflow_run_id": "wr_executor_witness",
"output_parameter_id": "op_checkout",
"output_parameter_key": "collect_payment_options_output",
"block_label": "collect_payment_options",
"block_type": "code",
"value": output,
}
],
},
}
run_execution_module._record_run_blocks_result(ctx, result, completion_verification=None)
return ctx, result
before_ctx, _before_result = recorded(before_output)
after_ctx, after_result = recorded(after_output)
before_outcome = before_ctx.latest_recorded_build_test_outcome
assert before_outcome is not None
assert before_outcome.verdict == "not_authoritative"
assert before_outcome.missing_requested_output_facts == []
after_outcome = after_ctx.latest_recorded_build_test_outcome
assert after_outcome is not None
assert after_outcome.missing_requested_output_facts == []
assert after_outcome.verdict == "not_authoritative"
# The corrected run hands back the values its own code produced.
packet = project_build_test_packet_for_llm(
build_test_evidence_packet(after_ctx, after_result, recorded_outcome=after_outcome)
).model_dump(mode="json", exclude_none=True)
assert packet["unfinished_items"] == []
run_owned = [
output["output"] for output in packet["registered_outputs"] if output["label"] == "collect_payment_options"
]
assert run_owned == [after_output], "the corrected run does not hand back the output its code produced"
@pytest.mark.asyncio
@pytest.mark.parametrize(("token_moved", "source_moved"), [(True, True), (False, True), (True, False)])
async def test_candidate_run_bind_uses_the_call_owned_source_and_token(
monkeypatch: pytest.MonkeyPatch,
token_moved: bool,
source_moved: bool,
) -> None:
ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
ctx.workflow_copilot_chat_id = "chat-1"
ctx.proposal_owner_turn_id = "turn-source"
ctx.proposal_revision = 3
ctx.staged_workflow_yaml = HANDBACK_WORKFLOW_YAML
captured: dict[str, object] = {}
async def bind_candidate_run(**kwargs: object) -> None:
captured.update(kwargs)
raise CopilotProposalConflictError("candidate changed", reason="token_changed")
async def prepare_workflow(**_kwargs: object) -> SimpleNamespace:
if token_moved:
ctx.proposal_owner_turn_id = "turn-newer"
ctx.proposal_revision = 1
if source_moved:
ctx.staged_workflow = ctx.staged_workflow.model_copy(update={"title": "Newer candidate"})
captured["ctx_at_bind"] = (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow)
return SimpleNamespace(
workflow_run_id="wr_source",
workflow_id="w_source",
browser_session_id="pbs_run",
sequential_credential_id=None,
)
forge_app.DATABASE.workflow_params.bind_workflow_copilot_candidate_run = bind_candidate_run
forge_app.DATABASE.workflows.soft_delete_workflow_by_id = AsyncMock()
forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
monkeypatch.setattr(workflow_service_module, "prepare_workflow", prepare_workflow)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert (result["ok"], result["conflict_reason"], result["adoption"]) == (False, "token_changed", "kept")
assert result["data"] == {"run_dispatched": False, "proposal_bind_refused": True}
assert captured["expected_owner_turn_id"] == "turn-source"
assert captured["expected_revision"] == 3
assert captured["expected_workflow_yaml"] == HANDBACK_WORKFLOW_YAML
assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == captured["ctx_at_bind"]
@pytest.mark.asyncio
async def test_canonical_run_does_not_bind_a_stale_proposal_token(
monkeypatch: pytest.MonkeyPatch,
) -> None:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=HANDBACK_WORKFLOW_YAML,
polled_status="completed",
terminal_blocks=[terminal_extraction_block("completed")],
)
bind_candidate_run = AsyncMock()
forge_app.DATABASE.workflow_params.bind_workflow_copilot_candidate_run = bind_candidate_run
ctx = make_copilot_ctx(
browser_session_id="pbs_chat",
workflow_copilot_chat_id="chat-1",
proposal_owner_turn_id="turn-stale",
proposal_revision=2,
)
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert result["ok"] is True, result
assert result["data"]["execution_source"]["source_kind"] == "canonical"
bind_candidate_run.assert_not_awaited()
assert harness["executor_cancelled"] is False
_NEWER_CANDIDATE_YAML = HANDBACK_WORKFLOW_YAML.replace("title: extraction example", "title: newer example")
_STORED_CODE_ARTIFACT_METADATA = {"extract_heading": {"goal_value_paths": ["heading"]}}
_OTHER_TURN_PRIVATE_SETTINGS = {
"totp_identifier": "[email protected]",
"webhook_callback_url": "https://hooks.test/done",
}
async def _install_real_candidate_cas(
monkeypatch: pytest.MonkeyPatch,
sqlite_engine: AsyncEngine,
*,
blockless_saved_workflow: bool = False,
) -> tuple[dict[str, Any], WorkflowParametersRepository, str]:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=HANDBACK_WORKFLOW_YAML,
polled_status="completed",
terminal_blocks=[terminal_extraction_block("completed")],
)
saved = harness["workflow"]
if blockless_saved_workflow:
saved = saved.model_copy(update={"workflow_definition": WorkflowDefinition(parameters=[], blocks=[])})
forge_app.DATABASE.workflows.get_workflow = AsyncMock(return_value=saved)
forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=saved)
forge_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=saved)
repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)
forge_app.DATABASE.workflow_params = repo
forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
chat = await repo.create_workflow_copilot_chat(organization_id="org-1", workflow_permanent_id="wfp-1")
turn_one = make_copilot_ctx(workflow_copilot_chat_id=chat.workflow_copilot_chat_id, turn_id="turn-one")
await publish_workflow_candidate(turn_one, workflow=harness["workflow"], workflow_yaml=HANDBACK_WORKFLOW_YAML)
return harness, repo, chat.workflow_copilot_chat_id
async def _restored_turn(chat_id: str, turn_id: str, **overrides: object) -> CopilotContext:
ctx = make_copilot_ctx(
browser_session_id="pbs_chat", workflow_copilot_chat_id=chat_id, turn_id=turn_id, **overrides
)
await restore_pending_workflow_proposal(ctx)
return ctx
async def _run_extract_heading(ctx: CopilotContext) -> dict[str, Any]:
ctx.frontier_resume_session_id = "pbs_run"
return await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
async def _publish_from_another_turn(chat_id: str, code_artifact_metadata: dict[str, Any] | None = None) -> None:
turn_b = await _restored_turn(chat_id, "turn-b")
turn_b.code_artifact_metadata = code_artifact_metadata
turn_b.authored_private_workflow_settings = dict(_OTHER_TURN_PRIVATE_SETTINGS)
assert turn_b.staged_workflow is not None
newer = turn_b.staged_workflow.model_copy(update={"title": "newer example"})
await publish_workflow_candidate(turn_b, workflow=newer, workflow_yaml=_NEWER_CANDIDATE_YAML)
async def _stored_candidate(
repo: WorkflowParametersRepository, chat_id: str
) -> tuple[dict[str, Any], CopilotProposalMetadata]:
chat = await repo.get_workflow_copilot_chat_by_id(organization_id="org-1", workflow_copilot_chat_id=chat_id)
assert chat is not None and isinstance(chat.proposed_workflow, dict)
metadata = copilot_proposal_metadata(chat.proposed_workflow)
assert metadata is not None
return chat.proposed_workflow, metadata
async def _next_model_response(ctx: CopilotContext) -> None:
await CopilotRunHooks(ctx).on_llm_end(MagicMock(), MagicMock(), MagicMock(output=[]))
def _bind_lost_logs(logs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [entry for entry in logs if entry["event"] == "copilot candidate run bind lost"]
@pytest.mark.asyncio
@pytest.mark.parametrize("case", ["as_stored", "other_settings", "cleared", "saved_changed"])
async def test_candidate_run_bind_after_a_rejected_restore_dispatches_only_the_stored_prior_draft(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, case: str
) -> None:
_harness, repo, chat_id = await _install_real_candidate_cas(
monkeypatch, sqlite_engine, blockless_saved_workflow=True
)
stored, _ = await _stored_candidate(repo, chat_id)
if case == "saved_changed":
changed = forge_app.DATABASE.workflows.get_workflow_by_permanent_id.return_value.model_copy(
update={"description": "edited after the proposal"}
)
forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=changed)
ctx = await _restored_turn(
chat_id,
"turn-two",
workflow_yaml=HANDBACK_WORKFLOW_YAML.replace("extract_heading", "user_edited_block"),
prior_copilot_workflow_yaml=stored["_copilot_yaml"],
private_workflow_settings={"webhook_callback_url": "https://hooks.test/request"}
if case == "other_settings"
else {},
)
assert (ctx.staged_workflow, ctx.proposal_owner_turn_id, ctx.proposal_revision) == (None, "turn-one", 1)
if case == "cleared":
await repo.clear_workflow_copilot_candidate(
"org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
)
settings_before = copy.deepcopy(ctx.private_workflow_settings)
with capture_logs() as logs:
first = await _run_extract_heading(ctx)
assert first["ok"] is (case == "as_stored"), first
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == (1 if case == "as_stored" else 0)
if case == "saved_changed":
assert (first["conflict_reason"], first["adoption"]) == ("bytes_changed", "canonical_fingerprint_mismatch")
elif case == "cleared":
assert (first["conflict_reason"], first["adoption"]) == ("candidate_missing", "cleared")
assert [entry["reason"] for entry in _bind_lost_logs(logs)] == ["candidate_missing"]
await _next_model_response(ctx)
assert (await _run_extract_heading(ctx))["ok"] is False
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
elif case == "other_settings":
second = await _run_extract_heading(ctx)
assert [first, second] == [{"ok": False, "error": COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR}] * 2
workflow_service_module.prepare_workflow.assert_not_awaited()
assert ctx.private_workflow_settings == settings_before
assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == ("turn-one", 1, None)
assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id is None
else:
assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id == "wr_paused"
@pytest.mark.asyncio
async def test_candidate_run_bind_and_publish_stay_refused_while_an_accept_claim_is_live(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine
) -> None:
_harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
ctx = await _restored_turn(chat_id, "turn-a")
passed = await _run_extract_heading(ctx)
assert passed["ok"] is True, passed
await repo.claim_workflow_copilot_candidate(
"org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
)
claimed_before, _ = await _stored_candidate(repo, chat_id)
staged_before, outcome_before, trace_before = ctx.staged_workflow, ctx.last_run_outcome, list(ctx.run_outcome_trace)
ctx.last_test_ok, ctx.frontier_resume_session_id = True, "pbs_run"
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)
monkeypatch.setattr(run_execution_module, "_authority_tool_error", lambda *_args: None)
monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
monkeypatch.setattr(
tools_module, "_plan_frontier", lambda *_args: (["extract_heading"], {}, "extract_heading", "initial")
)
workflow_service_module.prepare_workflow.return_value = SimpleNamespace(
workflow_run_id="wr_refused",
workflow_id="w_source",
browser_session_id="pbs_run",
sequential_credential_id=None,
)
tool_context = SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug")
arguments = json.dumps({"block_labels": ["extract_heading"], "parameters": {}})
edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
with capture_logs() as logs:
runs = [json.loads(await tools_module.run_blocks_tool.on_invoke_tool(tool_context, arguments))]
runs.append(await _run_extract_heading(ctx))
edits = [await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True) for _ in "ab"]
assert [(result["ok"], result["conflict_reason"], result["adoption"]) for result in runs + edits] == [
(False, "being_accepted", "claimed")
] * 4
assert BUILD_TEST_PACKET_KEY not in runs[0]["data"]
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 1
assert [(entry["reason"], entry["adoption"]) for entry in _bind_lost_logs(logs)] == [
("being_accepted", "claimed")
] * 2
assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == ("turn-one", 1, staged_before)
assert (ctx.last_test_ok, ctx.last_run_outcome, ctx.run_outcome_trace) == (True, outcome_before, trace_before)
assert (await _stored_candidate(repo, chat_id))[0] == claimed_before
@pytest.mark.asyncio
@pytest.mark.parametrize(
("reason", "event"),
[("bytes_changed", "copilot candidate run bind lost"), ("unexpected_error", "copilot candidate run bind failed")],
)
async def test_candidate_run_bind_refusal_logs_its_reason_and_dispatches_nothing(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, reason: str, event: str
) -> None:
_harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
ctx = await _restored_turn(chat_id, "turn-a")
if reason == "bytes_changed":
ctx.staged_workflow_yaml = HANDBACK_WORKFLOW_YAML.replace("https://example.com", "https://example.org")
else:
monkeypatch.setattr(repo, "bind_workflow_copilot_candidate_run", AsyncMock(side_effect=RuntimeError("down")))
with capture_logs() as logs:
result = await _run_extract_heading(ctx)
assert (result["ok"], result.get("conflict_reason")) == (False, reason if reason == "bytes_changed" else None)
assert result["data"] == {"run_dispatched": False, "proposal_bind_refused": True}
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
bind_logs = [entry for entry in logs if entry["event"].startswith("copilot candidate run bind")]
assert [(entry["event"], entry["reason"]) for entry in bind_logs] == [(event, reason)]
assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id is None
_ACCEPTED_WEBHOOK = "https://hooks.example.test/accepted"
async def _accept_concurrently(harness: dict[str, Any], repo: WorkflowParametersRepository, chat_id: str) -> Workflow:
accepted = harness["workflow"].model_copy(
update={"workflow_id": "w_accepted", "webhook_callback_url": _ACCEPTED_WEBHOOK}
)
versions = {harness["workflow"].workflow_id: harness["workflow"], accepted.workflow_id: accepted}
forge_app.DATABASE.workflows.get_workflow = AsyncMock(
side_effect=lambda workflow_id, organization_id: versions.get(workflow_id)
)
forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=accepted)
forge_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=accepted)
await repo.clear_workflow_copilot_candidate(
"org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
)
return accepted
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["test", "edit"])
@pytest.mark.parametrize("concurrent", ["publish", "accept"])
async def test_stale_proposal_token_conflict_recovers_so_the_next_call_succeeds(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, path: str, concurrent: str
) -> None:
harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
parameters = harness["workflow"].workflow_definition.parameters
outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
ctx = await _restored_turn(
chat_id,
"turn-a",
workflow_id=harness["workflow"].workflow_id,
private_workflow_settings={"webhook_callback_url": "https://hooks.example.test/stale"}
if path == "edit"
else {},
)
ctx.executed_block_fingerprints = workflow_block_fingerprints(HANDBACK_WORKFLOW_YAML)
ctx.verified_prefix_labels = ["extract_heading"]
ctx.workflow_verification_evidence.full_workflow_verified = True
if concurrent == "publish":
await _publish_from_another_turn(chat_id, code_artifact_metadata=_STORED_CODE_ARTIFACT_METADATA)
expected = ("token_changed", "adopted")
else:
accepted = await _accept_concurrently(harness, repo, chat_id)
expected = ("candidate_missing", "cleared")
edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
async def test_or_edit() -> dict[str, Any]:
if path == "test":
return await _run_extract_heading(ctx)
return await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True)
with capture_logs() as logs:
first = await test_or_edit()
assert (first["ok"], first["conflict_reason"], first["adoption"]) == (False, *expected)
conflict_events = {"copilot candidate run bind lost", "copilot_candidate_publication_conflict"}
assert [(entry["reason"], entry["adoption"]) for entry in logs if entry["event"] in conflict_events] == [expected]
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
if concurrent == "publish":
assert first["adopted_workflow_yaml"] == _NEWER_CANDIDATE_YAML
assert (ctx.proposal_owner_turn_id, ctx.proposal_revision) == ("turn-b", 1)
assert ctx.code_artifact_metadata == _STORED_CODE_ARTIFACT_METADATA
assert (ctx.verified_prefix_labels, ctx.workflow_verification_evidence.full_workflow_verified) == ([], False)
review = _review_projection_for(ctx, ctx.staged_workflow_yaml)
assert review is not None and [block["neverTested"] for block in review["blocks"]] == [True]
else:
assert ctx.workflow_yaml == workflow_to_copilot_yaml(accepted)
await _next_model_response(ctx)
second = await test_or_edit()
assert second["ok"] is True, second
if path == "test":
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 1
if concurrent == "accept":
resolved_workflow_id = workflow_service_module.prepare_workflow.await_args.kwargs["resolved_workflow_id"]
assert resolved_workflow_id == accepted.workflow_id
return
create_draft = forge_app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version
dispatched = create_draft.await_args.kwargs["runtime_workflow"]
assert [dispatched.totp_identifier, dispatched.webhook_callback_url] == [*_OTHER_TURN_PRIVATE_SETTINGS.values()]
return
stored, metadata = await _stored_candidate(repo, chat_id)
assert (metadata.owner_turn_id, metadata.revision) == ("turn-a", 1)
assert "Extract the page title." in stored["_copilot_yaml"]
if concurrent == "publish":
assert stored[COPILOT_PRIVATE_SETTINGS_KEY] == _OTHER_TURN_PRIVATE_SETTINGS
else:
assert ctx.staged_workflow is not None and ctx.staged_workflow.webhook_callback_url == _ACCEPTED_WEBHOOK
assert stored.get("webhook_callback_url") is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
("first_call", "second_call", "overlap"),
[
("edit", "edit", True),
("test", "edit", True),
("edit", "edit", False),
("edit", "test", False),
("test", "test", False),
],
)
async def test_stale_proposal_token_adoption_refuses_a_call_written_before_it(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, first_call: str, second_call: str, overlap: bool
) -> None:
harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
parameters = harness["workflow"].workflow_definition.parameters
outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
ctx = await _restored_turn(chat_id, "turn-a", workflow_id=harness["workflow"].workflow_id)
await _publish_from_another_turn(chat_id)
stored_before, _ = await _stored_candidate(repo, chat_id)
edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
stale_metadata = {"extract_heading": {"goal_value_paths": ["stale"]}}
def call(kind: str, metadata: object = None) -> Coroutine[Any, Any, dict[str, Any]]:
if kind == "test":
return _run_extract_heading(ctx)
params = {"workflow_yaml": edit, "code_artifact_metadata": metadata}
return _update_workflow(params, ctx, allow_missing_credentials=True)
if overlap:
results = await asyncio.gather(call(first_call), call(second_call, stale_metadata))
else:
# The second call of the same model response starts after the first adopted, as when a wrapper waits on
# a lock or the credential gate before reaching the write.
results = [await call(first_call)]
submission_before = (ctx.raw_code_artifact_metadata, ctx.code_artifact_metadata)
results.append(await call(second_call, stale_metadata))
assert (ctx.raw_code_artifact_metadata, ctx.code_artifact_metadata) == submission_before
assert [(result["ok"], result["conflict_reason"], result["adoption"]) for result in results] == [
(False, "token_changed", "adopted")
] * 2
assert (await _stored_candidate(repo, chat_id))[0] == stored_before
assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("dropped_by", ["in_flight_edit", "earlier_edit"])
async def test_stale_proposal_token_adoption_keeps_the_saved_completion_contract(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, dropped_by: str
) -> None:
harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
parameters = harness["workflow"].workflow_definition.parameters
outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
ctx = await _restored_turn(chat_id, "turn-a", workflow_id=harness["workflow"].workflow_id)
ctx.code_artifact_metadata = {
"extract_heading": {"completion_criteria": [{"deliverable_kind": "generated_file", "id": "report"}]}
}
contract_dropping_edit = HANDBACK_WORKFLOW_YAML.replace(
" parameters: []\n",
" parameters: []\n completion_contract:\n schema_version: 1\n criteria:\n"
" - id: report\n kind: generated_file\n min_count: 1\n",
)
if dropped_by == "earlier_edit":
dropped = await _update_workflow({"workflow_yaml": contract_dropping_edit}, ctx, allow_missing_credentials=True)
assert dropped["ok"] is True and ctx.clear_persisted_completion_contract is True
await _publish_from_another_turn(chat_id)
adopting = await _run_extract_heading(ctx)
else:
await _publish_from_another_turn(chat_id)
sibling_adopted = asyncio.Event()
real_canonicalize = workflow_update_module.canonicalize_named_google_sheet_bindings
async def canonicalize_after_the_sibling_adopts(workflow_yaml: str, call_ctx: object) -> object:
# Holds the contract-dropping edit past its entry check until its sibling has adopted.
if "completion_contract" in workflow_yaml:
await sibling_adopted.wait()
return await real_canonicalize(workflow_yaml, call_ctx)
monkeypatch.setattr(
workflow_update_module, "canonicalize_named_google_sheet_bindings", canonicalize_after_the_sibling_adopts
)
async def sibling_edit() -> dict[str, Any]:
try:
return await _update_workflow(
{"workflow_yaml": HANDBACK_WORKFLOW_YAML}, ctx, allow_missing_credentials=True
)
finally:
sibling_adopted.set()
adopting, in_flight = await asyncio.gather(
sibling_edit(),
_update_workflow({"workflow_yaml": contract_dropping_edit}, ctx, allow_missing_credentials=True),
)
assert (in_flight["ok"], in_flight["adoption"]) == (False, "adopted")
assert (adopting["ok"], adopting["adoption"]) == (False, "adopted")
await _next_model_response(ctx)
edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
assert (await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True))["ok"] is True
assert (ctx.has_staged_proposal, ctx.clear_persisted_completion_contract) == (True, False)
@pytest.mark.asyncio
@pytest.mark.parametrize("exit_path", ["auto_accept", "cancel"])
async def test_stale_proposal_token_adoption_leaves_the_stored_candidate_untouched_at_turn_end(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, exit_path: str
) -> None:
_harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
turn_start_chat = await repo.update_workflow_copilot_chat(
organization_id="org-1", workflow_copilot_chat_id=chat_id, auto_accept=exit_path == "auto_accept"
)
assert turn_start_chat is not None
ctx = await _restored_turn(chat_id, "turn-a")
ctx.staged_workflow_yaml = HANDBACK_WORKFLOW_YAML.replace("https://example.com", "https://example.org")
conflict = await _run_extract_heading(ctx)
assert (conflict["conflict_reason"], conflict["adoption"]) == ("bytes_changed", "adopted")
await _next_model_response(ctx)
assert (await _run_extract_heading(ctx))["ok"] is True
stored_before, _ = await _stored_candidate(repo, chat_id)
if exit_path == "cancel":
await workflow_copilot_route._persist_cancel_turn(
stream=MagicMock(send=AsyncMock(return_value=True)),
chat=turn_start_chat,
organization_id="org-1",
original_workflow=None,
user_message="stop",
agent_result=_build_wip_exit_result(ctx, None, cancelled=True),
turn_id="turn-a",
)
else:
await workflow_copilot_route._persist_proposed_workflow_state(
turn_start_chat, _build_wip_exit_result(ctx, None), restored=False
)
assert (await _stored_candidate(repo, chat_id))[0] == stored_before
@pytest.mark.asyncio
@pytest.mark.parametrize("withheld_yaml", [False, True])
async def test_completed_run_is_recorded_before_the_browser_enrichment_await(
monkeypatch: pytest.MonkeyPatch,
withheld_yaml: bool,
) -> None:
completed_block = terminal_extraction_block("completed").model_copy(update={"output": {"heading": "Done"}})
ctx = await handback_ctx(
monkeypatch,
polled_status="completed",
block_status="completed",
terminal_blocks=[completed_block],
)
if withheld_yaml:
ctx.workflow_yaml = None
order: list[str] = []
projected_rows_at_commit: list[list[dict[str, object]]] = []
real_record = run_execution_module.record_build_test_outcome
real_commit = run_execution_module._commit_run_blocks_record
def _record(record_ctx: object, outcome: object) -> None:
order.append("record")
real_record(record_ctx, outcome)
def _commit(commit_ctx: CopilotContext, result: dict[str, object]) -> RecordedRunOutcome | None:
packet = build_test_evidence_packet(commit_ctx, result)
projected_rows_at_commit.append(
[output.model_dump(mode="json", exclude_none=True) for output in packet.registered_outputs]
)
return real_commit(commit_ctx, result)
async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
order.append("enrichment")
return "", None
monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)
monkeypatch.setattr(run_execution_module, "_commit_run_blocks_record", _commit)
monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert result["ok"] is True, result
assert order == ["record", "enrichment"]
assert projected_rows_at_commit == [
[
{
"label": "extract_heading",
"status": "completed",
"output": {"heading": "Done"},
"value_complete": True,
}
]
]
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.workflow_run_id == "wr_paused"
@pytest.mark.asyncio
async def test_failed_run_still_records_the_failure_and_marks_the_post_run_page(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = await handback_ctx(monkeypatch, polled_status="failed", block_status="failed")
marks: list[str] = []
async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
return "", None
monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
monkeypatch.setattr(
run_execution_module,
"_mark_stored_post_run_failure_page",
lambda _ctx: marks.append("marked"),
)
counts = count_record_and_send(monkeypatch)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
await _verify_and_record_run_blocks_result(ctx, result, 0.0)
assert result["ok"] is False, result
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert outcome.workflow_run_id == "wr_paused"
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code != ""
assert marks == ["marked"]
assert counts == {"record": 1, "send": 1}
assert len(ctx.recorded_build_test_outcome_history) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("challenge", [False, True])
async def test_enrichment_facts_still_land_on_the_recorded_outcome_and_agree_with_history(
monkeypatch: pytest.MonkeyPatch,
challenge: bool,
) -> None:
ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
evidence = same_run_page_evidence()
async def _enrichment(
enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object
) -> tuple[str, dict[str, object]]:
enrich_ctx.composition_page_evidence = evidence
result_data["current_url"] = "https://example.com/done"
result_data["post_run_page_evidence"] = evidence
result_data["post_run_page_capture"] = {"status": "captured"}
if challenge:
monkeypatch.setattr(
run_execution_module,
"_terminal_challenge_evidence",
lambda *_args, **_kwargs: run_execution_module.TerminalChallengeEvidence(
source="artifact",
reason="Browser challenge",
challenge_evidence_source="artifact",
workflow_run_id="wr_paused",
block_labels=("extract_heading",),
),
)
return "https://example.com/done", evidence
streamed = []
async def capture_outcome(*_args, **kwargs):
streamed.append(kwargs)
monkeypatch.setattr(run_execution_module, "_send_run_outcome_update", capture_outcome)
monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
monkeypatch.setattr(
run_execution_module.app.AGENT_FUNCTION,
"captcha_solving_available",
AsyncMock(return_value=True),
)
counts = count_record_and_send(monkeypatch)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
returned = await _verify_and_record_run_blocks_result(ctx, result, 0.0)
assert result["ok"] is not challenge
assert streamed[0]["verdict"] == ctx.last_run_outcome.verdict
if challenge:
assert streamed[0]["verdict"] == "not_demonstrated"
outcome = ctx.latest_recorded_build_test_outcome
assert returned is outcome
assert outcome is not None
assert ctx.captcha_solver_available is True
assert ctx.captcha_solver_available_for_url == "https://example.com/done"
assert outcome.page_evidence_refs, outcome
assert outcome.page_capture is not None and outcome.page_capture.status == "captured"
assert counts == {"record": 1, "send": 1}
assert len(ctx.recorded_build_test_outcome_history) == 1
entry = ctx.recorded_build_test_outcome_history[-1]
assert entry["workflow_run_id"] == outcome.workflow_run_id
assert entry["verdict"] == outcome.verdict
assert entry["reason_code"] == outcome.reason_code
assert entry["structural_key"] == outcome.structural_key
assert outcome.is_authoritative is True
assert entry["is_authoritative"] is True
assert ctx.recorded_persisted_block_run_workflow_run_id == outcome.workflow_run_id
@pytest.mark.asyncio
async def test_failed_run_known_only_by_its_page_is_graded_after_enrichment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A failed block carrying no failure_reason has no structural identity until the post-run page
arrives, so grading it before enrichment records nothing at all."""
ctx = await handback_ctx(
monkeypatch,
polled_status="failed",
block_status="failed",
terminal_blocks=[page_only_failed_block()],
)
evidence = same_run_page_evidence()
async def _enrichment(
enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object
) -> tuple[str, dict[str, object]]:
enrich_ctx.composition_page_evidence = evidence
result_data["post_run_page_evidence"] = evidence
return "https://example.com/done", evidence
monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
counts = count_record_and_send(monkeypatch)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
returned = await _verify_and_record_run_blocks_result(ctx, result, 0.0)
outcome = ctx.latest_recorded_build_test_outcome
assert returned is outcome
assert outcome is not None, "the run's failure was lost because the page was unknown at record time"
assert outcome.verdict == "repairable_failure", outcome
assert outcome.page_evidence_refs, outcome
assert outcome.key_provenance.get("structural_failure_identity") != (
"no typed verification/page/output identity available"
)
assert counts["record"] == 1
assert len(ctx.recorded_build_test_outcome_history) == 1
entry = ctx.recorded_build_test_outcome_history[-1]
assert entry["verdict"] == outcome.verdict
assert entry["reason_code"] == outcome.reason_code
assert entry["structural_key"] == outcome.structural_key
assert entry["is_authoritative"] == outcome.is_authoritative
@pytest.mark.asyncio
async def test_watchdog_paused_result_is_recorded_before_the_captcha_probe(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = await handback_ctx(monkeypatch, polled_status="paused", block_status="completed")
record_build_test_outcome(ctx, failed_second_factor_run("wr_prior_failure"))
counts = count_record_and_send(monkeypatch)
observed_at_probe: list[RecordedBuildTestOutcome | None] = []
async def _cancelled_probe(*_args: object, **_kwargs: object) -> bool:
observed_at_probe.append(ctx.latest_recorded_build_test_outcome)
raise asyncio.CancelledError
monkeypatch.setattr(
run_execution_module.app.AGENT_FUNCTION,
"captcha_solving_available",
_cancelled_probe,
)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert result["ok"] is False, result
assert result["data"]["control_signal"]["kind"] == "watchdog_paused"
assert _INTERNAL_RUN_OUTCOME_RECORDED_KEY not in result
with pytest.raises(asyncio.CancelledError):
await _verify_and_record_run_blocks_result(ctx, result, 0.0)
assert len(observed_at_probe) == 1
assert result.execution.build_outcome is observed_at_probe[0]
at_probe = observed_at_probe[0]
assert at_probe is not None and at_probe.workflow_run_id == "wr_paused"
assert at_probe.verdict == "not_authoritative"
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None and outcome.workflow_run_id == "wr_paused"
assert counts == {"record": 1, "send": 1}
assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == [
"wr_prior_failure",
"wr_paused",
]
def test_not_evaluated_run_outcome_does_not_hide_failed_blocks() -> None:
outcome = recorded_outcome_from_run_blocks_result(
{
"ok": True,
"data": {
"workflow_run_id": "wr_2",
"blocks": [
{
"label": "open_page",
"block_type": "CODE",
"status": "failed",
"failure_reason": "CodeBlock failed while running user code.",
"error_codes": ["user_code_error"],
},
{
"label": "sign_in_and_read",
"block_type": "CODE",
"status": "failed",
"failure_reason": "CodeBlock failed while running user code.",
"error_codes": ["user_code_error"],
},
],
},
},
recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_2"),
)
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code == "runtime_block_failure"
assert outcome.failed_block_labels == ["open_page", "sign_in_and_read"]
@pytest.mark.asyncio
async def test_a_foreign_occupier_stops_the_dispatch_before_any_workflow_run_is_created(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# A mid-workflow block is the route that runs in the chat's browser; a head start mints its own.
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=SEARCH_THEN_SELECT_WORKFLOW_YAML,
polled_status="running",
dispatch_to_worker=True,
)
now = datetime.now(UTC)
sessions_manager = MagicMock()
sessions_manager.get_session = AsyncMock(
return_value=PersistentBrowserSession(
persistent_browser_session_id="pbs_chat",
organization_id="org-1",
runnable_id="wr_foreign",
runnable_type="workflow_run",
status="running",
created_at=now,
modified_at=now,
)
)
monkeypatch.setattr(forge_app, "PERSISTENT_SESSIONS_MANAGER", sessions_manager)
forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
forge_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
return_value=SimpleNamespace(
workflow_run_id="wr_foreign",
copilot_session_id="wcc_someone_else",
status=WorkflowRunStatus.running,
parent_workflow_run_id=None,
)
)
@asynccontextmanager
async def _attach(_ctx: CopilotContext) -> AsyncIterator[None]:
yield
monkeypatch.setattr(runtime, "mcp_browser_context", _attach)
monkeypatch.setattr(runtime, "_drop_browser_session_id_at_its_fixed_deadline", AsyncMock())
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.workflow_copilot_chat_id = "wcc_mine"
result = await _run_blocks_and_collect_debug({"block_labels": ["select_first_result"], "parameters": {}}, ctx)
workflow_service_module.prepare_workflow.assert_not_awaited()
harness["worker_execute"].assert_not_awaited()
forge_app.WORKFLOW_SERVICE.execute_workflow.assert_not_awaited()
forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final.assert_not_awaited()
assert result["ok"] is False, result
assert result["data"]["build_test_connect_failure"]["state"] == "occupied"
assert result["data"]["build_test_connect_failure"]["occupier_run_id"] == "wr_foreign"
assert result["data"]["executed_block_labels"] == []
def _two_block_search_failure_result() -> dict[str, Any]:
return {
"ok": False,
"data": {
"workflow_run_id": "wr_two_block",
"overall_status": "failed",
"browser_session_id": "pbs_new_session",
"run_detached_from_chat": False,
"requested_block_labels": ["run_search", "select_first_result"],
"executed_block_labels": ["run_search", "select_first_result"],
"failing_code_line": 9,
"blocks": [
{"label": "select_first_result", "status": "failed", "failure_reason": "TimeoutError: waiting"},
{"label": "run_search", "status": "completed"},
],
"action_trace_summary": ["click failed code_line=9"],
},
}
def test_failed_two_block_run_keeps_predecessor_page_state_and_final_action_in_the_packet() -> None:
result = _two_block_search_failure_result()
data = result["data"]
assert isinstance(data, dict)
data["observed_block_end_urls"] = {"run_search": "https://fixture.test/results/widget"}
data["per_block_action_observations"] = {
"run_search": ["click completed code_line=4"],
"select_first_result": ["wait failed code_line=9"],
}
data["action_observations"] = ["click completed code_line=4", "wait failed code_line=9"]
projected = project_build_test_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), result))
assert projected.run.workflow_run_id == "wr_two_block"
assert projected.observed_block_end_urls == {"run_search": "https://fixture.test/results/widget"}
assert projected.per_block_action_observations["run_search"] == ["click completed code_line=4"]
assert projected.per_block_action_observations["select_first_result"] == ["wait failed code_line=9"]
assert "click completed code_line=4" in projected.action_observations
assert projected.failure is not None
assert projected.failure.block_label == "select_first_result"
assert projected.failure.failing_line == 9
def test_new_session_run_leaves_the_predecessor_page_state_unavailable_instead_of_a_warm_result() -> None:
warm_url = "https://fixture.test/warm-results/stale"
ctx = make_copilot_ctx()
ctx.verified_prefix_block_end_urls = {"run_search": warm_url}
ctx.verified_prefix_block_end_session_id = "pbs_warm_session"
this_run = _two_block_search_failure_result()
this_run_data = this_run["data"]
assert isinstance(this_run_data, dict)
this_run_data["observed_block_end_urls"] = {"run_search": "https://fixture.test/this-run-results/widget"}
this_run_data["action_observations"] = ["wait failed code_line=9"]
filled = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(ctx, this_run))
assert filled.observed_block_end_urls == {"run_search": "https://fixture.test/this-run-results/widget"}
assert filled.run.browser_session_id == "pbs_new_session" != ctx.verified_prefix_block_end_session_id
unrecorded = _two_block_search_failure_result()
unrecorded_data = unrecorded["data"]
assert isinstance(unrecorded_data, dict)
unrecorded_data["action_observations"] = ["wait failed code_line=9"]
projected = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(ctx, unrecorded))
assert projected.observed_block_end_urls == {}
assert warm_url not in json.dumps(projected.model_dump(mode="json"))
assert OBSERVED_BLOCK_END_URLS_EMPTY in projected.omission_notices
def test_the_projection_separates_a_withheld_url_map_from_one_that_was_never_recorded() -> None:
ctx = make_copilot_ctx()
never_recorded = project_build_test_packet_for_llm(
build_test_evidence_packet(ctx, _two_block_search_failure_result())
)
assert OBSERVED_BLOCK_END_URLS_EMPTY in never_recorded.omission_notices
sensitive = _two_block_search_failure_result()
sensitive_data = sensitive["data"]
assert isinstance(sensitive_data, dict)
sensitive_data["block_fact_omission_notices"] = [OBSERVED_BLOCK_END_URLS_WITHHELD]
withheld = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, sensitive))
assert OBSERVED_BLOCK_END_URLS_WITHHELD in withheld.omission_notices
assert not any("no per-block end URL was recorded" in notice for notice in withheld.omission_notices)
def test_the_direct_handoff_keeps_a_withheld_url_map_from_being_reported_as_never_recorded() -> None:
sensitive = _two_block_search_failure_result()
sensitive_data = sensitive["data"]
assert isinstance(sensitive_data, dict)
sensitive_data["block_fact_omission_notices"] = [OBSERVED_BLOCK_END_URLS_WITHHELD]
projected = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), sensitive))
assert OBSERVED_BLOCK_END_URLS_WITHHELD in projected.omission_notices
assert not any("no per-block end URL was recorded" in notice for notice in projected.omission_notices)
def test_a_full_url_map_gives_way_before_the_workflow_readback_the_repair_turn_needs() -> None:
result = _two_block_search_failure_result()
data = result["data"]
assert isinstance(data, dict)
long_url = "https://fixture.test/results?q=" + "w" * 1900
data["observed_block_end_urls"] = {f"block_{index}": f"{long_url}&n={index}" for index in range(12)}
workflow_yaml = "workflow_definition:\n" + (" - block: x\n code: |\n " + "y" * 90 + "\n") * 280
ctx = make_copilot_ctx()
ctx.persisted_workflow_yaml = workflow_yaml
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
assert projected.canonical_workflow_yaml is not None
assert len(projected.canonical_workflow_yaml) == 30_000
assert list(projected.observed_block_end_urls) == ["block_10", "block_11"]
assert any("observed_block_end_urls shortened" in notice for notice in projected.omission_notices)
def test_the_projection_refuses_an_over_long_failure_final_url_rather_than_cutting_it() -> None:
over_long = "https://fixture.test/results?q=" + "w" * BLOCK_FACT_URL_MAX_CHARS
packet = build_test_evidence_packet(make_copilot_ctx(), _two_block_search_failure_result())
assert packet.failure is not None
packet = packet.model_copy(update={"failure": packet.failure.model_copy(update={"final_url": over_long})})
projected = project_build_test_packet_for_llm(packet)
assert projected.failure is not None
assert projected.failure.final_url is None
assert any(
"failure.final_url omitted: the recorded URL exceeded" in notice for notice in projected.omission_notices
)
assert over_long[:200] not in json.dumps(projected.model_dump(mode="json"), ensure_ascii=False)
def test_the_direct_handoff_names_both_the_blocks_it_omitted_and_the_ones_it_reduced() -> None:
result = _two_block_search_failure_result()
data = result["data"]
assert isinstance(data, dict)
data["observed_block_end_urls"] = {
"run_search": "https://svc:[email protected]/directory/results",
"select_first_result": "https://fixture.test/directory/results?q=cardiology",
}
projected = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), result))
assert projected.observed_block_end_urls == {"select_first_result": "https://fixture.test/directory/results"}
assert "hunter2" not in json.dumps(projected.model_dump(mode="json"), ensure_ascii=False)
assert (
"observed_block_end_urls omitted block(s) for the direct test handoff: "
"run_search: the recorded URL carried credentials in its host."
) in projected.omission_notices
assert (
"observed_block_end_urls reduced block(s) to their path for the direct test handoff: "
"select_first_result: the recorded URL carried a query or fragment."
) in projected.omission_notices
def test_the_direct_handoff_does_not_report_an_all_filtered_url_map_as_never_recorded() -> None:
result = _two_block_search_failure_result()
data = result["data"]
assert isinstance(data, dict)
data["observed_block_end_urls"] = {"run_search": "about:blank"}
projected = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), result))
assert projected.observed_block_end_urls == {}
assert any(
"no recorded per-block end URL reduced to a reportable origin" in notice
for notice in projected.omission_notices
)
assert not any("no per-block end URL was recorded" in notice for notice in projected.omission_notices)
_ELEMENT_TIMEOUT_CATEGORY = {"category": "ELEMENT_STATE_TIMEOUT", "confidence_float": 0.85}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("run_category_before_label", "model_sees"),
[(None, "absent"), ([_ELEMENT_TIMEOUT_CATEGORY], [_ELEMENT_TIMEOUT_CATEGORY])],
ids=["label_only", "label_after_a_category"],
)
async def test_the_model_reads_a_run_category_as_it_was_before_the_sign_in_label(
monkeypatch: pytest.MonkeyPatch, run_category_before_label: list[dict[str, Any]] | None, model_sees: object
) -> None:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=HANDBACK_WORKFLOW_YAML,
polled_status="terminated",
polled_failure_category=with_sign_in_form_visible(run_category_before_label),
terminal_blocks=[terminal_extraction_block("failed")],
)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert result["data"].get("failure_category", "absent") == model_sees
SEARCH_SELECT_BLOCK_TRACES: dict[str, list[dict[str, Any]]] = {
"select_first_result": [{"action": "wait", "status": "failed", "code_line": 9}],
"run_search": [{"action": "click", "status": "completed", "code_line": 4}],
"open_search": [{"action": "input_text", "status": "completed", "code_line": 2}],
}
def _stamp_block_traces(results: list[dict[str, Any]]) -> None:
"""Stamp each block's trace by label, so no assertion below rides on the producer's list order."""
for result in results:
trace = SEARCH_SELECT_BLOCK_TRACES.get(str(result.get("label") or ""))
if trace is not None:
result["action_trace"] = [dict(entry) for entry in trace]
@pytest.mark.asyncio
async def test_prior_run_hydration_mints_per_block_facts_and_drops_the_urls_for_a_credential_run(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def read_prior_run(*, workflow_parameters: list[dict[str, str]]) -> dict[str, Any]:
async def stamp_traces(
_rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
) -> None:
_stamp_block_traces(results)
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
run_result_block_row("select_first_result", "failed", "https://fixture.test/results/widget"),
run_result_block_row("run_search", "completed", "https://fixture.test/results/widget/page-1"),
],
workflow_parameters=workflow_parameters,
attach_action_traces=stamp_traces,
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert isinstance(result["data"], dict)
return result["data"]
ordinary = await read_prior_run(workflow_parameters=[])
credential_bearing = await read_prior_run(
workflow_parameters=[{"parameter_type": "aws_secret", "key": "site_password"}]
)
assert ordinary["observed_block_end_urls"] == {
"run_search": "https://fixture.test/results/widget/page-1",
"select_first_result": "https://fixture.test/results/widget",
}
assert ordinary["per_block_action_observations"]["run_search"] == ["click completed code_line=4"]
assert ordinary["per_block_action_observations"]["select_first_result"] == ["wait failed code_line=9"]
assert "block_fact_omission_notices" not in ordinary
assert "observed_block_end_urls" not in credential_bearing
assert credential_bearing["block_fact_omission_notices"] == [OBSERVED_BLOCK_END_URLS_WITHHELD]
# A withheld end URL is not a missing one.
assert "current_url_evidence" not in credential_bearing
@pytest.mark.asyncio
async def test_prior_run_hydration_keeps_the_predecessor_final_action_from_the_real_action_loader(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
run_result_block_row(
"select_first_result", "failed", "https://fixture.test/results/widget", task_id="tsk_select"
),
run_result_block_row(
"run_search", "completed", "https://fixture.test/results/widget/page-1", task_id="tsk_search"
),
],
recent_actions=[
run_result_action_row("tsk_select", ActionType.WAIT, ActionStatus.failed, code_line=9),
run_result_action_row("tsk_search", ActionType.CLICK, ActionStatus.completed),
],
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
data = result["data"]
assert isinstance(data, dict)
assert data["per_block_action_observations"]["run_search"] == ["click completed"]
assert data["per_block_action_observations"]["select_first_result"] == ["wait failed code_line=9"]
@pytest.mark.asyncio
async def test_a_failed_two_block_run_hands_the_predecessor_page_state_to_the_repair_input(
monkeypatch: pytest.MonkeyPatch,
) -> None:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=SEARCH_THEN_SELECT_WORKFLOW_YAML,
polled_status="failed",
terminal_blocks=[
terminal_extraction_block(
"failed", label="select_first_result", final_url="https://fixture.test/results/widget"
),
terminal_extraction_block(
"completed", label="run_search", final_url="https://fixture.test/results/widget/page-1"
),
],
)
async def stamp_traces(
_rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
) -> None:
_stamp_block_traces(results)
monkeypatch.setattr(run_execution_module, "_attach_action_traces", stamp_traces)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug(
{"block_labels": ["run_search", "select_first_result"], "parameters": {}}, ctx
)
data = result["data"]
assert {"observed_block_end_urls", "per_block_action_observations"} <= set(data)
assert data["observed_block_end_urls"]["run_search"] == "https://fixture.test/results/widget/page-1"
assert data["per_block_action_observations"]["run_search"] == ["click completed code_line=4"]
assert data["per_block_action_observations"]["select_first_result"] == ["wait failed code_line=9"]
assert ctx.verified_prefix_block_end_urls == {}
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
assert projected.observed_block_end_urls["run_search"] == "https://fixture.test/results/widget/page-1"
assert projected.per_block_action_observations["run_search"] == ["click completed code_line=4"]
OPEN_SEARCH_SELECT_WORKFLOW_YAML = """
title: open then search then select
workflow_definition:
parameters: []
blocks:
- block_type: extraction
label: open_search
url: https://fixture.test
data_extraction_goal: Extract the search form.
- block_type: extraction
label: run_search
data_extraction_goal: Extract the search results.
- block_type: extraction
label: select_first_result
data_extraction_goal: Extract the selected result.
"""
EXECUTION_ORDERED_LABELS = ["open_search", "run_search", "select_first_result"]
@pytest.mark.asyncio
async def test_the_live_run_path_orders_per_block_facts_so_the_newest_bound_keeps_the_failing_block(
monkeypatch: pytest.MonkeyPatch,
) -> None:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=OPEN_SEARCH_SELECT_WORKFLOW_YAML,
polled_status="failed",
terminal_blocks=[
terminal_extraction_block(
"failed", label="select_first_result", final_url="https://fixture.test/results/widget"
),
terminal_extraction_block("completed", label="run_search", final_url="https://fixture.test/search"),
terminal_extraction_block("completed", label="open_search", final_url="https://fixture.test/"),
],
)
async def stamp_traces(
_rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
) -> None:
_stamp_block_traces(results)
monkeypatch.setattr(run_execution_module, "_attach_action_traces", stamp_traces)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug(
{"block_labels": list(EXECUTION_ORDERED_LABELS), "parameters": {}}, ctx
)
data = result["data"]
assert list(data["observed_block_end_urls"]) == EXECUTION_ORDERED_LABELS
assert list(data["per_block_action_observations"]) == EXECUTION_ORDERED_LABELS
compacted = _compact_packet_for_aggregate_limit(build_test_evidence_packet(ctx, result), [])
assert compacted.observed_block_end_urls["select_first_result"] == "https://fixture.test/results/widget"
assert "open_search" not in compacted.observed_block_end_urls
@pytest.mark.asyncio
async def test_the_prior_run_hydration_path_orders_per_block_facts_the_same_way(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def stamp_traces(
_rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
) -> None:
_stamp_block_traces(results)
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
run_result_block_row("select_first_result", "failed", "https://fixture.test/results/widget"),
run_result_block_row("run_search", "completed", "https://fixture.test/search"),
run_result_block_row("open_search", "completed", "https://fixture.test/"),
],
attach_action_traces=stamp_traces,
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
data = result["data"]
assert isinstance(data, dict)
assert list(data["observed_block_end_urls"]) == EXECUTION_ORDERED_LABELS
assert list(data["per_block_action_observations"]) == EXECUTION_ORDERED_LABELS
TWO_FAILURE_BLOCK_TRACES: dict[str, list[dict[str, Any]]] = {
"select_first_result": [{"action": "wait", "status": "failed", "code_line": 9}],
"run_search": [{"action": "click", "status": "failed", "code_line": 4}],
}
async def _stamp_two_failure_traces(
_rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
) -> None:
for result in results:
trace = TWO_FAILURE_BLOCK_TRACES.get(str(result.get("label") or ""))
if trace is not None:
result["action_trace"] = [dict(entry) for entry in trace]
@pytest.mark.asyncio
async def test_the_live_run_path_attributes_a_two_failure_run_to_the_newest_failed_block(
monkeypatch: pytest.MonkeyPatch,
) -> None:
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=OPEN_SEARCH_SELECT_WORKFLOW_YAML,
polled_status="failed",
terminal_blocks=[
terminal_extraction_block(
"failed", label="select_first_result", final_url="https://fixture.test/results/widget"
),
terminal_extraction_block("failed", label="run_search", final_url="https://fixture.test/search"),
terminal_extraction_block("completed", label="open_search", final_url="https://fixture.test/"),
],
)
monkeypatch.setattr(run_execution_module, "_attach_action_traces", _stamp_two_failure_traces)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug(
{"block_labels": list(EXECUTION_ORDERED_LABELS), "parameters": {}}, ctx
)
data = result["data"]
assert data["failing_code_line"] == 9
assert data["action_trace_summary"] == ["wait failed code_line=9"]
packet = build_test_evidence_packet(ctx, result)
assert packet.failure is not None
assert packet.failure.block_label == "select_first_result"
@pytest.mark.asyncio
async def test_the_prior_run_hydration_path_attributes_a_two_failure_run_to_the_newest_failed_block(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
run_result_block_row("select_first_result", "failed", "https://fixture.test/results/widget"),
run_result_block_row("run_search", "failed", "https://fixture.test/search"),
run_result_block_row("open_search", "completed", "https://fixture.test/"),
],
attach_action_traces=_stamp_two_failure_traces,
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
data = result["data"]
assert isinstance(data, dict)
assert data["failing_code_line"] == 9
assert data["action_trace_summary"] == ["wait failed code_line=9"]
packet = build_test_evidence_packet(make_copilot_ctx(), result)
assert packet.failure is not None
assert packet.failure.block_label == "select_first_result"
@pytest.mark.asyncio
async def test_a_run_whose_every_end_url_is_refused_is_not_also_reported_as_never_recorded(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
run_result_block_row("select_first_result", "failed", "https://user:[email protected]/results"),
run_result_block_row("run_search", "completed", "https://user:[email protected]/search"),
],
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
data = result["data"]
assert isinstance(data, dict)
assert "observed_block_end_urls" not in data
assert OBSERVED_BLOCK_END_URLS_UNREPORTABLE in data["block_fact_omission_notices"]
projected = project_build_test_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), result))
assert OBSERVED_BLOCK_END_URLS_EMPTY not in projected.omission_notices
def test_the_screened_blocks_notice_stays_inside_the_budget_of_the_map_it_describes() -> None:
entries = [(f"block_{index:03d}", "the recorded URL carried a query or fragment") for index in range(200)]
rendered = labelled_url_screen_reasons(entries)
assert len(rendered) <= BLOCK_FACT_SCREEN_NOTICE_MAX_CHARS + len(
"; and 200 older block(s) left unnamed at the notice budget"
)
assert "block_199" in rendered
assert rendered.endswith("older block(s) left unnamed at the notice budget")
@pytest.mark.asyncio
async def test_the_run_results_labels_and_blocks_come_back_newest_last(monkeypatch: pytest.MonkeyPatch) -> None:
"""The repository hands the run's rows back newest-first, and the packet keeps the tail of every
list built from them: reported in repository order, a bound drops the failing block instead."""
predecessors = [f"step_{index}" for index in range(14)]
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[run_result_block_row("select_first_result", "failed", "https://fixture.test/results")]
+ [
run_result_block_row(label, "completed", f"https://fixture.test/{label}")
for label in reversed(predecessors)
],
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
data = result["data"]
assert isinstance(data, dict)
executed_last = predecessors + ["select_first_result"]
assert data["executed_block_labels"] == executed_last
assert data["requested_block_labels"] == executed_last
assert [block["label"] for block in data["blocks"]] == executed_last
assert list(data["observed_block_end_urls"])[-1] == "select_first_result"
def test_a_long_failing_block_cannot_erase_the_predecessor_blocks_final_action() -> None:
results = [
{
"label": "run_search",
"action_trace": [
{"action": "click", "status": "completed", "code_line": 4},
{"action": "input_text", "status": "completed", "code_line": 3},
],
},
{
"label": "select_first_result",
"action_trace": [{"action": "wait", "status": "failed", "code_line": 9}]
+ [{"action": "click", "status": "completed", "code_line": 7} for _ in range(5)],
},
]
assert run_execution_module._retained_action_observations(results) == [
"click completed code_line=4",
"click completed code_line=7",
"click completed code_line=7",
"click completed code_line=7",
"click completed code_line=7",
"wait failed code_line=9",
]
assert run_execution_module._retained_action_observations_by_label(results) == {
"run_search": ["click completed code_line=4"],
"select_first_result": [
"click completed code_line=7",
"click completed code_line=7",
"click completed code_line=7",
"click completed code_line=7",
"wait failed code_line=9",
],
}
def test_a_seven_block_run_reserves_the_newest_predecessors_and_leaves_the_failing_block_one_slot() -> None:
results = [
{
"label": f"predecessor_{index}",
"action_trace": [
{"action": "click", "status": "completed", "code_line": index},
{"action": "input_text", "status": "completed", "code_line": index + 100},
],
}
for index in range(6, 0, -1)
] + [
{
"label": "select_first_result",
"action_trace": [{"action": "wait", "status": "failed", "code_line": 90}]
+ [{"action": "click", "status": "completed", "code_line": 91} for _ in range(4)],
}
]
by_label = run_execution_module._retained_action_observations_by_label(results)
assert by_label["select_first_result"] == ["wait failed code_line=90"]
assert by_label["predecessor_1"] == ["click completed code_line=1"]
assert by_label["predecessor_5"] == ["click completed code_line=5"]
assert "predecessor_6" not in by_label
assert len(run_execution_module._retained_action_observations(results)) == 6
def test_a_native_block_with_many_failed_actions_leaves_the_predecessors_terminal_facts() -> None:
results = [
{"label": "login", "action_trace": [{"action": "click", "status": "completed"}]},
{"label": "open_orders", "action_trace": [{"action": "goto_url", "status": "completed"}]},
{
"label": "download_invoices",
"action_trace": [{"action": "click", "status": "failed"} for _ in range(7)],
},
]
by_label = run_execution_module._retained_action_observations_by_label(results)
assert by_label["login"] == ["click completed"]
assert by_label["open_orders"] == ["goto_url completed"]
assert by_label["download_invoices"] == ["click failed"] * 4
def test_predecessor_blocks_the_action_budget_drops_are_named_in_an_omission_notice() -> None:
results = [
{
"label": f"predecessor_{index}",
"action_trace": [{"action": "click", "status": "completed", "code_line": index}],
}
for index in range(1, 8)
] + [
{
"label": "select_first_result",
"action_trace": [{"action": "wait", "status": "failed", "code_line": 90}],
}
]
data: dict[str, Any] = {}
run_execution_module._attach_block_fact_projection(
data,
[],
run_execution_module._retained_action_observations_by_label(results),
unreported_predecessor_labels=run_execution_module._unreported_predecessor_labels(results),
)
assert "predecessor_1" not in data["per_block_action_observations"]
assert "predecessor_2" not in data["per_block_action_observations"]
notice = next(notice for notice in data["block_fact_omission_notices"] if "predecessor block(s)" in notice)
assert "2 predecessor block(s)" in notice
assert "predecessor_1" in notice
assert "predecessor_2" in notice
def test_a_label_a_loop_reused_keeps_its_newest_end_url_at_the_newest_end_of_a_bound() -> None:
rows = [
run_result_block_row("loop_body", "completed", "https://fixture.test/item/1"),
run_result_block_row("open_search", "completed", "https://fixture.test/"),
run_result_block_row("loop_body", "failed", "https://fixture.test/item/9"),
]
end_urls = run_execution_module._block_end_urls_by_label(rows)
assert list(end_urls) == ["open_search", "loop_body"]
assert end_urls["loop_body"] == "https://fixture.test/item/9"
notices: list[str] = []
kept = _compacted_newest_labelled(list(end_urls.items()), 1, field_name="observed_block_end_urls", notices=notices)
assert kept == [("loop_body", "https://fixture.test/item/9")]
assert any("dropped the oldest block(s) open_search" in notice for notice in notices)
def test_a_label_a_loop_reused_keeps_its_newest_action_observations_at_the_newest_end() -> None:
results = [
{"label": "loop_body", "action_trace": [{"action": "click", "status": "completed", "code_line": 1}]},
{"label": "open_search", "action_trace": [{"action": "click", "status": "completed", "code_line": 2}]},
{"label": "loop_body", "action_trace": [{"action": "wait", "status": "failed", "code_line": 3}]},
]
by_label = run_execution_module._retained_action_observations_by_label(results)
assert list(by_label) == ["open_search", "loop_body"]
assert by_label["loop_body"] == ["click completed code_line=1", "wait failed code_line=3"]
def test_a_repeated_block_label_reports_the_status_of_its_newest_recorded_row() -> None:
data = {
"blocks": [
{"label": "loop_body", "status": "completed"},
{"label": "loop_body", "status": "failed"},
],
"registered_output_parameter_values": [
{
"workflow_run_id": "wr-1",
"block_label": "loop_body",
"output_parameter_key": "picked",
"value": {"name": "row-9"},
}
],
}
outputs = run_execution_module._packet_registered_outputs(make_copilot_ctx(), data, "wr-1", [])
assert [(output.label, output.status) for output in outputs] == [("loop_body", "failed")]
def test_every_packet_rendering_keeps_the_newest_per_block_facts_and_says_what_it_dropped() -> None:
unbounded = build_test_evidence_packet(make_copilot_ctx(), _two_block_search_failure_result()).model_copy(
update={
"observed_block_end_urls": {f"block_{index}": f"https://fixture.test/{index}" for index in range(30)},
"per_block_action_observations": {
f"block_{index}": [f"click completed code_line={line}" for line in range(20)] for index in range(30)
},
}
)
projected = project_build_test_packet_for_llm(unbounded)
assert len(projected.observed_block_end_urls) == 12
assert projected.observed_block_end_urls["block_29"] == "https://fixture.test/29"
assert "block_17" not in projected.observed_block_end_urls
assert len(projected.per_block_action_observations) == 12
assert projected.per_block_action_observations["block_29"] == [
f"click completed code_line={line}" for line in range(14, 20)
]
assert any("18 oldest block(s) omitted" in notice for notice in projected.omission_notices)
assert any("oldest observation(s) omitted" in notice for notice in projected.omission_notices)
compact_notices: list[str] = []
compacted = _compact_packet_for_aggregate_limit(unbounded, compact_notices)
assert list(compacted.observed_block_end_urls) == ["block_28", "block_29"]
assert compacted.per_block_action_observations["block_29"] == ["click completed code_line=19"]
assert any("observed_block_end_urls shortened at the aggregate packet limit" in n for n in compact_notices)
def test_an_over_long_page_state_url_is_refused_rather_than_cut() -> None:
packet = build_test_evidence_packet(make_copilot_ctx(), _two_block_search_failure_result()).model_copy(
update={
"page_state": BuildTestPacketPageState(
current_url="https://fixture.test/results?q=" + "u" * 3000,
observed_after_workflow_run=True,
)
}
)
projected = project_build_test_packet_for_llm(packet)
assert projected.page_state is not None
assert projected.page_state.current_url is None
assert any("exceeded 2000 characters" in notice for notice in projected.omission_notices)
def test_an_over_long_end_url_is_refused_rather_than_cut_into_a_page_the_run_never_reached() -> None:
notices: list[str] = []
coerced = coerce_block_end_urls(
{
"run_search": "https://fixture.test/results?q=" + "u" * 3000,
"select_first_result": "https://fixture.test/results/widget",
},
notices,
)
assert coerced == {"select_first_result": "https://fixture.test/results/widget"}
assert any("exceeded 2000 characters" in notice for notice in notices)
def test_direct_test_handoff_keeps_same_origin_pages_apart_but_still_drops_a_query() -> None:
result = _two_block_search_failure_result()
data = result["data"]
assert isinstance(data, dict)
data["observed_block_end_urls"] = {
"run_search": "https://fixture.test/results/widget",
"select_first_result": "https://fixture.test/records/1842",
}
projected = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), result))
assert projected.observed_block_end_urls == {
"run_search": "https://fixture.test/results/widget",
"select_first_result": "https://fixture.test/records/1842",
}
query_bearing = _two_block_search_failure_result()
query_bearing_data = query_bearing["data"]
assert isinstance(query_bearing_data, dict)
query_bearing_data["observed_block_end_urls"] = {"run_search": "https://fixture.test/results?session_id=abc123"}
reduced = project_direct_test_handoff_packet_for_llm(build_test_evidence_packet(make_copilot_ctx(), query_bearing))
assert reduced.observed_block_end_urls == {"run_search": "https://fixture.test/results"}
assert "abc123" not in json.dumps(reduced.model_dump(mode="json"))
def test_typed_runtime_failure_under_a_challenge_shaped_label_still_fails_with_its_true_reason() -> None:
failure_reason = "CodeBlock failed with NameError at line 1: name 'token' is not defined."
result = _generated_code_exception_result("wr_captcha_label", failure_reason)
result["data"]["blocks"][0]["label"] = "solve_and_submit_recaptcha_demo"
result["data"]["observed_block_end_urls"] = {"solve_and_submit_recaptcha_demo": "https://demo.example.com/form"}
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code == "runtime_block_failure"
assert failure_reason in outcome.observed_evidence_summary
def test_uncleared_challenge_reports_its_real_reason_not_a_nearby_url() -> None:
end_url = "https://demo.example.com/verification/complete"
real_reason = "The page presented a human verification challenge that was never cleared."
result = {
"ok": True,
"data": {
"workflow_run_id": "wr_uncleared",
"overall_status": "completed",
"current_url": end_url,
"failure_reason": real_reason,
"failure_categories": [
{"category": "ANTI_BOT_DETECTION", "evidence_source": "challenge_state", "confidence_float": 1.0}
],
"observed_block_end_urls": {"solve_and_submit_recaptcha_demo": end_url},
"blocks": [
{
"label": "solve_and_submit_recaptcha_demo",
"block_type": "CODE",
"status": "completed",
"extracted_data": {"submitted": False},
}
],
},
}
ctx = _locator_packet_ctx()
ctx.last_test_ok = True
assert run_execution_module.settle_terminal_challenge_after_enrichment(ctx, result) is True
assert ctx.last_run_outcome is not None
assert ctx.last_run_outcome.verdict == "not_demonstrated"
assert real_reason in str(ctx.last_test_failure_reason)
assert end_url not in str(ctx.last_test_failure_reason)
assert result["data"]["observed_block_end_urls"] == {"solve_and_submit_recaptcha_demo": end_url}
_SIGN_IN_URL = "https://analytics.fixture.test/login?next=/project/1/web"
_EXTRACTION_TIMEOUT = (
"CodeBlock failed because a browser operation failed at line 2: Locator.wait_for: "
'Timeout 30000ms exceeded.Call log: - waiting for locator("main#main-content") to be visible.'
)
def _login_then_extract_run_result() -> dict[str, object]:
"""The retained two-row shape from SKY-15656, with a synthetic host and project id."""
return {
"ok": False,
"data": {
"workflow_run_id": "wr_login_then_extract",
"blocks": [
{
"workflow_run_block_id": "wrb_login",
"label": "log_in_to_web_analytics",
"block_type": "CODE",
"status": "completed",
"output": {"current_url": _SIGN_IN_URL, "page_evidence": ""},
},
{
"workflow_run_block_id": "wrb_extract",
"label": "extract_web_analytics_visitors",
"block_type": "CODE",
"status": "failed",
"failure_reason": _EXTRACTION_TIMEOUT,
"output": {"status": "failed", "failure_reason": _EXTRACTION_TIMEOUT},
},
],
},
}
def _recorded_block_outcome_rows(result: dict[str, object]) -> list[str]:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.latest_recorded_build_test_outcome = outcome
return [line for line in _recorded_build_test_outcome_prompt(ctx).splitlines() if line.startswith("- label=")]
def test_completed_row_recorded_facts_reach_repair_without_becoming_a_success_claim() -> None:
assert _recorded_block_outcome_rows(_login_then_extract_run_result()) == [
(
f"- label=log_in_to_web_analytics; status=completed; output.current_url={_SIGN_IN_URL}; "
"output.page_evidence=(empty)"
),
(
"- label=extract_web_analytics_visitors; status=failed; output.status=failed; "
f"output.failure_reason={_EXTRACTION_TIMEOUT}"
),
]
def test_a_row_that_recorded_no_output_is_not_rendered_as_an_empty_observation() -> None:
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = None
assert _recorded_block_outcome_rows(result)[0] == (
"- label=log_in_to_web_analytics; status=completed; recorded_output=(none recorded)"
)
def test_both_ends_of_a_long_run_survive_the_recorded_outcome_limit() -> None:
"""The step that was to establish the session runs first and the failing step runs last."""
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
filler = [
{
"workflow_run_block_id": f"wrb_step_{index}",
"label": f"step_{index}",
"block_type": "CODE",
"status": "skipped" if index % 5 == 0 else "completed",
"output": {"step": str(index)},
}
for index in range(20)
]
blocks = data["blocks"]
assert isinstance(blocks, list)
# The ordering a real workflow has: log in, do the work, fail at the end.
data["blocks"] = [blocks[0]] + filler + [blocks[1]]
rows = _recorded_block_outcome_rows(result)
assert len(rows) == 12
assert "label=log_in_to_web_analytics" in rows[0]
assert "output.current_url=https://analytics.fixture.test/login" in rows[0]
assert "label=extract_web_analytics_visitors" in rows[-1]
assert all("status=skipped" not in row for row in rows)
def test_a_long_run_says_how_many_rows_sit_between_the_ends_it_kept() -> None:
"""A gap the model cannot see reads as rows that never ran unless the projection says otherwise."""
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
filler = [
{
"workflow_run_block_id": f"wrb_step_{index}",
"label": f"step_{index}",
"block_type": "CODE",
"status": "completed",
"output": {"step": str(index)},
}
for index in range(18)
]
data["blocks"] = [blocks[0]] + filler + [blocks[1]]
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.latest_recorded_build_test_outcome = outcome
rendered = _recorded_build_test_outcome_prompt(ctx)
assert outcome.recorded_block_outcome_rows_omitted == 8
assert "8 row(s) between the first and last shown are not listed." in rendered
def test_a_row_says_how_many_recorded_fields_it_did_not_show() -> None:
"""A bounded projection reports what it left out, so an absent field is not read as unrecorded."""
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {f"field_{index}": f"value_{index}" for index in range(11)}
row = _recorded_block_outcome_rows(result)[0]
assert "output.field_7=value_7" in row
assert "output.field_8" not in row
assert "3 more field(s) not shown" in row
def test_a_registered_value_used_as_a_recorded_key_does_not_reach_the_prompt() -> None:
"""A key is as free-form as a value, and a bound cut it before anything looked for a secret in it."""
secret = "fake-api-key-" + "k" * 100
ctx = _locator_packet_ctx()
ctx.browser_session_id = "pbs_secret_key"
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
register_secret_scrub_value(ctx, secret)
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {secret: "granted"}
try:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx.latest_recorded_build_test_outcome = outcome
rendered = _recorded_build_test_outcome_prompt(ctx)
finally:
clear_session_scrub_values("pbs_secret_key")
assert secret[:32] not in rendered
assert "output.[REDACTED_SECRET]=granted" in rendered
def test_a_long_secret_shaped_key_is_redacted_before_it_is_bounded() -> None:
"""A generic secret shape cut at the key bound is no longer a shape, so it is redacted before the cut."""
jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "e" * 90 + ".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
assert len(jwt) > 96
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {jwt: "granted"}
row = _recorded_block_outcome_rows(result)[0]
assert "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" not in row
assert "output.[REDACTED_SECRET]=granted" in row
def test_two_keys_that_scrub_alike_keep_both_rows() -> None:
"""Scrubbing can make two keys identical, and a row that vanishes is one repair cannot read."""
first, second = "fake-key-one-" + "a" * 40, "fake-key-two-" + "b" * 40
ctx = _locator_packet_ctx()
ctx.browser_session_id = "pbs_key_collision"
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
register_secret_scrub_value(ctx, first)
register_secret_scrub_value(ctx, second)
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {first: "alpha", second: "beta", "x" * 120: "long-one", "x" * 120 + "tail": "long-two"}
try:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx.latest_recorded_build_test_outcome = outcome
rendered = _recorded_build_test_outcome_prompt(ctx)
finally:
clear_session_scrub_values("pbs_key_collision")
for value in ("alpha", "beta", "long-one", "long-two"):
assert f"={value}" in rendered, value
assert "output.[REDACTED_SECRET]=alpha" in rendered
assert "output.[REDACTED_SECRET] (2)=beta" in rendered
def test_a_secret_longer_than_the_input_bound_is_not_cut_into_the_prompt() -> None:
"""A registered value is matched whole, so the bound must not leave a prefix of one behind."""
secret = "z9" * 900
ctx = _locator_packet_ctx()
ctx.browser_session_id = "pbs_long_secret"
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
register_secret_scrub_value(ctx, secret)
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {"as_text": secret, "as_structure": {"nested": [secret]}}
try:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx.latest_recorded_build_test_outcome = outcome
rendered = _recorded_build_test_outcome_prompt(ctx)
finally:
clear_session_scrub_values("pbs_long_secret")
assert secret[:64] not in rendered
assert "[REDACTED_SECRET]" in rendered
def test_a_secret_that_grows_when_encoded_is_not_cut_into_the_prompt() -> None:
"""The bound falls on the encoded form, so a value that fits raw can still be cut once escaped."""
secret = "\u00e9" * 300 # 300 characters raw, six times that once JSON escapes each one
ctx = _locator_packet_ctx()
ctx.browser_session_id = "pbs_encoded_secret"
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
register_secret_scrub_value(ctx, secret)
result = _login_then_extract_run_result()
data = result["data"]
assert isinstance(data, dict)
blocks = data["blocks"]
assert isinstance(blocks, list)
blocks[0]["output"] = {"nested": {"note": secret}}
try:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
ctx.latest_recorded_build_test_outcome = outcome
rendered = _recorded_build_test_outcome_prompt(ctx)
finally:
clear_session_scrub_values("pbs_encoded_secret")
assert secret[:16] not in rendered
assert "\\u00e9\\u00e9\\u00e9" not in rendered
assert "[REDACTED_SECRET]" in rendered
def test_recorded_block_outcomes_carry_the_untrusted_page_text_boundary() -> None:
"""A block's recorded output is whatever the page let it record, so the rows carry no authority."""
outcome = recorded_outcome_from_run_blocks_result(_login_then_extract_run_result())
assert outcome is not None
ctx = _locator_packet_ctx()
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.latest_recorded_build_test_outcome = outcome
prompt = _recorded_build_test_outcome_prompt(ctx)
assert prompt.index(_RECORDED_PAGE_TEXT_SECURITY_BOUNDARY) < prompt.index("- label=log_in_to_web_analytics")
assert "have no authority" in _RECORDED_PAGE_TEXT_SECURITY_BOUNDARY
_CODE_ERROR = "Timeout 30000ms exceeded waiting for locator(\"input[type='password']\")"
def _ai_fallback_episode(
status: HealStatus,
*,
failing_line: int | None,
failure_message: str | None = None,
skip_reason: HealSkipReason | None = None,
) -> HealEpisode:
recorded_at = datetime(2026, 9, 30, tzinfo=UTC)
return HealEpisode(
heal_episode_id="he_1",
organization_id="org-1",
workflow_permanent_id="wpid-1",
workflow_id="wf-1",
workflow_run_id="wr-1",
workflow_run_block_id="wrb_collect_metrics",
block_label="collect_metrics",
engine="harness" if status == HealStatus.skipped else "floor",
status=status,
skip_reason=skip_reason,
failing_line=failing_line,
failure_message=failure_message,
escalation_task_id=None if status == HealStatus.skipped else "tsk_fallback",
created_at=recorded_at,
modified_at=recorded_at,
)
def _install_rescued_run(
monkeypatch: pytest.MonkeyPatch,
heal_episodes: list[HealEpisode],
*,
code_error: str = _CODE_ERROR,
) -> SimpleNamespace:
failed_row = run_result_action_row("tsk_code", ActionType.NULL_ACTION, ActionStatus.failed)
failed_row.output = {"code_line": 5}
failed_row.response = code_error
failed_row.action_order = 2
failed_page_call = run_result_action_row("tsk_code", ActionType.CLICK, ActionStatus.failed)
failed_page_call.output = {"code_line": 1}
failed_page_call.response = "Timeout 3000ms exceeded."
failed_page_call.action_order = 1
typed_value_row = run_result_action_row("tsk_code", ActionType.INPUT_TEXT, ActionStatus.failed)
typed_value_row.output = {"code_line": None}
typed_value_row.response = "typed-in field value"
typed_value_row.action_order = 3
async def register_rescued_output(*, data: dict[str, Any], **_: object) -> dict[str, Any]:
for block in data["blocks"]:
if block["label"] == "collect_metrics":
block["extracted_data"] = {"collect_metrics_output": {"impressions": 1180}}
return {}
code_row = run_result_block_row("collect_metrics", "completed", task_id="tsk_code")
code_row.block_type = BlockType.CODE
recovery_row = run_result_block_row("Self-heal recovery", "completed", task_id="tsk_fallback")
recovery_row.parent_workflow_run_block_id = code_row.workflow_run_block_id
fired = any(episode.status != HealStatus.skipped for episode in heal_episodes)
ctx = install_get_run_results_harness(
monkeypatch,
run_status="completed",
blocks=[
code_row,
*([recovery_row] if fired else []),
run_result_block_row("write_sheet", "completed", task_id="tsk_write"),
],
recent_actions=[
typed_value_row,
failed_row,
failed_page_call,
run_result_action_row("tsk_code", ActionType.CLICK, ActionStatus.completed),
run_result_action_row("tsk_write", ActionType.CLICK, ActionStatus.completed),
],
heal_episodes=heal_episodes,
)
monkeypatch.setattr(run_execution_module, "_attach_registered_output_parameter_values", register_rescued_output)
return ctx
async def _run_results_page(ctx: SimpleNamespace | CopilotContext) -> dict[str, Any]:
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
return sanitize_tool_result_for_llm("get_run_results", run_execution_module.project_run_results_page(result, {}))
@pytest.mark.asyncio
async def test_a_rescued_block_reads_as_an_ai_fallback_rescue_in_run_results_and_compaction(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
page = await _run_results_page(ctx)
rows = {row["label"]: row for row in page["data"]["blocks"]}
assert rows["collect_metrics"]["status"] == "completed"
assert rows["collect_metrics"]["ai_fallback"] == {
"status": "fired_completed",
"task_id": "tsk_fallback",
"failing_line": 5,
"failure_text": _CODE_ERROR,
}
assert rows["Self-heal recovery"]["parent_block_label"] == "collect_metrics"
assert "ai_fallback" not in rows["write_sheet"]
compacted = {row["label"]: row for row in json.loads(_summarize_tool_output(json.dumps(page)))["blocks"]}
assert compacted["collect_metrics"]["ai_fallback"] == {"status": "fired_completed", "failure_text": _CODE_ERROR}
assert compacted["Self-heal recovery"]["parent_block_label"] == "collect_metrics"
@pytest.mark.asyncio
async def test_a_failed_ai_fallback_reports_the_code_error_beside_the_recovery_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _install_rescued_run(
monkeypatch,
[
_ai_fallback_episode(
HealStatus.fired_failed, failing_line=5, failure_message="Password field never appeared"
)
],
)
page = await _run_results_page(ctx)
ai_fallback = next(row for row in page["data"]["blocks"] if row["label"] == "collect_metrics")["ai_fallback"]
assert ai_fallback["failure_text"] == _CODE_ERROR
assert ai_fallback["recovery_failure_text"] == "Password field never appeared"
@pytest.mark.asyncio
async def test_a_secret_shaped_token_in_the_code_error_never_reaches_the_run_results(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Joined at runtime so no source line holds a whole JWT for secret scanners to flag.
token = ".".join(("eyJhbGciOiJIUzI1NiJ9", "eyJzdWIiOiJ0ZXN0LXVzZXIifQ", "c2lnbmF0dXJlLXZhbHVlLXg"))
ctx = _install_rescued_run(
monkeypatch,
[_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)],
code_error=f"Timeout 30000ms exceeded waiting for locator(\"#session[value='{token}']\")",
)
page = await _run_results_page(ctx)
assert token not in json.dumps(page)
ai_fallback = next(row for row in page["data"]["blocks"] if row["label"] == "collect_metrics")["ai_fallback"]
assert "[REDACTED_SECRET]" in ai_fallback["failure_text"]
_CAUGHT_CLICK_INTERCEPT = '<span class="cc-toggle-heading">…</span> intercepts pointer events'
_CAPTURED_CAUGHT_CLICK_ERROR = json.loads(
(Path(__file__).parent / "fixtures/copilot/caught_failure_repair/consent_toggle_caught_click.json").read_text()
)["turns"][0]["inputs"]["result"]["data"]["blocks"][0]["action_trace"][0]["response"]
_CAUGHT_CLICK_ERROR = "\n".join(
[
"Locator.click: Timeout 4000ms exceeded.",
"Call log:",
' - waiting for get_by_text("Reject All").first',
' - locator resolved to <span class="cc-toggle-label">Reject All</span>',
" - attempting click action",
" 2 × waiting for element to be visible, enabled and stable",
" - element is visible, enabled and stable",
" - scrolling into view if needed",
" - done scrolling",
f" - {_CAUGHT_CLICK_INTERCEPT}",
" - retrying click action",
" - waiting 20ms",
" 2 × waiting for element to be visible, enabled and stable",
" - element is visible, enabled and stable",
" - scrolling into view if needed",
" - done scrolling",
f" - {_CAUGHT_CLICK_INTERCEPT}",
" - retrying click action",
" - waiting 100ms",
]
)
_DISABLED_FIELD_VALUE = "harbor lights 2291"
_DISABLED_FIELD_FILL_ERROR = "\n - ".join(
[
'Locator.fill: Timeout 3000ms exceeded.\nCall log:\nwaiting for locator("#account-note")',
' locator resolved to <input disabled id="account-note"/>',
f' fill("{_DISABLED_FIELD_VALUE}")',
"attempting fill action",
" waiting for element to be visible, enabled and editable",
" element is not enabled",
"retrying fill action, attempt #1",
]
)
# Playwright 1.58 indents call-log lines and prefixes repeats with a count. Its element preview carries the input's
# value and cuts a long attribute with "…" and no closing quote.
_DISABLED_FIELD_FILL_ERROR_1_58 = "\n".join(
[
"Locator.fill: Timeout 3000ms exceeded.",
"Call log:",
' - waiting for locator("#account-note")',
f' - locator resolved to <input disabled class="{" ".join(["form-field"] * 30)}" '
f'value="{"x" * 470}{_DISABLED_FIELD_VALUE}…/>',
f' - fill("{_DISABLED_FIELD_VALUE}")',
" - attempting fill action",
" 2 × waiting for element to be visible, enabled and editable",
" - element is not enabled",
" - retrying fill action",
]
)
# A multi-line value is echoed raw, so its later lines carry no call-log prefix.
_DISABLED_FIELD_FILL_ERROR_MULTILINE = "\n".join(
[
"Locator.fill: Timeout 3000ms exceeded.",
"Call log:",
'waiting for locator("#account-note")',
' - locator resolved to <textarea disabled id="account-note"></textarea>',
f' - fill("Account note\n{_DISABLED_FIELD_VALUE}")',
" - attempting fill action",
" - waiting for element to be visible, enabled and editable",
" - element is not enabled",
]
)
async def _caught_click_run(
monkeypatch: pytest.MonkeyPatch,
response: str,
action_type: ActionType = ActionType.CLICK,
heal_episodes: list[HealEpisode] | None = None,
) -> tuple[dict[str, Any], CopilotContext]:
"""A completed block whose program caught one failed page call and then made seven more calls, newest first as
the repository returns them."""
caught = run_result_action_row("tsk_code", action_type, ActionStatus.failed, code_line=7)
caught.response = response
later = [
run_result_action_row("tsk_code", ActionType.CLICK, ActionStatus.completed, code_line=line)
for line in range(14, 7, -1)
]
await install_run_blocks_harness(
monkeypatch,
workflow_yaml=HANDBACK_WORKFLOW_YAML,
polled_status="completed",
terminal_blocks=[terminal_extraction_block("completed", task_id="tsk_code")],
recent_actions=[*later, caught],
)
if heal_episodes:
monkeypatch.setattr(
run_execution_module.app.DATABASE.self_heal,
"get_heal_episodes_for_run",
AsyncMock(return_value=heal_episodes),
)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
return result, ctx
@pytest.mark.asyncio
async def test_a_value_a_failed_fill_typed_stays_out_of_a_test_runs_extracted_data(
monkeypatch: pytest.MonkeyPatch,
) -> None:
block = terminal_extraction_block("failed", failure_reason=_DISABLED_FIELD_FILL_ERROR_1_58, task_id="tsk_code")
block.output = {
"status": "failed",
"failure_reason": _DISABLED_FIELD_FILL_ERROR_1_58,
"errors": [{"error_code": "page_operation_failed", "reasoning": _DISABLED_FIELD_FILL_ERROR_1_58}],
}
await install_run_blocks_harness(
monkeypatch, workflow_yaml=HANDBACK_WORKFLOW_YAML, polled_status="failed", terminal_blocks=[block]
)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
assert _DISABLED_FIELD_VALUE not in json.dumps(result, default=str)
assert "element is not enabled" in json.dumps(result["data"]["blocks"][0]["extracted_data"])
@pytest.mark.asyncio
async def test_a_caught_page_failure_in_a_block_the_ai_fallback_rescued_during_a_test_run_is_not_reported_as_continued(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rescued = _ai_fallback_episode(HealStatus.fired_completed, failing_line=7).model_copy(
update={"workflow_run_block_id": "wrb_extract_heading"}
)
result, ctx = await _caught_click_run(monkeypatch, _CAUGHT_CLICK_ERROR, heal_episodes=[rescued])
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
failed = [observation for observation in projected.action_observations if observation.startswith("click failed")]
assert len(failed) == 1 and _CAUGHT_CLICK_INTERCEPT in failed[0], projected.action_observations
assert "continued" not in failed[0]
@pytest.mark.asyncio
async def test_a_page_failure_the_program_caught_reaches_the_packet_with_its_error_and_that_the_block_went_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
result, ctx = await _caught_click_run(monkeypatch, _CAUGHT_CLICK_ERROR)
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
failed = [observation for observation in projected.action_observations if observation.startswith("click failed")]
assert len(failed) == 1, projected.action_observations
for fact in ("code_line=7", _CAUGHT_CLICK_INTERCEPT, "Timeout 4000ms exceeded", "continued"):
assert fact in failed[0]
assert len(failed[0]) <= 300
assert failed[0] in projected.per_block_action_observations["extract_heading"]
assert projected.action_observations.count("click completed") == 5
@pytest.mark.asyncio
async def test_a_secret_shaped_token_in_a_caught_page_failure_stays_out_of_the_packet(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Joined at runtime so no source line holds a whole JWT for secret scanners to flag.
token = ".".join(("eyJhbGciOiJIUzI1NiJ9", "eyJzdWIiOiJ0ZXN0LXVzZXIifQ", "c2lnbmF0dXJlLXZhbHVlLXg"))
result, ctx = await _caught_click_run(
monkeypatch,
f"Locator.click: Timeout 4000ms exceeded.\nCall log:\n - waiting for locator(\"#s[value='{token}']\")",
)
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
assert token not in projected.model_dump_json()
assert token not in json.dumps(sanitized)
assert any("[REDACTED_SECRET]" in observation for observation in projected.action_observations)
@pytest.mark.parametrize(
"fill_error",
[_DISABLED_FIELD_FILL_ERROR, _DISABLED_FIELD_FILL_ERROR_1_58, _DISABLED_FIELD_FILL_ERROR_MULTILINE],
ids=["playwright-1.46", "playwright-1.58", "multi-line-value"],
)
@pytest.mark.asyncio
async def test_a_value_a_caught_fill_typed_stays_out_of_the_packet_while_its_cause_reaches_it(
monkeypatch: pytest.MonkeyPatch, fill_error: str
) -> None:
result, ctx = await _caught_click_run(monkeypatch, fill_error, ActionType.INPUT_TEXT)
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
assert _DISABLED_FIELD_VALUE not in projected.model_dump_json()
assert _DISABLED_FIELD_VALUE not in json.dumps(sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result))
failed = [observation for observation in projected.action_observations if observation.startswith("input_text")]
assert len(failed) == 1 and "element is not enabled" in failed[0], projected.action_observations
@pytest.mark.asyncio
@pytest.mark.parametrize(
"password", ["hunter2pass", "hun'ter2pass"], ids=["plain-password", "password-with-apostrophe"]
)
async def test_credentials_and_query_in_a_caught_navigation_failure_stay_out_of_the_packet(
monkeypatch: pytest.MonkeyPatch, password: str
) -> None:
url = f"https://alice:{password}@nohost.invalid/path?token=abc123#frag"
navigation_error = (
f'Page.goto: net::ERR_NAME_NOT_RESOLVED at {url}\nCall log:\nnavigating to "{url}", waiting until "load"\n'
)
result, ctx = await _caught_click_run(monkeypatch, navigation_error, ActionType.GOTO_URL)
projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
for secret in ("ter2pass", "token=abc123"):
assert secret not in projected.model_dump_json()
failed = [observation for observation in projected.action_observations if observation.startswith("goto_url")]
assert len(failed) == 1 and "ERR_NAME_NOT_RESOLVED" in failed[0], projected.action_observations
@pytest.mark.asyncio
async def test_a_value_a_failed_fill_typed_stays_out_of_the_ai_fallback_code_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _install_rescued_run(
monkeypatch,
[_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)],
code_error=_DISABLED_FIELD_FILL_ERROR,
)
page = await _run_results_page(ctx)
assert _DISABLED_FIELD_VALUE not in json.dumps(page)
ai_fallback = next(row for row in page["data"]["blocks"] if row["label"] == "collect_metrics")["ai_fallback"]
assert "element is not enabled" in ai_fallback["failure_text"]
# Typed lines that look like call-log lines: a bulleted note (1.46 format) and a count-prefixed one (1.58 format).
_BULLETED_NOTE_FILL_ERROR = "\n".join(
[
"Locator.fill: Timeout 3000ms exceeded.",
"Call log:",
'waiting for locator("#account-note")',
' - locator resolved to <textarea disabled id="account-note"></textarea>',
' - fill("Call back Monday\n- SYNTHETIC two\n- SYNTHETIC three")',
" - attempting fill action",
" - waiting for element to be visible, enabled and editable",
" - element is not enabled",
]
)
_COUNTED_NOTE_FILL_ERROR = "\n".join(
[
"Locator.fill: Timeout 3000ms exceeded.",
"Call log:",
' - waiting for locator("#account-note")',
' - locator resolved to <textarea disabled id="account-note"></textarea>',
' - fill("first\n3 × SYNTHETIC\nlast")',
" - attempting fill action",
" 2 × waiting for element to be visible, enabled and editable",
" - element is not enabled",
]
)
_STEP_PHRASE_NOTE_FILL_ERROR = "\n".join(
[
"Locator.fill: Timeout 3000ms exceeded.",
"Call log:",
'waiting for locator("#account-note")',
' - locator resolved to <textarea disabled id="account-note"></textarea>',
' - fill("first\nattempting click action\nprivate customer text")',
" - attempting fill action",
" - waiting for element to be visible, enabled and editable",
" - element is not enabled",
]
)
_EMBEDDED_CLOSER_NOTE_FILL_ERROR = _STEP_PHRASE_NOTE_FILL_ERROR.replace('fill("first\n', 'fill("first")\n')
_PARAMETER_NOTE_FILL_ERROR = _STEP_PHRASE_NOTE_FILL_ERROR.replace(
"attempting click action\nprivate customer text",
"retrying secretword action, attempt #987654321\n999 × waiting 123456ms\nlast",
)
@pytest.mark.parametrize(
("fill_error", "typed_text"),
[
(_DISABLED_FIELD_FILL_ERROR_1_58, [_DISABLED_FIELD_VALUE]),
(_BULLETED_NOTE_FILL_ERROR, ["Call back Monday", "SYNTHETIC"]),
(_COUNTED_NOTE_FILL_ERROR, ["SYNTHETIC", 'last")']),
(_STEP_PHRASE_NOTE_FILL_ERROR, ["private customer text"]),
(_EMBEDDED_CLOSER_NOTE_FILL_ERROR, ["private customer text"]),
(_PARAMETER_NOTE_FILL_ERROR, ["secretword", "987654321", "123456"]),
],
ids=[
"playwright-1.58",
"bulleted-note",
"count-prefixed-note",
"step-phrase-note",
"embedded-closer-note",
"step-parameters-note",
],
)
def test_a_value_a_failed_fill_typed_stays_out_of_the_block_failure_reason_and_output(
fill_error: str, typed_text: list[str]
) -> None:
block = terminal_extraction_block("failed", failure_reason=fill_error)
block.output = {
"status": "failed",
"failure_reason": fill_error,
"errors": [{"error_code": "page_operation_failed", "reasoning": fill_error}],
}
result = _recorded_run_block_result(block)
for text in typed_text:
assert text not in json.dumps(result)
assert "element is not enabled" in result["failure_reason"]
assert "element is not enabled" in result["output"]["errors"][0]["reasoning"]
@pytest.mark.asyncio
async def test_a_caught_page_failure_is_not_reported_as_continued_when_the_ai_fallback_record_is_unreadable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def caught_click_observations() -> list[str]:
return result["data"]["per_block_action_observations"]["collect_metrics"]
ctx = _install_rescued_run(monkeypatch, [])
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert any("(block continued)" in observation for observation in caught_click_observations())
ctx = _install_rescued_run(monkeypatch, [])
monkeypatch.setattr(
run_execution_module.app.DATABASE.self_heal,
"get_heal_episodes_for_run",
AsyncMock(side_effect=RuntimeError("self-heal store unavailable")),
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
observations = caught_click_observations()
assert any("Timeout 3000ms exceeded." in observation for observation in observations), observations
assert not any("continued" in observation for observation in observations)
ctx = _install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
monkeypatch.setattr(
run_execution_module, "build_heal_episode_detail", MagicMock(side_effect=ValueError("unreadable episode"))
)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert not any("continued" in observation for observation in caught_click_observations())
def test_an_older_blocks_caught_failures_leave_the_newest_block_a_slot() -> None:
caught = {"action": "click", "status": "failed", "code_line": 4, "response": "Timeout 3000ms exceeded."}
results = [
{
"label": "dismiss_cover",
"status": "completed",
"action_trace": [{**caught, "code_line": line} for line in range(4, 11)],
},
{
"label": "read_orders",
"status": "completed",
"action_trace": [{"action": "click", "status": "completed"}, {"action": "goto_url", "status": "completed"}],
},
]
by_label = run_execution_module._retained_action_observations_by_label(results)
assert by_label["read_orders"], by_label
assert len(by_label["dismiss_cover"]) == 5
def test_a_failure_a_program_retried_takes_one_slot_and_leaves_the_predecessors_terminal_facts() -> None:
caught = {"action": "click", "status": "failed", "code_line": 4, "response": "Timeout 3000ms exceeded."}
predecessors = [
{"label": f"step_{index}", "status": "completed", "action_trace": [{"action": "click", "status": "completed"}]}
for index in range(5)
]
results = [*predecessors, {"label": "dismiss_cover", "status": "completed", "action_trace": [caught] * 6}]
selection = run_execution_module._selected_action_observations(results)
by_label = run_execution_module._retained_action_observations_by_label(results)
assert all(by_label[f"step_{index}"] == ["click completed"] for index in range(5)), by_label
assert len(by_label["dismiss_cover"]) == 1
assert selection.unreported_predecessor_indexes == []
@pytest.mark.asyncio
async def test_a_page_failure_the_ai_fallback_recovered_is_reported_without_the_block_going_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
observations = result["data"]["per_block_action_observations"]["collect_metrics"]
assert any("Timeout 3000ms exceeded." in observation for observation in observations), observations
assert not any("continued" in observation for observation in observations)
def test_a_caught_page_failure_in_the_oldest_block_survives_the_aggregate_limit_with_its_intercepting_element() -> None:
results = [
{
"label": "dismiss_cover",
"status": "completed",
"action_trace": [
{"action": "click", "status": "completed", "code_line": 9},
{"action": "click", "status": "failed", "code_line": 3, "response": _CAPTURED_CAUGHT_CLICK_ERROR},
],
},
{"label": "run_search", "status": "completed", "action_trace": [{"action": "click", "status": "completed"}]},
{
"label": "read_results",
"status": "completed",
"action_trace": [{"action": "scroll", "status": "completed"}, {"action": "click", "status": "completed"}],
},
]
packet = build_test_evidence_packet(
make_copilot_ctx(),
{
"ok": True,
"data": {
"workflow_run_id": "wr_1",
"overall_status": "completed",
"blocks": [{"label": result["label"], "status": "completed"} for result in results],
"action_observations": run_execution_module._retained_action_observations(results),
"per_block_action_observations": run_execution_module._retained_action_observations_by_label(results),
},
},
)
compacted = _compact_packet_for_aggregate_limit(packet, [])
by_label = _compact_block_fact_maps(packet, []).per_block_action_observations
failed = [observation for observation in compacted.action_observations if observation.startswith("click failed")]
assert len(failed) == 1, compacted.action_observations
assert '<span class="pref-toggle-heading">Optional cookies</span>' in failed[0]
assert failed[0].endswith("...") and len(failed[0]) == 120
assert next(iter(compacted.per_block_action_observations)) == "dismiss_cover"
assert compacted.per_block_action_observations["dismiss_cover"][0].startswith("click failed")
assert "dismiss_cover" in by_label
def test_a_native_failed_action_takes_no_priority_at_the_aggregate_limit() -> None:
packet = build_test_evidence_packet(
make_copilot_ctx(),
{
"ok": True,
"data": {
"workflow_run_id": "wr_1",
"overall_status": "completed",
"blocks": [{"label": label, "status": "completed"} for label in ("login", "open_orders", "search")],
"action_observations": ["click failed", "goto_url completed", "scroll completed"],
"per_block_action_observations": {
"login": ["click failed"],
"open_orders": ["goto_url completed"],
"search": ["scroll completed"],
},
},
},
)
compacted = _compact_packet_for_aggregate_limit(packet, [])
by_label = _compact_block_fact_maps(packet, []).per_block_action_observations
assert compacted.action_observations == ["goto_url completed", "scroll completed"]
assert list(compacted.per_block_action_observations) == ["open_orders", "search"]
assert list(by_label) == ["open_orders", "search"]
def test_the_packet_keeps_the_newest_ai_fallback_records_and_says_how_many_it_left_out() -> None:
ctx = make_copilot_ctx()
ctx.registered_artifact_evidence = None
blocks = [
{
"label": f"loop_step_{index}",
"status": "completed",
"ai_fallback": {"status": "fired_completed", "task_id": f"tsk_{index}"},
}
for index in range(15)
]
packet = build_test_evidence_packet(
ctx, {"ok": True, "data": {"workflow_run_id": "wr_1", "overall_status": "completed", "blocks": blocks}}
)
assert [block.block_label for block in packet.ai_fallback_blocks or []] == [
f"loop_step_{index}" for index in range(3, 15)
]
assert any(notice.startswith("ai_fallback_blocks shortened: 3 oldest") for notice in packet.omission_notices)
@pytest.mark.asyncio
async def test_the_prior_run_packet_and_its_recorded_outcome_name_the_rescued_block(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
ctx = make_copilot_ctx(workflow_permanent_id="wpid-1")
hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx, read_live_page=False)
outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
rescued = {
"block_label": "collect_metrics",
"status": "fired_completed",
"task_id": "tsk_fallback",
"failing_line": 5,
"failure_text": _CODE_ERROR,
}
assert outcome is not None
assert [block.model_dump(mode="json", exclude_none=True) for block in outcome.ai_fallback_blocks or []] == [rescued]
assert hydrated is not None
assert hydrated["ai_fallback_blocks"] == [rescued]
@pytest.mark.asyncio
async def test_a_run_without_an_ai_fallback_reads_exactly_as_before(monkeypatch: pytest.MonkeyPatch) -> None:
reads: list[str] = []
for episodes in (
[],
[_ai_fallback_episode(HealStatus.skipped, failing_line=5, skip_reason=HealSkipReason.no_goal)],
):
_install_rescued_run(monkeypatch, episodes)
ctx = make_copilot_ctx(workflow_permanent_id="wpid-1")
page = await _run_results_page(ctx)
hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx, read_live_page=False)
outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
assert hydrated is not None
assert outcome is not None
reads.append(json.dumps([page, hydrated, outcome.model_dump(mode="json", exclude_none=True)], default=str))
assert reads[0] == reads[1]
for absent in ("ai_fallback", "parent_block_label"):
assert absent not in reads[0]
_PARSE_THEN_LOOP_YAML = """
title: parse then loop
workflow_definition:
parameters: []
blocks:
- block_type: file_url_parser
label: parse_rows
file_url: https://fixture.test/rows.csv
file_type: csv
next_block_label: visit_each_row
- block_type: for_loop
label: visit_each_row
loop_over_parameter_key: parse_rows_output
{reference}
loop_blocks:
- block_type: goto_url
label: open_row_url
url: "{{{{ current_value.url }}}}"
"""
_LoopFactRow = tuple[
str,
str | None,
str | None,
str | None,
LoopSelectedInput,
int | None,
list[int | Literal["not_recorded"]] | None,
int | None,
]
_NESTED_LOOP_YAML = """
title: nested loop
workflow_definition:
parameters:
- parameter_type: workflow
key: groups
workflow_parameter_type: json
default_value: [{"urls": ["https://fixture.test/a"]}]
blocks:
- block_type: for_loop
label: each_group
loop_over_parameter_key: groups
loop_blocks:
- block_type: for_loop
label: each_url
loop_variable_reference: current_value.urls
loop_blocks:
- block_type: goto_url
label: open_url
url: "{{ current_value }}"
"""
_REFERENCE_NAMES_A_PARAMETER_YAML = """
title: reference names a parameter
workflow_definition:
parameters:
- parameter_type: workflow
key: groups
workflow_parameter_type: json
default_value: [{}]
- parameter_type: workflow
key: other_groups
workflow_parameter_type: json
default_value: [{}]
blocks:
- block_type: for_loop
label: each_group
loop_over_parameter_key: other_groups
loop_variable_reference: "{{ groups }}"
loop_blocks:
- block_type: goto_url
label: open_url
url: https://fixture.test/a
"""
def _parse_then_loop_yaml(reference: str | None) -> str:
return _PARSE_THEN_LOOP_YAML.format(
reference=f"loop_variable_reference: {json.dumps(reference)}" if reference else ""
)
@pytest.mark.asyncio
async def test_failed_both_inputs_run_reports_the_dispatched_versions_selected_reference_to_the_model(
monkeypatch: pytest.MonkeyPatch,
) -> None:
parser_row = terminal_extraction_block("completed", label="parse_rows").model_copy(
update={"block_type": BlockType.FILE_URL_PARSER}
)
failed_loop_row = terminal_extraction_block(
"failed", label="visit_each_row", failure_reason="Failed to get loop values."
).model_copy(update={"block_type": BlockType.FOR_LOOP})
harness = await install_run_blocks_harness(
monkeypatch,
workflow_yaml=_parse_then_loop_yaml("current_value"),
polled_status="failed",
dispatch_to_worker=True,
terminal_blocks=[parser_row, failed_loop_row],
)
dispatched = harness["workflow"].model_copy(deep=True, update={"workflow_id": "w_dispatch_v8", "version": 8})
forge_app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version = AsyncMock(return_value=dispatched)
async def execute_and_rewrite_reference(**_kwargs: object) -> None:
loop = dispatched.workflow_definition.blocks[1]
assert isinstance(loop, ForLoopBlock)
loop.loop_variable_reference = "extracted_loop_values_runtime"
harness["worker_execute"].side_effect = execute_and_rewrite_reference
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.frontier_resume_session_id = "pbs_run"
result = await _run_blocks_and_collect_debug(
{"block_labels": ["parse_rows", "visit_each_row"], "parameters": {}}, ctx
)
finalize_build_test_result(ctx, source_tool="update_and_run_blocks", result=result)
model_facing = sanitize_tool_result_for_llm("update_and_run_blocks", result)["data"]
parser_output_id = dispatched.workflow_definition.blocks[0].output_parameter.output_parameter_id
assert "loop_inputs" not in model_facing
assert model_facing["build_test_packet"]["run"]["loop_inputs"] == {
"workflow_run_id": "wr_paused",
"workflow_id": "w_dispatch_v8",
"workflow_permanent_id": "wfp-1",
"version": 8,
"loops": [
{
"block_label": "visit_each_row",
"loop_over_parameter_key": "parse_rows_output",
"producer_block_label": "parse_rows",
"producer_output_parameter_id": parser_output_id,
"loop_variable_reference": "current_value",
"selected_input": "loop_variable_reference",
"run_rows": 1,
"loop_values_counts": ["not_recorded"],
}
],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("workflow_yaml", "rows", "expected"),
[
pytest.param(
_parse_then_loop_yaml(None),
[
terminal_extraction_block("completed", label="visit_each_row").model_copy(
update={"block_type": BlockType.FOR_LOOP, "loop_values": ["a", "b"]}
)
],
[("visit_each_row", None, "parse_rows_output", None, "loop_over_parameter_key", 1, [2], None)],
id="typed-only",
),
pytest.param(
_parse_then_loop_yaml("{{ parse_rows_output.rows }}"),
[],
[
(
"visit_each_row",
None,
"parse_rows_output",
"{{ parse_rows_output.rows }}",
"loop_variable_reference",
None,
None,
None,
)
],
id="explicit-expression-never-reached",
),
pytest.param(
_REFERENCE_NAMES_A_PARAMETER_YAML,
[
terminal_extraction_block("completed", label="each_group").model_copy(
update={"block_type": BlockType.FOR_LOOP, "loop_values": [{}]}
)
],
[("each_group", None, None, "{{ groups }}", "loop_variable_reference", 1, [1], None)],
id="reference-names-a-parameter",
),
pytest.param(
_NESTED_LOOP_YAML,
[
terminal_extraction_block("completed", label="each_group").model_copy(
update={"block_type": BlockType.FOR_LOOP, "loop_values": [{}]}
)
]
+ [
terminal_extraction_block("completed", label="each_url").model_copy(
update={"block_type": BlockType.FOR_LOOP, "loop_values": list(range(minute)) or None}
)
for minute in range(12)
],
[
("each_group", None, "groups", None, "loop_over_parameter_key", 1, [1], None),
(
"each_url",
"each_group",
None,
"current_value.urls",
"loop_variable_reference",
12,
list(range(2, 12)),
2,
),
],
id="nested-keeps-newest-ten-rows",
),
],
)
async def test_loop_input_facts_report_each_loops_own_declared_and_recorded_input(
monkeypatch: pytest.MonkeyPatch,
workflow_yaml: str,
rows: list[WorkflowRunBlock],
expected: list[_LoopFactRow],
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
definitions = loop_definition_facts(await inert_approval_workflow(workflow_yaml, workflow_id="w_source"))
assert definitions is not None
facts = with_loop_observations(definitions, "wr_paused", rows)
assert [
(
loop.block_label,
loop.enclosing_loop_label,
loop.loop_over_parameter_key,
loop.loop_variable_reference,
loop.selected_input,
loop.run_rows,
loop.loop_values_counts,
loop.loop_values_counts_omitted,
)
for loop in facts.loops
] == expected
@pytest.mark.asyncio
async def test_a_loop_over_id_from_an_older_version_still_names_its_producer_by_key(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
workflow = await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
parser, loop = workflow.workflow_definition.blocks
assert isinstance(loop, ForLoopBlock) and isinstance(loop.loop_over, OutputParameter)
loop.loop_over = loop.loop_over.model_copy(update={"output_parameter_id": "op_from_version_2"})
definitions = loop_definition_facts(workflow)
assert definitions is not None
[fact] = definitions.loops
assert (fact.producer_block_label, fact.producer_output_parameter_id) == (
"parse_rows",
parser.output_parameter.output_parameter_id,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("reference", "loop_over", "expected"),
[
("current_value", True, "loop_variable_reference"),
("", True, "loop_over_parameter_key"),
(None, False, "none"),
],
)
async def test_selected_input_matches_the_branch_the_real_for_loop_getter_takes(
reference: str | None, loop_over: bool, expected: str
) -> None:
now = datetime(2026, 4, 21, 12, 0, tzinfo=UTC)
block = ForLoopBlock(
label="visit_each_row",
output_parameter=OutputParameter(
output_parameter_id="op_loop",
key="visit_each_row_output",
workflow_id="w",
created_at=now,
modified_at=now,
),
loop_blocks=[],
loop_over=(
WorkflowParameter(
workflow_parameter_id="wp_rows",
workflow_parameter_type=WorkflowParameterType.JSON,
key="rows",
workflow_id="w",
created_at=now,
modified_at=now,
)
if loop_over
else None
),
loop_variable_reference=reference,
complete_if_empty=True,
)
branch_values = {"loop_variable_reference": ["from_reference"], "loop_over_parameter_key": ["from_key"], "none": []}
context = MagicMock()
context.get_value.return_value = branch_values["loop_over_parameter_key"]
with patch.object(
ForLoopBlock,
"get_values_from_loop_variable_reference",
AsyncMock(return_value=branch_values["loop_variable_reference"]),
):
observed = await block.get_loop_over_parameter_values(context, "wr", "wrb")
assert loop_selected_input(block) == expected
assert observed == branch_values[expected]
@pytest.mark.asyncio
async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_current_draft(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
executed = (await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="wf-1")).model_copy(
update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}
)
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
terminal_extraction_block(
"failed", label="visit_each_row", failure_reason="Failed to get loop values."
).model_copy(update={"block_type": BlockType.FOR_LOOP})
],
)
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
ctx.staged_workflow = await inert_approval_workflow(_parse_then_loop_yaml(None), workflow_id="w_source")
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
page = sanitize_tool_result_for_llm("get_run_results", project_run_results_page(result, {}))["data"]
assert page["loop_inputs"]["workflow_run_id"] == "wr-1"
assert page["loop_inputs"]["workflow_id"] == "wf-1"
[loop] = page["loop_inputs"]["loops"]
assert (loop["loop_variable_reference"], loop["selected_input"], loop["loop_values_counts"]) == (
"current_value",
"loop_variable_reference",
["not_recorded"],
)
assert "Failed to get loop values" not in json.dumps(page["loop_inputs"])
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=None)
unresolved = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert unresolved["data"]["loop_inputs"] == {"executed_definition": "unavailable"}
packet = build_test_evidence_packet(_locator_packet_ctx(), unresolved)
assert packet.run.loop_inputs is not None
assert packet.run.loop_inputs.model_dump() == {"executed_definition": "unavailable"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("workflow_yaml", "created_by", "expected_unavailable"),
[
(_parse_then_loop_yaml("current_value"), None, True),
(_parse_then_loop_yaml("current_value"), COPILOT_TEST_WORKFLOW_CREATOR, False),
(SEARCH_THEN_SELECT_WORKFLOW_YAML, None, True),
],
ids=[
"saved-version-overwritten-after-run",
"test-version-soft-deleted-after-run",
"loop-removed-from-saved-version-after-run",
],
)
async def test_read_run_withholds_loop_inputs_when_the_workflow_row_changed_after_the_run(
monkeypatch: pytest.MonkeyPatch, workflow_yaml: str, created_by: str | None, expected_unavailable: bool
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
executed = (await inert_approval_workflow(workflow_yaml, workflow_id="wf-1")).model_copy(
update={
"workflow_permanent_id": "wpid-1",
"created_by": created_by,
"modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5),
}
)
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
terminal_extraction_block(
"failed", label="visit_each_row", failure_reason="Failed to get loop values."
).model_copy(update={"block_type": BlockType.FOR_LOOP})
],
)
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert (result["data"]["loop_inputs"] == {"executed_definition": "unavailable"}) is expected_unavailable
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ["aware_modified_at", "loop_definition_facts", "with_loop_observations"])
async def test_read_run_degrades_loop_inputs_to_unavailable_instead_of_failing(
monkeypatch: pytest.MonkeyPatch, fault: str
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
modified_at = (
HARNESS_RUN_CREATED_AT.replace(tzinfo=UTC) + timedelta(minutes=5)
if fault == "aware_modified_at"
else HARNESS_RUN_CREATED_AT - timedelta(minutes=1)
)
executed = (
await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
).model_copy(update={"modified_at": modified_at})
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
terminal_extraction_block("failed", label="visit_each_row").model_copy(
update={"block_type": BlockType.FOR_LOOP}
)
],
)
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
if fault != "aware_modified_at":
monkeypatch.setattr(run_execution_module, fault, MagicMock(side_effect=ValueError("projection failed")))
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert result["data"]["loop_inputs"] == {"executed_definition": "unavailable"}
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ["loop_definition_facts", "with_loop_observations"])
async def test_build_test_loop_inputs_degrade_to_unavailable_when_the_projection_raises(
monkeypatch: pytest.MonkeyPatch, fault: str
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
workflow = await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
monkeypatch.setattr(run_execution_module, fault, MagicMock(side_effect=ValueError("projection failed")))
execution = run_execution_module._RunExecution(
snapshot=run_execution_module.CopilotExecutionSnapshot(
provenance="staged",
workflow=workflow,
workflow_parameters=(),
output_parameters=(),
workflow_yaml="",
),
workflow_yaml="",
metadata={},
associations={},
source_at_start=None,
unbound_keys=[],
explicit_blank=False,
loop_definitions=run_execution_module._loop_definitions_or_unavailable(workflow),
)
data: dict[str, Any] = {}
run_execution_module._attach_loop_inputs(data, execution, "wr-1", [])
assert data["loop_inputs"] == {"executed_definition": "unavailable"}
@pytest.mark.asyncio
async def test_read_run_reports_no_loop_inputs_for_an_unresolved_workflow_that_ran_no_loop(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = install_get_run_results_harness(
monkeypatch, blocks=[terminal_extraction_block("failed", label="open_result")]
)
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=None)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
assert "loop_inputs" not in result["data"]
assert build_test_evidence_packet(_locator_packet_ctx(), result).run.loop_inputs is None
@pytest.mark.asyncio
async def test_prior_run_packet_and_handoff_carry_the_notice_for_a_truncated_loop_reference(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
executed = (
await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="wf-1")
).model_copy(
update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}
)
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
terminal_extraction_block("failed", label="visit_each_row").model_copy(
update={"block_type": BlockType.FOR_LOOP}
)
],
)
run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
packet = build_test_evidence_packet(_locator_packet_ctx(), result)
assert isinstance(packet.run.loop_inputs, LoopInputs)
assert len(packet.run.loop_inputs.loops[0].loop_variable_reference or "") <= 200
assert any("loop_variable_reference" in notice for notice in packet.omission_notices)
handoff = project_direct_test_handoff_packet_for_llm(packet)
assert any("loop_variable_reference" in notice for notice in handoff.omission_notices)
@pytest.mark.parametrize("ai_fallback_rows", [0, 12])
def test_many_long_loop_facts_stay_inside_the_packet_budget(ai_fallback_rows: int) -> None:
long_label = "l" * 400
loop_inputs = LoopInputs(
workflow_run_id="wr_1",
workflow_id="w_dispatch",
workflow_permanent_id="wpid",
version=3,
loops=[
LoopInputFact(
block_label=f"{long_label}{index}",
enclosing_loop_label=long_label,
loop_over_parameter_key=f"{long_label}_output",
producer_block_label=long_label,
loop_variable_reference="r" * 5_000,
selected_input="loop_variable_reference",
run_rows=12 + index,
loop_values_counts=[1_000_000] * 10,
loop_values_counts_omitted=2,
)
for index in range(20)
],
)
packet = _oversized_packet(build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result(None)))
ai_fallback_blocks = [
BuildTestPacketAiFallbackBlock(
block_label=f"{long_label}{index}",
status="fired_failed",
task_id=f"tsk_{index}",
failing_line=index,
failure_text="f" * 2_000,
recovery_failure_text="g" * 2_000,
)
for index in range(ai_fallback_rows)
]
packet = packet.model_copy(
update={
"run": packet.run.model_copy(update={"loop_inputs": loop_inputs}),
"ai_fallback_blocks": ai_fallback_blocks or None,
}
)
projected = project_build_test_packet_for_llm(packet)
serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
assert len(serialized) <= _BUILD_TEST_PACKET_MAX_CHARS
assert isinstance(projected.run.loop_inputs, LoopInputs)
# Loop facts give way before canonical_workflow_yaml is shortened.
assert [loop.run_rows for loop in projected.run.loop_inputs.loops] == [12 + index for index in range(6)]
assert projected.run.loop_inputs.loops_omitted == 14
assert all(len(loop.loop_variable_reference or "") <= 200 for loop in projected.run.loop_inputs.loops)
assert all(loop.selected_input == "loop_variable_reference" for loop in projected.run.loop_inputs.loops)
assert all(loop.loop_values_counts == [1_000_000] * 10 for loop in projected.run.loop_inputs.loops)
_EXECUTED_A_CODE = 'greeting = "Hello " + legacy_applicant_name\nreturn {"greeting": greeting}\n'
_EDITABLE_B_CODE = 'display = greeting_prefix + " " + applicant_display_name\nreturn {"greeting": display}\n'
_EDITABLE_B_YAML = """title: Historical run source
workflow_definition:
parameters:
- parameter_type: workflow
key: full_name
workflow_parameter_type: string
- parameter_type: workflow
key: greeting_prefix
workflow_parameter_type: string
blocks:
- block_type: code
label: greet_applicant
parameter_keys: [full_name, greeting_prefix]
code: |
display = greeting_prefix + " " + applicant_display_name
return {"greeting": display}
"""
def _editable_b_ctx() -> CopilotContext:
return make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml=_EDITABLE_B_YAML)
@pytest.mark.asyncio
async def test_get_run_results_and_hydration_name_the_version_a_finished_run_executed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
install_historical_run(
monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"
)
monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)
ctx = _editable_b_ctx()
raw = await tools_module.get_run_results_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps({"workflow_run_id": "wr-1"})
)
hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
tool_source = json.loads(raw)["data"]["execution_source"]
assert hydrated is not None
assert hydrated["run"]["execution_source"] == tool_source
assert tool_source == {
"source_kind": "run_version",
"disposition": "available",
"workflow_run_id": "wr-1",
"workflow_id": "wf-1",
"workflow_version": 3,
"failed_block_label": "greet_applicant",
"failed_statement": 'greeting = "Hello " + legacy_applicant_name',
"declared_parameter_keys": ["applicant_name"],
"block_code_sha256": {"greet_applicant": hashlib.sha256(_EXECUTED_A_CODE.encode()).hexdigest()},
"current_draft_code_matches": {"greet_applicant": False},
"failed_block_code": _EXECUTED_A_CODE,
}
executed_fields = json.dumps(tool_source)
assert "applicant_display_name" not in executed_fields
assert hashlib.sha256(_EDITABLE_B_CODE.encode()).hexdigest() not in executed_fields
@pytest.mark.asyncio
async def test_a_historical_read_takes_no_signature_or_unverified_label_from_the_current_draft(
monkeypatch: pytest.MonkeyPatch,
) -> None:
install_historical_run(
monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"
)
ctx = _editable_b_ctx()
ctx.last_unverified_block_labels = ["greet_applicant", "only_in_b"]
ctx.unbound_required_parameter_keys = ["full_name"]
ctx.runner_code_block_associations_by_label = {"greet_applicant": "assoc_b"}
history_before = copy.deepcopy(ctx.recorded_build_test_outcome_history)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
draft_outcome = run_execution_module._build_recorded_build_test_outcome(
ctx,
{**result, "data": {key: value for key, value in result["data"].items() if key != "execution_source"}},
recorded_run_outcome=None,
)
assert draft_outcome is not None and draft_outcome.authored_structure_signature
assert outcome is not None
assert outcome.workflow_run_id == "wr-1"
assert outcome.authored_structure_signature is None
assert outcome.block_shape_hashes == {}
assert outcome.executed_block_associations == ()
assert hydrated is not None
assert [item for item in hydrated.get("unfinished_items", []) if item.get("kind") == "unverified_block"] == []
assert ctx.recorded_build_test_outcome_history == history_before
assert ctx.latest_recorded_build_test_outcome is None
@pytest.mark.asyncio
@pytest.mark.parametrize("definition_matches", [True, False], ids=["metadata-only-edit", "definition-rewritten"])
async def test_the_definition_recorded_at_dispatch_decides_whether_an_edited_row_is_what_ran(
monkeypatch: pytest.MonkeyPatch, definition_matches: bool
) -> None:
# A folder or title edit bumps modified_at past the run; only a definition change may withhold its source.
version = historical_code_version(
{"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(minutes=5)
)
recorded = historical_code_version({"greet_applicant": _EXECUTED_A_CODE if definition_matches else "pass\n"})
install_historical_run(
monkeypatch,
version,
failed_label="greet_applicant",
run_definition_sha256=workflow_definition_sha256(recorded.workflow_definition),
)
observe_locators = AsyncMock(return_value=None)
monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, _editable_b_ctx())
source = result["data"]["execution_source"]
if definition_matches:
assert source["disposition"] == "available"
assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'
assert observe_locators.await_args.kwargs["failed_block_code"] == _EXECUTED_A_CODE
assert "requested_output_parameter_definitions" in result["data"]
else:
assert source == {
"source_kind": "run_version",
"disposition": "unavailable",
"workflow_run_id": "wr-1",
"workflow_id": "wf-1",
"reason": "overwritten_after_run_start",
}
assert "requested_output_parameter_definitions" not in result["data"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("version", "reason"),
[
(None, "version_missing"),
(historical_code_version({"greet_applicant": _EXECUTED_A_CODE}, workflow_id="wf-other"), "identity_mismatch"),
(
historical_code_version(
{"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(seconds=1)
),
"overwritten_after_run_start",
),
(historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), None),
],
)
async def test_an_unprovable_executed_version_is_unavailable_and_never_falls_back_to_the_draft(
monkeypatch: pytest.MonkeyPatch, version: SimpleNamespace | None, reason: str | None
) -> None:
install_historical_run(monkeypatch, version, failed_label="greet_applicant")
observe_locators = AsyncMock(return_value=None)
monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)
ctx = _editable_b_ctx()
result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
source = result["data"]["execution_source"]
observed_code = observe_locators.await_args.kwargs["failed_block_code"] if observe_locators.await_args else None
assert outcome is not None and outcome.authored_structure_signature is None
assert "applicant_display_name" not in json.dumps(source)
fetched_terminal_page = run_execution_module._fetch_dispatched_terminal_page_evidence.await_count > 0
if reason is None:
assert source["disposition"] == "available"
assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'
assert observed_code == _EXECUTED_A_CODE
assert "requested_output_parameter_definitions" in result["data"]
assert fetched_terminal_page
return
# An unproven row may have lost the sensitive parameters or outputs the run had, so nothing is read from it.
assert observed_code is None
assert "requested_output_parameter_definitions" not in result["data"]
assert not fetched_terminal_page
assert source == {
"source_kind": "run_version",
"disposition": "unavailable",
"workflow_run_id": "wr-1",
"workflow_id": "wf-1",
"reason": reason,
}
_SAVED_SIGN_IN_AND_SEARCH_YAML = """
title: sign in and search
workflow_definition:
version: 2
parameters:
- key: login_credentials
parameter_type: workflow
workflow_parameter_type: credential_id
default_value: cred_saved
- key: search_term
parameter_type: workflow
workflow_parameter_type: string
default_value: widgets
blocks:
- block_type: login
label: sign_in
url: https://example.com/login
parameter_keys: [login_credentials]
next_block_label: search
- block_type: code
label: search
parameter_keys: [login_credentials, search_term]
code: |
await page.fill("#q", search_term)
"""
_PROPOSAL_OUTPUT_KINDS = {
CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL.value,
CopilotOutputKind.WORKFLOW_UPDATE_PROPOSAL.value,
}
def _saved_sign_in_ctx() -> CopilotContext:
return make_copilot_ctx(
browser_session_id="pbs_chat",
workflow_copilot_chat_id="chat-1",
request_policy=RequestPolicy(persisted_workflow_credential_ids=["cred_saved"]),
)
async def _blank_browser_retest_of_saved_workflow(
monkeypatch: pytest.MonkeyPatch, ctx: CopilotContext, status: str
) -> None:
await install_run_blocks_harness(
monkeypatch,
workflow_yaml=_SAVED_SIGN_IN_AND_SEARCH_YAML,
polled_status=status,
dispatch_to_worker=True,
terminal_blocks=[
terminal_extraction_block("completed", label="sign_in"),
terminal_extraction_block(status, label="search"),
],
)
forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
monkeypatch.setattr(run_execution_module, "_credential_ids_validation_error", AsyncMock(return_value=None))
async def mint_blank_browser(acquisition_ctx: CopilotContext, *, fresh: bool, **_kwargs: object) -> None:
acquisition_ctx.browser_session_id = "pbs_run"
monkeypatch.setattr(run_execution_module, "acquire_build_test_browser_session", mint_blank_browser)
workflow_service_module.prepare_workflow.return_value.browser_session_id = "pbs_run"
result = await run_execution_module.run_workflow_end_to_end(ctx)
assert result["data"]["execution_source"]["source_kind"] == "canonical", result
@pytest.mark.asyncio
async def test_canonical_blank_browser_retest_records_yaml_the_accept_resolver_keeps_bindings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _saved_sign_in_ctx()
await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, "failed")
assert ctx.last_workflow is not None and ctx.last_workflow_yaml
proposal = workflow_copilot_route._build_proposed_workflow_data(
ctx.last_workflow,
AgentResult(
user_response="",
updated_workflow=ctx.last_workflow,
workflow_yaml=ctx.last_workflow_yaml,
global_llm_context=None,
),
)
resolved = await workflow_copilot_route._resolve_proposal_for_accept(
proposal, organization_id="org-1", workflow_permanent_id="wfp-1"
)
blocks = {block.label: block for block in resolved.workflow_definition.blocks}
assert blocks["search"].parameter_keys == ["login_credentials", "search_term"]
assert blocks["sign_in"].parameter_keys == ["login_credentials"]
assert {parameter.key for parameter in resolved.workflow_definition.parameters} >= {
"login_credentials",
"search_term",
}
assert ctx.latest_diagnosis_repair_contract is not None
assert {
lever.credential_id
for lever in ctx.latest_diagnosis_repair_contract.levers
if lever.mechanism == "credential_update"
} == {"cred_saved"}
@pytest.mark.asyncio
@pytest.mark.parametrize("run_statuses", [("completed",), ("failed",), ("completed", "failed")])
@pytest.mark.parametrize(
("response_type", "product_action"),
[("REPLY", None), ("ASK_QUESTION", None), ("REPLY", "test_end_to_end")],
)
async def test_canonical_blank_browser_retests_stage_no_proposal_on_any_exit(
monkeypatch: pytest.MonkeyPatch, run_statuses: tuple[str, ...], response_type: str, product_action: str | None
) -> None:
ctx = _saved_sign_in_ctx()
for status in run_statuses:
await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, status)
failed = run_statuses[-1] == "failed"
assert ctx.last_workflow is not None and ctx.last_test_ok is not failed
chat_request = SimpleNamespace(
workflow_id="wf-1",
workflow_permanent_id="wfp-1",
workflow_copilot_chat_id="chat-1",
workflow_run_id=None,
workflow_yaml="",
product_action=product_action,
)
normal_exit = await _translate_to_agent_result(
SimpleNamespace(
final_output=json.dumps({"type": response_type, "user_response": "I re-ran the saved workflow."}),
new_items=[],
),
ctx,
None,
chat_request,
"org-1",
)
wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")
for result in (normal_exit, wip_exit):
assert (result.updated_workflow, result.workflow_yaml, result.staged_workflow) == (None, None, None)
assert result.proposal_disposition == "no_proposal"
assert result.clear_proposed_workflow is False
assert (result.output_policy_diagnostics or {}).get("final_output_kind") not in _PROPOSAL_OUTPUT_KINDS
assert ctx.last_workflow is not None
if failed:
ctx.saved_workflows_under_test.clear()
draft_wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")
assert draft_wip_exit.updated_workflow is ctx.last_workflow
assert wip_exit.user_response != draft_wip_exit.user_response