Skip to content

Automate browser based workflows with AI

SKY-11031: promote copilot outcome-verification and persisted-criteria to unconditional defaults (#6591)

Andrew Neilsona35c768b3b81
diff --git a/skyvern/config.py b/skyvern/config.py
index 7c560a99f..daa107a1b 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -151,17 +151,13 @@ class Settings(BaseSettings):
COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0
COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0
COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0
- # Gate copilot verified-success on per-criterion outcome evidence.
- # Off restores the prior run-status/suspicious-success gate and classifier prompt.
- COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = True
+ # Capture bounded page evidence synchronously in scout interaction post-hooks.
+ # Off restores schema-less interaction packets plus standalone page inspects.
+ COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True
COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0
# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process
# instead of only advising. Off (default) leaves the advisory steer as the only response.
COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False
- # Persist copilot completion criteria per goal epoch with tri-state verdicts
- # and definition-plane grading. Off restores per-turn re-derivation, the legacy
- # run-status success claim, and unknown-verdicts-as-unsatisfied consumer behavior.
- COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED: bool = True
# Dispatch flag for the workflow copilot v2 (openai-agents-SDK rewrite).
# Off = existing direct-LLM copilot at workflow_copilot_chat_post.
# On = new agent-SDK path under skyvern.forge.sdk.copilot.
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index c6b081e3d..dc37ff00e 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -1,5 +1,5 @@
Classify the latest Workflow Copilot request. Return only JSON with keys:
-testing_intent=require_test|skip_test|unspecified; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; {% if outcome_verification_enabled %}completion_criteria; {% endif %}raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
+testing_intent=require_test|skip_test|unspecified; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
testing_intent:
- skip_test: the user asks for an untested draft, says they will test it themselves, says draft only, just sketch it, no validation run, or skip testing.
@@ -24,7 +24,7 @@ completion_contract:
- The value must be an exact span copied from the latest user message. Do not paraphrase, summarize, or infer a completion condition.
- Set to null when the user does not provide an explicit completion condition.
-{% if outcome_verification_enabled %}completion_criteria:
+completion_criteria:
- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, implicit, method_mandated, level}.
- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.
@@ -35,7 +35,6 @@ completion_contract:
- Do not emit a criterion that restates or negates another criterion in the list (for example a "values are not hardcoded" duplicate of a reusable-inputs criterion); one outcome, one criterion. State a definition-level outcome as the positive checkable property of the named inputs, never as a negation.
- When ACTIVE COMPLETION CRITERIA are provided below, they are the canonical phrasing of the chat's current goal. If the latest request keeps that goal — including repeat, retest, fix, and follow-up turns — emit each still-required outcome COPIED VERBATIM from that list, character for character, punctuation included; never rephrase or reword a covered outcome. Write new phrasing only for an outcome the list does not cover, and drop an item only when the latest request abandons or replaces it.
- Set to [] when the user gives no goal with a checkable outcome (for example a pure draft-only or sketch request).
-{% endif %}
raw_secret_evidence:
- When credential_input_kind=raw_secret, set this to the verbatim substring of the LATEST user message that is the pasted secret value (e.g. the password string, JWT, API key, OTP digits, or PEM body). Do not paraphrase. Do not cite a token that appears only in prior turns or in the workflow_yaml — the substring must occur in the latest user message.
@@ -85,6 +84,6 @@ Retained recent history:
```{{ retained_history }}```
Current workflow YAML:
```{{ workflow_yaml }}```
-{% if outcome_verification_enabled and active_completion_criteria %}ACTIVE COMPLETION CRITERIA (canonical phrasing for the current goal):
+{% if active_completion_criteria %}ACTIVE COMPLETION CRITERIA (canonical phrasing for the current goal):
```{{ active_completion_criteria }}```
{% endif %}Global context: ```{{ global_llm_context }}```
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 1e2d80ccc..04db6f3d5 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -30,7 +30,6 @@ import yaml
from litellm.exceptions import NotFoundError as LiteLLMNotFoundError
from pydantic import ValidationError
-from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -365,8 +364,6 @@ def _request_policy_agent_inputs(
def _stored_active_completion_criteria(
policy_inputs: RequestPolicyGuardrailInputs,
) -> list[CompletionCriterion] | None:
- if not (settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED and settings.COPILOT_OUTCOME_VERIFICATION_ENABLED):
- return None
snapshot = policy_inputs.stored_completion_criteria
if snapshot is None or snapshot.active is None:
return None
@@ -378,8 +375,6 @@ def _reconcile_completion_criteria_on_context(
policy: RequestPolicy,
policy_inputs: RequestPolicyGuardrailInputs,
) -> None:
- if not (settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED and settings.COPILOT_OUTCOME_VERIFICATION_ENABLED):
- return
snapshot = policy_inputs.stored_completion_criteria
if snapshot is None:
return
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index e5827bb05..ef406881f 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -53,14 +53,6 @@ VerificationStatus = Literal["evaluated", "unavailable"]
CriterionState = Literal["satisfied", "unsatisfied", "unknown"]
-def resolve_unknown(state: CriterionState) -> CriterionState:
- """Effective verdict state for consumers: with persisted-criteria off, ``unknown``
- collapses to ``unsatisfied`` (the pre-tri-state equivalent of a missing verdict)."""
- if state == "unknown" and not settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED:
- return "unsatisfied"
- return state
-
-
@dataclass(frozen=True)
class CriterionVerdict:
criterion_id: str
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 9b03f8eec..0f6e2f8f6 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -13,7 +13,6 @@ from urllib.parse import urlparse
import structlog
from agents.run import Runner
-from skyvern.config import settings
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
@@ -268,8 +267,6 @@ def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:
def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return False
result = ctx.completion_verification_result
return result is not None and result.status == "evaluated"
@@ -307,13 +304,9 @@ def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:
def verified_goal_claim_authorized(ctx: CopilotContext) -> bool:
"""Whether the terminal may CLAIM a tested success. Turn completion keeps
- flowing through ``verified_goal_satisfied_context``; with persisted criteria
- enabled, the claim tier additionally requires judge-confirmed outcome evidence —
- criteria-less or judge-less terminals end the turn but render built-but-unverified.
- With either flag off the judge cannot be authoritative, so the claim falls back
- to the legacy gate."""
- if not (settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED and settings.COPILOT_OUTCOME_VERIFICATION_ENABLED):
- return verified_goal_satisfied_context(ctx)
+ flowing through ``verified_goal_satisfied_context``; the claim tier additionally
+ requires judge-confirmed outcome evidence — criteria-less or judge-less terminals
+ end the turn but render built-but-unverified."""
return outcome_fully_verified(ctx)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 81b21dbfe..1afe7bb75 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -716,7 +716,6 @@ async def _classify_request(
transcript = build_transcript_context(chat_history, safe_user_message)
prompt = prompt_engine.load_prompt(
template=PROMPT_NAME,
- outcome_verification_enabled=settings.COPILOT_OUTCOME_VERIFICATION_ENABLED,
user_message=escape_code_fences(safe_user_message),
raw_secret_present=str(raw_secret_present).lower(),
workflow_yaml=escape_code_fences(redact_raw_secrets_for_prompt(workflow_yaml)[:2048]),
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 1892d1ac3..ca8349778 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -12,7 +12,6 @@ from skyvern.forge.sdk.copilot.completion_verification import (
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
- resolve_unknown,
summarize_unsatisfied_outcomes,
)
from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
@@ -46,8 +45,6 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:
def _split_criteria_by_plane(criteria: list[Any]) -> tuple[list[CompletionCriterion], list[CompletionCriterion]]:
- if not settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED:
- return list(criteria), []
run_criteria = [c for c in criteria if getattr(c, "level", "run") != "definition"]
definition_criteria = [c for c in criteria if getattr(c, "level", "run") == "definition"]
return run_criteria, definition_criteria
@@ -119,8 +116,6 @@ async def _maybe_run_completion_verification_from_page_observation(
) -> CompletionVerificationResult | None:
"""Verify completion only for post-run page observations after failed tests."""
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return None
existing = getattr(copilot_ctx, "completion_verification_result", None)
if isinstance(existing, CompletionVerificationResult) and existing.is_fully_satisfied():
return existing
@@ -269,8 +264,6 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
async def _maybe_run_completion_verification(
copilot_ctx: Any, result: dict[str, Any], handler_start: float
) -> CompletionVerificationResult | None:
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return None
if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
return None
criteria = _completion_verification_criteria(copilot_ctx)
@@ -365,9 +358,7 @@ def _outcome_failure_warrants_repair(
return True
# Repair needs at least one affirmatively unsatisfied criterion; unknown alone
# (absent judge signal, unmappable definition checks) never warrants repair.
- if settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED and not any(
- resolve_unknown(verdict.state) == "unsatisfied" for verdict in completion_verification.verdicts
- ):
+ if not any(verdict.state == "unsatisfied" for verdict in completion_verification.verdicts):
return False
return _current_workflow_has_evidence_block(copilot_ctx)
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 5153a7a98..ed0cdad30 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -9,7 +9,6 @@ from urllib.parse import urlparse
import structlog
-from skyvern.config import settings
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
RunEvidenceSnapshot,
@@ -73,8 +72,6 @@ class ActiveRunTerminalEvidenceSample:
async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return False
if not getattr(copilot_ctx, "browser_session_id", None):
return False
if not getattr(copilot_ctx, "discovery_mcp_server", None):
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 19f555329..e13fd3ebb 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -394,11 +394,8 @@ async def _clear_proposed_workflow(chat: Any) -> None:
async def _load_completion_criteria_snapshot(chat: Any) -> StoredCriteriaSnapshot | None:
- """None disables the lifecycle for this turn (flag off, or load failure —
- which degrades to today's per-turn regeneration rather than risking a
- duplicate-epoch write)."""
- if not (settings.COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED and settings.COPILOT_OUTCOME_VERIFICATION_ENABLED):
- return None
+ """None disables the lifecycle for this turn on load failure, which degrades to
+ today's per-turn regeneration rather than risking a duplicate-epoch write."""
try:
latest = await app.DATABASE.workflow_params.get_latest_workflow_copilot_completion_criteria_set(
organization_id=chat.organization_id,
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 8d95efae5..ab2c87357 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -6,9 +6,6 @@ from __future__ import annotations
from unittest.mock import MagicMock
-import pytest
-
-from skyvern.config import settings
from skyvern.forge.sdk.copilot.agent import (
RequestPolicyGuardrailInputs,
_reconcile_completion_criteria_on_context,
@@ -32,7 +29,6 @@ from skyvern.forge.sdk.copilot.completion_verification import (
_coerce_result,
combine_verification_results,
grade_definition_criteria,
- resolve_unknown,
run_plane_all_no_evidence,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -306,13 +302,6 @@ def test_coerce_result_tristate_mapping() -> None:
assert result.verdict_state_counts() == {"satisfied": 1, "unsatisfied": 3, "unknown": 2}
-def test_resolve_unknown_collapses_to_unsatisfied_only_when_flag_off(monkeypatch: pytest.MonkeyPatch) -> None:
- assert resolve_unknown("unknown") == "unknown"
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- assert resolve_unknown("unknown") == "unsatisfied"
- assert resolve_unknown("satisfied") == "satisfied"
-
-
_PARAMETERIZED_YAML = """\
title: provider lookup
workflow_definition:
@@ -438,7 +427,7 @@ def test_combine_results_merges_planes_and_fills_unknown() -> None:
assert combined.criterion_ids == ["c0", "c1", "c2"]
-def test_split_criteria_by_plane_respects_flag(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_split_criteria_by_plane() -> None:
criteria = [
_criterion("c0", "the item is in the cart"),
_criterion("c1", "inputs are reusable", level="definition"),
@@ -447,11 +436,6 @@ def test_split_criteria_by_plane_respects_flag(monkeypatch: pytest.MonkeyPatch)
assert [c.id for c in run_criteria] == ["c0"]
assert [c.id for c in definition_criteria] == ["c1"]
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
- assert [c.id for c in run_criteria] == ["c0", "c1"]
- assert definition_criteria == []
-
def test_parse_completion_criteria_coerces_level() -> None:
parsed = _parse_completion_criteria(
@@ -502,12 +486,6 @@ def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:
assert verified_goal_claim_authorized(ctx) is False
-def test_claim_authorized_with_flag_off_matches_legacy_gate(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- ctx = _legacy_verified_ctx()
- assert verified_goal_claim_authorized(ctx) is True
-
-
def test_claim_authorized_with_adjudicated_evidence() -> None:
ctx = _legacy_verified_ctx()
ctx.completion_verification_result = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))
@@ -520,13 +498,6 @@ def test_unknown_only_verdicts_never_route_to_repair() -> None:
assert _outcome_failure_warrants_repair(ctx, all_unknown) is False
-def test_unknown_verdicts_route_to_repair_with_flag_off(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- ctx = _ctx(last_workflow_yaml="workflow_definition:\n blocks:\n - block_type: extraction\n label: e\n")
- all_unknown = _evaluated(_verdict("c0", "unknown", "unknown"))
- assert _outcome_failure_warrants_repair(ctx, all_unknown) is True
-
-
def _policy_inputs(snapshot: StoredCriteriaSnapshot | None) -> RequestPolicyGuardrailInputs:
return RequestPolicyGuardrailInputs(
user_message="build a workflow",
@@ -553,27 +524,20 @@ def test_reconcile_on_context_adopts_stored_criteria_onto_policy() -> None:
assert ctx.completion_criteria_turn_state.decision.reason == "kept"
-def test_reconcile_on_context_skips_without_snapshot_or_flag(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_reconcile_on_context_skips_without_snapshot() -> None:
policy = RequestPolicy(completion_criteria=[_criterion("c0", "the item is in the cart")])
ctx = _ctx()
_reconcile_completion_criteria_on_context(ctx, policy, _policy_inputs(None))
assert ctx.completion_criteria_turn_state is None
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- _reconcile_completion_criteria_on_context(ctx, policy, _policy_inputs(StoredCriteriaSnapshot()))
- assert ctx.completion_criteria_turn_state is None
-
-def test_stored_active_criteria_forwarded_only_with_active_set(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_stored_active_criteria_forwarded_only_with_active_set() -> None:
stored = _stored("the item is in the cart")
snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=2)
assert _stored_active_completion_criteria(_policy_inputs(snapshot)) == list(stored.criteria)
assert _stored_active_completion_criteria(_policy_inputs(StoredCriteriaSnapshot())) is None
assert _stored_active_completion_criteria(_policy_inputs(None)) is None
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- assert _stored_active_completion_criteria(_policy_inputs(snapshot)) is None
-
def test_plan_persistence_for_supersede_creates_new_epoch_and_points_back() -> None:
stored = _stored("the item is in the cart", set_id="wccs_old", epoch=2)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 7da1e7fef..0a83c621c 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -32,7 +32,6 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
_verification_satisfaction,
)
from skyvern.forge.sdk.copilot.enforcement import (
- _outcome_criteria_evaluated,
outcome_fully_verified,
verified_goal_satisfied_context,
)
@@ -360,14 +359,6 @@ def test_gate_bypasses_heuristic_only_on_evaluated_verdict() -> None:
assert verified_goal_satisfied_context(retained) is False
-def test_gate_flag_off_ignores_verdict(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", False)
- ctx = _gate_ctx()
- ctx.completion_verification_result = _evaluated(("c0", True))
- assert _outcome_criteria_evaluated(ctx) is False
- assert verified_goal_satisfied_context(ctx) is False
-
-
def test_gate_withholds_on_evaluated_unconfirmed_even_with_clean_run_status() -> None:
# The judge verdict is authoritative in both directions: an evaluated-but-
# unconfirmed verdict withholds even when run-status latches and the diagnosis
@@ -826,7 +817,7 @@ def test_active_run_terminal_evidence_blocks_same_turn_mutation_tools() -> None:
assert ctx.blocker_signal.internal_reason_code == "tool_error_active_run_terminal_evidence"
-def test_outcome_fully_verified_predicate(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_outcome_fully_verified_predicate() -> None:
ctx = _gate_ctx()
ctx.completion_verification_result = _evaluated(("c0", True))
assert outcome_fully_verified(ctx) is True
@@ -834,9 +825,6 @@ def test_outcome_fully_verified_predicate(monkeypatch: pytest.MonkeyPatch) -> No
assert outcome_fully_verified(ctx) is False
ctx.completion_verification_result = None
assert outcome_fully_verified(ctx) is False
- ctx.completion_verification_result = _evaluated(("c0", True))
- monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", False)
- assert outcome_fully_verified(ctx) is False
def test_gate_recognizes_canceled_run_on_full_evidence() -> None:
@@ -1272,7 +1260,6 @@ async def test_completion_verification_receives_verified_context_labels(monkeypa
"skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
- monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", True)
ctx = _run_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
@@ -1364,27 +1351,3 @@ def test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None
# An unavailable verdict means the outcome could not be verified: do not surface
# the workflow as verified on run status alone.
assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]
-
-
-def test_request_policy_prompt_gates_completion_criteria_on_flag() -> None:
- from skyvern.forge.prompts import prompt_engine
-
- common = dict(
- user_message="",
- raw_secret_present="false",
- workflow_yaml="",
- earliest_user_turn="",
- latest_prior_user_turn="",
- latest_assistant_turn="",
- retained_history="",
- global_llm_context="",
- )
- on = prompt_engine.load_prompt(
- template="workflow-copilot-request-policy", outcome_verification_enabled=True, **common
- )
- off = prompt_engine.load_prompt(
- template="workflow-copilot-request-policy", outcome_verification_enabled=False, **common
- )
- assert "completion_criteria" in on
- # Flag off must not perturb the classifier prompt with the criteria contract.
- assert "completion_criteria" not in off
diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py
index 7081c76c0..aa37461b9 100644
--- a/tests/unit/test_copilot_make_agent_result_backfill.py
+++ b/tests/unit/test_copilot_make_agent_result_backfill.py
@@ -99,15 +99,6 @@ def test_backfill_verified_success_requires_adjudicated_evidence() -> None:
assert result.narrative_payload["verifiedSuccess"] is False
-def test_backfill_verified_success_true_when_legacy_gate_passes_with_flag_off(monkeypatch) -> None:
- from skyvern.config import settings
-
- monkeypatch.setattr(settings, "COPILOT_PERSISTED_COMPLETION_CRITERIA_ENABLED", False)
- result = _result(_verified_goal_ctx(), turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=_payload())
- assert result.narrative_payload is not None
- assert result.narrative_payload["verifiedSuccess"] is True
-
-
def test_backfill_verified_success_true_when_outcome_fully_verified() -> None:
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,