Skip to content

Automate browser based workflows with AI

SKY-11464 Stabilize Copilot completion criteria outputs (#6814)

Andrew Neilson80c40f0bdf78
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 35b9e2fc6..aeb2c31e4 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -404,10 +404,14 @@ def _reconcile_completion_criteria_on_context(
snapshot = policy_inputs.stored_completion_criteria
if snapshot is None:
return
+ requested_output_path_aliases = (
+ ctx.copilot_config.requested_output_path_aliases if ctx.copilot_config is not None else None
+ )
decision = reconcile_completion_criteria(
snapshot,
list(policy.completion_criteria),
actionable=policy.user_response_policy != "ask_clarification",
+ requested_output_path_aliases=requested_output_path_aliases,
)
if decision.action == "adopt_stored":
policy.completion_criteria = list(decision.criteria)
@@ -3105,6 +3109,7 @@ def _build_copilot_input_guardrails(
organization_id=policy_inputs.organization_id,
handler=policy_inputs.handler,
active_criteria=_stored_active_completion_criteria(policy_inputs),
+ config=getattr(ctx, "copilot_config", None) if isinstance(ctx, CopilotContext) else None,
)
if isinstance(ctx, CopilotContext):
turn_intent_classifier_result = None
@@ -3551,6 +3556,7 @@ async def _run_copilot_turn_impl(
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
block_authoring_policy=copilot_config.block_authoring_policy,
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
+ copilot_config=copilot_config,
target_block_label=getattr(chat_request, "target_block_label", None),
)
# Fail loud if a future caller skips the kwarg and gets a fresh UUID from
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index f677522f7..1a987da0d 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
is_fallback_floor_criterion,
normalized_criterion_outcome_key,
+ requested_output_path_for_field,
)
ReconcileAction = Literal["create", "adopt_stored", "none"]
@@ -112,6 +113,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"implicit": criterion.implicit,
"method_mandated": criterion.method_mandated,
"level": criterion.level,
+ "output_path": criterion.output_path,
}
for criterion in criteria
]
@@ -129,6 +131,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
if not isinstance(criterion_id, str) or not isinstance(outcome, str) or not outcome.strip():
continue
level = item.get("level")
+ output_path = item.get("output_path")
criteria.append(
CompletionCriterion(
id=criterion_id,
@@ -136,13 +139,120 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
implicit=bool(item.get("implicit")),
method_mandated=bool(item.get("method_mandated")),
level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]
+ output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,
)
)
return tuple(criteria)
+def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
+ if criterion.output_path:
+ return f"output_path:{criterion.output_path}"
+ return f"outcome:{normalized_criterion_outcome_key(criterion.outcome)}"
+
+
def _outcome_key_set(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:
- return {normalized_criterion_outcome_key(criterion.outcome) for criterion in criteria}
+ return {_criterion_reconcile_key(criterion) for criterion in criteria}
+
+
+def _word_tokens(text: str) -> list[str]:
+ return "".join(char if char.isalnum() else " " for char in text.casefold()).split()
+
+
+_REQUESTED_OUTPUT_WORDS = frozenset(
+ "capture captured extract extracted final include included includes output read record result return returned".split()
+)
+_REQUESTED_OUTPUT_FIELD_TOKENS = frozenset(
+ "address addresses date dates email emails id identifier identifiers license licenses location locations name names "
+ "npi number numbers phone phones result specialties specialty status statuses taxonomy".split()
+)
+_GENERIC_PROFILE_MARKERS = (
+ "profile details",
+ "profile information",
+ "profile is captured",
+ "profile is extracted",
+ "intended end state",
+ "expected output",
+)
+
+
+def _requested_output_tokens(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:
+ tokens: set[str] = set()
+ for criterion in criteria:
+ if criterion.level == "definition" or criterion.method_mandated:
+ continue
+ outcome_tokens = _word_tokens(criterion.outcome)
+ if not any(word in _REQUESTED_OUTPUT_WORDS for word in outcome_tokens):
+ continue
+ for token in outcome_tokens:
+ if token in _REQUESTED_OUTPUT_FIELD_TOKENS:
+ tokens.add(token)
+ return tokens
+
+
+def _requested_output_paths(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:
+ return {
+ criterion.output_path
+ for criterion in criteria
+ if criterion.output_path and criterion.level != "definition" and not criterion.method_mandated
+ }
+
+
+def _criterion_mentions_output_path(
+ criterion: CompletionCriterion,
+ output_path: str,
+ aliases: dict[str, str] | None = None,
+) -> bool:
+ outcome_tokens = _word_tokens(criterion.outcome)
+ max_span_len = min(4, len(outcome_tokens))
+ for span_len in range(max_span_len, 0, -1):
+ for start in range(len(outcome_tokens) - span_len + 1):
+ field_name = " ".join(outcome_tokens[start : start + span_len])
+ if requested_output_path_for_field(field_name, aliases) == output_path:
+ return True
+ return False
+
+
+def _is_generic_profile_criterion(criterion: CompletionCriterion) -> bool:
+ key = normalized_criterion_outcome_key(criterion.outcome)
+ return is_fallback_floor_criterion(criterion) or any(marker in key for marker in _GENERIC_PROFILE_MARKERS)
+
+
+def _fresh_generic_rephrase_lacks_stored_requested_outputs(
+ stored: tuple[CompletionCriterion, ...],
+ fresh: list[CompletionCriterion],
+ *,
+ requested_output_path_aliases: dict[str, str] | None = None,
+) -> bool:
+ stored_requested_criteria = tuple(
+ criterion
+ for criterion in stored
+ if criterion.output_path and criterion.level != "definition" and not criterion.method_mandated
+ )
+ stored_requested_paths = {
+ output_path for criterion in stored_requested_criteria if (output_path := criterion.output_path) is not None
+ }
+ if stored_requested_paths:
+ fresh_requested_paths = _requested_output_paths(fresh)
+ missing_paths = stored_requested_paths - fresh_requested_paths
+ if not missing_paths:
+ return False
+ if fresh and all(_is_generic_profile_criterion(criterion) for criterion in fresh):
+ return True
+ return any(
+ criterion.output_path is None
+ and _criterion_mentions_output_path(criterion, output_path, requested_output_path_aliases)
+ for criterion in fresh
+ for output_path in missing_paths
+ )
+
+ stored_requested_tokens = _requested_output_tokens(stored)
+ if not stored_requested_tokens:
+ return False
+ if not fresh or not all(_is_generic_profile_criterion(criterion) for criterion in fresh):
+ return False
+ fresh_tokens = set().union(*(_word_tokens(criterion.outcome) for criterion in fresh))
+ return bool(stored_requested_tokens - fresh_tokens)
def reconcile_completion_criteria(
@@ -150,6 +260,7 @@ def reconcile_completion_criteria(
fresh: list[CompletionCriterion],
*,
actionable: bool,
+ requested_output_path_aliases: dict[str, str] | None = None,
) -> ReconcileDecision:
"""Decide once per turn whether the stored set survives or a new epoch begins.
@@ -176,6 +287,14 @@ def reconcile_completion_criteria(
return ReconcileDecision(
action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria
)
+ if _fresh_generic_rephrase_lacks_stored_requested_outputs(
+ stored.criteria,
+ fresh,
+ requested_output_path_aliases=requested_output_path_aliases,
+ ):
+ return ReconcileDecision(
+ action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria
+ )
if not actionable:
return ReconcileDecision(
action="adopt_stored", reason="not_actionable", epoch=stored.goal_epoch, criteria=stored.criteria
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index ebd8191be..6ff5f1109 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -252,7 +252,8 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:
flags += " (implicit)"
if criterion.method_mandated:
flags += " (method_mandated)"
- parts.append(f"- {criterion.id}: {criterion.outcome}{flags}")
+ output_path = f" [required_output_path={criterion.output_path}]" if criterion.output_path else ""
+ parts.append(f"- {criterion.id}: {criterion.outcome}{output_path}{flags}")
return "\n".join(parts)
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index 4398a6617..61c385fef 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -366,6 +366,7 @@ class CopilotConfig:
fallback_llm_key: str | None = field(default_factory=_default_fallback_llm_key)
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
impose_synthesized_code_block: bool = False
+ requested_output_path_aliases: dict[str, str] = field(default_factory=dict)
def nudge(self, key: str) -> str:
return self.enforcement_nudges.get(key, DEFAULT_ENFORCEMENT_NUDGES[key])
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 9b600d355..9f18f2fa2 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -11,7 +11,7 @@ from pydantic import BaseModel, Field
from typing_extensions import NotRequired, TypedDict
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.workflow.models.workflow import Workflow
@@ -95,6 +95,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+ from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
@@ -386,6 +387,7 @@ class CopilotContext(AgentContext):
block_goal_main_goal: str = ""
allow_untested_workflow_draft: bool = False
request_policy: RequestPolicy | None = None
+ copilot_config: CopilotConfig | None = None
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
impose_synthesized_code_block: bool = False
target_block_label: str | None = None
@@ -406,6 +408,7 @@ class CopilotContext(AgentContext):
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
tool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)
turn_halt: TurnHalt | None = None
+ latest_schema_incompatibility: SchemaIncompatibility | None = None
# ``None`` until usage is observed; ``0`` only when a provider explicitly
# reported zero. Distinct values let cost grading flag missing telemetry.
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index ffa79d7bf..6bf184051 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.output_policy import url_origin
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name
+from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_FAILURE_TYPE
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
@@ -50,6 +51,7 @@ class DiagnosisFailureType(StrEnum):
REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"
ACTIVE_RUN_TERMINAL_EVIDENCE = "active_run_terminal_evidence"
UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"
+ SCHEMA_INCOMPATIBILITY = "schema_incompatibility"
UNKNOWN = "unknown"
@@ -235,6 +237,11 @@ def build_diagnosis_repair_contract(
"Stop the current retry loop: the active run reached the requested browser state, "
"but the reusable workflow is not verified end-to-end."
)
+ elif failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:
+ decision_summary = (
+ "Stop re-authoring: the edited extraction schema declares fields that map to no output the "
+ "workflow produces, so the mismatch is not repairable without user input."
+ )
completion_check = {
RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",
RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",
@@ -482,6 +489,8 @@ def _failure_type(
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
if _safe_str(data.get("failure_type")) == "missing_credential_or_init":
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ if _safe_str(data.get("failure_type")) == SCHEMA_INCOMPATIBILITY_FAILURE_TYPE:
+ return DiagnosisFailureType.SCHEMA_INCOMPATIBILITY
if repair_context is not None:
return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
if outcome_verified:
@@ -511,6 +520,8 @@ def _next_action(
return RepairNextAction.STOP
if failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:
return RepairNextAction.STOP
+ if failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:
+ return RepairNextAction.STOP
if ctx.last_test_non_retriable_nav_error:
return RepairNextAction.STOP
if _last_test_anti_bot_is_terminal(ctx, data) and failure_type in {
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 9b1703475..bbc848627 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -13,6 +13,7 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import StructuredContext
from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
@@ -144,6 +145,7 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "
_MAX_COMPLETION_CRITERIA = 8
_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200
+_REQUESTED_OUTPUT_CRITERION_ID_PREFIX = "__copilot_requested_output__"
FALLBACK_FLOOR_CRITERION_ID_PREFIX = "__copilot_fallback_floor__"
_FALLBACK_FLOOR_BASE_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}run"
@@ -154,6 +156,40 @@ _FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and re
CriterionLevel = Literal["definition", "run"]
_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})
+_OUTPUT_INTENT_RE = re.compile(
+ r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"
+ r"final\s+(?:extracted\s+)?fields?|result\s+records?|returned\s+records?)\b",
+ re.I,
+)
+_OUTPUT_SPAN_END_RE = re.compile(r"[\n.!?]")
+_OUTPUT_METHOD_TAIL_RE = re.compile(
+ r"\b(?:by|via|using|after|before|then|click(?:ing)?|open(?:ing)?|select(?:ing)?|"
+ r"choose|choosing|search(?:ing)?|navigate|go\s+to)\b.*",
+ re.I,
+)
+_OUTPUT_SPLIT_RE = re.compile(r",|;|\band\b|\bplus\b|&", re.I)
+_OUTPUT_FIELD_CONNECTOR_RE = re.compile(
+ r"\b(?:with|including|include|includes|containing|contains|fields?|result)\b[:\s]+",
+ re.I,
+)
+_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")
+_OUTPUT_FIELD_WORDS = frozenset(
+ "address addresses date dates email emails id identifier identifiers license licenses location locations "
+ "name names number numbers phone phones specialties specialty status statuses taxonomy".split()
+)
+_OUTPUT_GENERIC_WORDS = frozenset(
+ "a all an data detail details each entity final for information its of output outputs profile record records "
+ "result results structured the value values".split()
+)
+_OUTPUT_METHOD_WORDS = frozenset("choose click open plan search select setup show".split())
+_OUTPUT_LEADING_FIELD_WORDS = frozenset(
+ "capture captured extract extracted include included includes output read return returned".split()
+)
+_OUTPUT_INPUT_ONLY_WORDS = frozenset({"input", "inputs", "parameter", "parameters", "reusable"})
+_OUTPUT_OUTCOME_WORDS = frozenset(
+ "capture captured extract extracted final include included includes output read record result return returned".split()
+)
+
@dataclass(frozen=True)
class CompletionCriterion:
@@ -164,6 +200,7 @@ class CompletionCriterion:
# "definition": a property of the workflow definition itself, graded against the
# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".
level: CriterionLevel = "run"
+ output_path: str | None = None
@dataclass
@@ -528,7 +565,9 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
outcome = " ".join(outcome_raw.split())[:_COMPLETION_CRITERION_OUTCOME_MAX_CHARS].strip()
if not outcome:
continue
- key = normalized_criterion_outcome_key(outcome)
+ output_path_raw = item.get("output_path")
+ output_path = output_path_raw.strip() if isinstance(output_path_raw, str) and output_path_raw.strip() else None
+ key = output_path or normalized_criterion_outcome_key(outcome)
if key in seen:
continue
seen.add(key)
@@ -542,6 +581,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
level=cast(CriterionLevel, level_raw)
if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELS
else "run",
+ output_path=output_path,
)
)
if len(criteria) >= _MAX_COMPLETION_CRITERIA:
@@ -554,20 +594,321 @@ def normalized_criterion_outcome_key(outcome: str) -> str:
return collapsed.strip().lower().rstrip(".!?;:,").strip()
+def _output_intent_spans(user_message: str) -> list[str]:
+ message = user_message or ""
+ spans: list[str] = []
+ for match in _OUTPUT_INTENT_RE.finditer(message):
+ tail = message[match.start() :]
+ end_match = _OUTPUT_SPAN_END_RE.search(tail)
+ span = tail[: end_match.start()] if end_match is not None else tail
+ span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")
+ if span:
+ spans.append(span)
+ return spans
+
+
+def _normalize_requested_output_aliases(aliases: dict[str, str] | None) -> dict[str, str]:
+ normalized: dict[str, str] = {}
+ for field_name, output_path in (aliases or {}).items():
+ if not isinstance(field_name, str) or not isinstance(output_path, str):
+ continue
+ path = output_path.strip()
+ if not path.startswith("output."):
+ continue
+ key = " ".join(_word_tokens(field_name))
+ if key:
+ normalized[key] = path
+ return normalized
+
+
+def _lookup_requested_output_path_alias(field_name: str, aliases: dict[str, str] | None) -> str | None:
+ normalized_aliases = _normalize_requested_output_aliases(aliases)
+ field_key = " ".join(_word_tokens(field_name))
+ if not field_key:
+ return None
+ if field_key in normalized_aliases:
+ return normalized_aliases[field_key]
+ field_words = field_key.split()
+ for alias_key, output_path in normalized_aliases.items():
+ alias_words = alias_key.split()
+ if _tokens_contain_sequence(field_words, alias_words) or _tokens_contain_sequence(alias_words, field_words):
+ return output_path
+ return None
+
+
+def _schema_output_path_aliases_from_criteria(criteria: list[CompletionCriterion]) -> dict[str, str]:
+ aliases: dict[str, str] = {}
+ for criterion in criteria:
+ if criterion.level == "definition" or criterion.method_mandated or not criterion.output_path:
+ continue
+ path_tokens = _word_tokens(criterion.output_path.removeprefix("output."))
+ keep_generic_words = (
+ {"record"} if any(token in {"id", "identifier", "identifiers"} for token in path_tokens) else set()
+ )
+ path_phrase = " ".join(
+ token for token in path_tokens if token not in _OUTPUT_GENERIC_WORDS or token in keep_generic_words
+ )
+ if path_phrase:
+ aliases.setdefault(path_phrase, criterion.output_path)
+ for token in path_tokens:
+ if token and token not in _OUTPUT_GENERIC_WORDS:
+ aliases.setdefault(token, criterion.output_path)
+ return aliases
+
+
+def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | None = None) -> str | None:
+ candidate = _OUTPUT_METHOD_TAIL_RE.sub("", segment).strip(" :-")
+ if not candidate:
+ return None
+
+ connector_matches = list(_OUTPUT_FIELD_CONNECTOR_RE.finditer(candidate))
+ if connector_matches:
+ candidate = candidate[connector_matches[-1].end() :]
+ candidate = re.sub(r"\b([A-Za-z0-9]+)'s\b", r"\1", candidate)
+ candidate = " ".join(re.sub(r"[^A-Za-z0-9 _/-]+", " ", candidate).split()).strip(" :-")
+ if not candidate:
+ return None
+ normalized_aliases = _normalize_requested_output_aliases(aliases)
+ if " ".join(_word_tokens(candidate)) in normalized_aliases:
+ return candidate
+ candidate_words = candidate.split()
+ for word in candidate_words:
+ if " ".join(_word_tokens(word)) in normalized_aliases:
+ return word
+ if _lookup_requested_output_path_alias(candidate, aliases) is not None:
+ return candidate
+
+ acronyms = [
+ token
+ for token in _OUTPUT_ACRONYM_RE.findall(candidate)
+ if token.casefold() not in _OUTPUT_METHOD_WORDS
+ and token.casefold() not in _OUTPUT_GENERIC_WORDS
+ and _lookup_requested_output_path_alias(token, aliases) is not None
+ ]
+ if acronyms:
+ return acronyms[0]
+
+ words = candidate.split()
+ normalized_words = [word.casefold().strip("_-/") for word in words if word.strip("_-/")]
+ if not normalized_words:
+ return None
+ while normalized_words and normalized_words[0] in _OUTPUT_LEADING_FIELD_WORDS:
+ words = words[1:]
+ normalized_words = normalized_words[1:]
+ if not normalized_words:
+ return None
+ if any(word in _OUTPUT_METHOD_WORDS for word in normalized_words):
+ return None
+ if all(word in _OUTPUT_GENERIC_WORDS for word in normalized_words):
+ return None
+ field_indexes = [i for i, word in enumerate(normalized_words) if word in _OUTPUT_FIELD_WORDS]
+ if not field_indexes:
+ return None
+ status_indexes = [i for i, word in enumerate(normalized_words) if word in {"status", "statuses"}]
+ if status_indexes:
+ field_indexes = status_indexes
+ last_field_index = field_indexes[-1]
+ start = max(0, last_field_index - 2)
+ keep_generic_words = (
+ {"record"} if normalized_words[last_field_index] in {"id", "identifier", "identifiers"} else set()
+ )
+ phrase_words = [
+ word
+ for word in words[start : last_field_index + 1]
+ if word.casefold().strip("_-/") not in _OUTPUT_GENERIC_WORDS
+ or word.casefold().strip("_-/") in keep_generic_words
+ ]
+ if not phrase_words:
+ return None
+ return " ".join(phrase_words)
+
+
+def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:
+ fields: list[str] = []
+ seen: set[str] = set()
+ for span in _output_intent_spans(user_message):
+ for segment in _OUTPUT_SPLIT_RE.split(span):
+ field_name = _clean_requested_output_candidate(segment, aliases)
+ if field_name is None:
+ continue
+ key = field_name.casefold()
+ if key in seen:
+ continue
+ seen.add(key)
+ fields.append(field_name)
+ return fields
+
+
+def requested_output_path_for_field(field_name: str, aliases: dict[str, str] | None = None) -> str:
+ alias_path = _lookup_requested_output_path_alias(field_name, aliases)
+ if alias_path is not None:
+ return alias_path
+ words = _word_tokens(field_name)
+ slug = "_".join(words) or "field"
+ return f"output.{slug}"
+
+
+def _requested_output_field_label(field_name: str, output_path: str) -> str:
+ stripped = field_name.strip()
+ if stripped.isupper():
+ return stripped
+ return " ".join(_word_tokens(field_name))
+
+
+def _tokens_contain_sequence(haystack: list[str], needle: list[str]) -> bool:
+ if not needle or len(needle) > len(haystack):
+ return False
+ for index in range(len(haystack) - len(needle) + 1):
+ if haystack[index : index + len(needle)] == needle:
+ return True
+ return False
+
+
+def _criterion_text_covers_requested_output(criterion: CompletionCriterion, field_name: str) -> bool:
+ if criterion.level == "definition" or criterion.method_mandated:
+ return False
+ outcome_words = _word_tokens(criterion.outcome)
+ field_words = _word_tokens(field_name)
+ if not _tokens_contain_sequence(outcome_words, field_words):
+ return False
+ if any(word in _OUTPUT_INPUT_ONLY_WORDS for word in outcome_words) and not any(
+ word in _OUTPUT_OUTCOME_WORDS for word in outcome_words
+ ):
+ return False
+ return True
+
+
+def _criterion_covers_requested_output(
+ criterion: CompletionCriterion, field_name: str, aliases: dict[str, str] | None = None
+) -> bool:
+ if criterion.level == "definition" or criterion.method_mandated:
+ return False
+ output_path = requested_output_path_for_field(field_name, aliases)
+ return criterion.output_path == output_path or _criterion_text_covers_requested_output(criterion, field_name)
+
+
+def _criterion_text_covers_any_requested_output(criterion: CompletionCriterion, requested_fields: list[str]) -> bool:
+ return any(_criterion_text_covers_requested_output(criterion, field_name) for field_name in requested_fields)
+
+
+def _requested_output_criterion_id(output_path: str) -> str:
+ slug = "_".join(_word_tokens(output_path)) or "field"
+ return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"
+
+
+def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:
+ key = normalized_criterion_outcome_key(criterion.outcome)
+ if is_fallback_floor_criterion(criterion):
+ return True
+ generic_markers = (
+ "intended end state",
+ "expected output",
+ "profile details",
+ "profile information",
+ "requested identifier-like value",
+ "target entity",
+ "grouped row entries",
+ )
+ return any(marker in key for marker in generic_markers)
+
+
+def _criterion_drop_priority(criterion: CompletionCriterion, requested_output_paths: set[str]) -> int:
+ if criterion.output_path in requested_output_paths:
+ return 0
+ if criterion.level == "definition":
+ return 0
+ if is_fallback_floor_criterion(criterion):
+ return 4
+ if criterion.method_mandated:
+ return 0
+ if _generic_completion_criterion(criterion):
+ return 3
+ return 2
+
+
+def _cap_completion_criteria(
+ criteria: list[CompletionCriterion], requested_output_paths: set[str]
+) -> list[CompletionCriterion]:
+ capped = list(criteria)
+ while len(capped) > _MAX_COMPLETION_CRITERIA:
+ drop_index = max(
+ range(len(capped)),
+ key=lambda index: (_criterion_drop_priority(capped[index], requested_output_paths), index),
+ )
+ if _criterion_drop_priority(capped[drop_index], requested_output_paths) == 0:
+ break
+ del capped[drop_index]
+ return capped
+
+
+def _apply_requested_output_completion_criteria(
+ policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
+) -> None:
+ schema_aliases = _schema_output_path_aliases_from_criteria(policy.completion_criteria)
+ config_aliases = _normalize_requested_output_aliases(aliases)
+ schema_aliases = _normalize_requested_output_aliases(schema_aliases)
+ detection_aliases = {**config_aliases, **schema_aliases}
+ requested_fields = _requested_output_fields(user_message, detection_aliases)
+ if not requested_fields:
+ return
+
+ requested_specs: list[tuple[str, str, str]] = []
+ requested_output_paths: set[str] = set()
+ for field_name in requested_fields:
+ output_path = (
+ _lookup_requested_output_path_alias(field_name, schema_aliases)
+ or _lookup_requested_output_path_alias(field_name, config_aliases)
+ or requested_output_path_for_field(field_name)
+ )
+ field_label = _requested_output_field_label(field_name, output_path)
+ if output_path in requested_output_paths:
+ continue
+ requested_output_paths.add(output_path)
+ requested_specs.append((field_name, output_path, field_label))
+
+ if not requested_output_paths:
+ return
+
+ preserved_criteria = [
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.level == "definition"
+ or criterion.method_mandated
+ or (
+ criterion.output_path not in requested_output_paths
+ and not _criterion_text_covers_any_requested_output(criterion, requested_fields)
+ )
+ ]
+ canonical_requested_criteria = [
+ CompletionCriterion(
+ id=_requested_output_criterion_id(output_path),
+ outcome=f"The returned record includes {field_label}.",
+ level="run",
+ output_path=output_path,
+ )
+ for _field_name, output_path, field_label in requested_specs
+ ]
+ policy.completion_criteria = _cap_completion_criteria(
+ preserved_criteria + canonical_requested_criteria,
+ requested_output_paths,
+ )
+
+
def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | None) -> str:
if not criteria:
return ""
- return json.dumps(
- [
- {
- "outcome": criterion.outcome,
- "implicit": criterion.implicit,
- "method_mandated": criterion.method_mandated,
- "level": criterion.level,
- }
- for criterion in criteria
- ]
- )
+ rendered: list[dict[str, Any]] = []
+ for criterion in criteria:
+ item: dict[str, Any] = {
+ "outcome": criterion.outcome,
+ "implicit": criterion.implicit,
+ "method_mandated": criterion.method_mandated,
+ "level": criterion.level,
+ }
+ if criterion.output_path:
+ item["output_path"] = criterion.output_path
+ rendered.append(item)
+ return json.dumps(rendered)
def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:
@@ -604,6 +945,7 @@ def _classifier_fallback_policy(
failure_kind: str,
retry_count: int = 0,
user_message: str = "",
+ requested_output_path_aliases: dict[str, str] | None = None,
) -> RequestPolicy:
if failure_kind not in _CLASSIFIER_FAILURE_KINDS:
failure_kind = "provider_error"
@@ -626,7 +968,7 @@ def _classifier_fallback_policy(
classifier_failure_kind=failure_kind,
completion_criterion_ids=[criterion.id for criterion in fallback_criteria],
)
- return RequestPolicy(
+ policy = RequestPolicy(
credential_input_kind="credential_id" if ids else "none",
credential_refs=ids,
completion_criteria=fallback_criteria or build_classifier_fallback_floor(ids),
@@ -635,6 +977,10 @@ def _classifier_fallback_policy(
classifier_retry_count=retry_count,
completion_contract_status="present" if fallback_criteria else "unknown",
)
+ _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
+ if policy.graded_completion_criteria():
+ policy.completion_contract_status = "present"
+ return policy
def _word_tokens(text: str) -> list[str]:
@@ -754,7 +1100,9 @@ async def _classify_request(
handler: Any,
*,
active_criteria: list[CompletionCriterion] | None = None,
+ config: CopilotConfig | None = None,
) -> RequestPolicy:
+ requested_output_path_aliases = config.requested_output_path_aliases if config is not None else {}
ids = _credential_ids(user_message)
raw_secret_present = _raw_secret_detected(user_message)
if raw_secret_present and handler is None:
@@ -763,6 +1111,7 @@ async def _classify_request(
raw_secret_present=True,
failure_kind="raw_secret_no_handler",
user_message=user_message,
+ requested_output_path_aliases=requested_output_path_aliases,
)
structural_reason = _structural_clarification_reason(user_message)
if structural_reason != "none" and not raw_secret_present:
@@ -778,6 +1127,7 @@ async def _classify_request(
raw_secret_present=False,
failure_kind="missing_handler",
user_message=user_message,
+ requested_output_path_aliases=requested_output_path_aliases,
)
# Raw-secret turns intentionally reach the classifier when available so it
@@ -806,6 +1156,7 @@ async def _classify_request(
failure_kind=failure_kind,
retry_count=retry_count,
user_message=user_message,
+ requested_output_path_aliases=requested_output_path_aliases,
)
raw_payload = _coerce_classifier_payload(raw)
@@ -817,6 +1168,7 @@ async def _classify_request(
failure_kind="provider_error",
retry_count=retry_count,
user_message=user_message,
+ requested_output_path_aliases=requested_output_path_aliases,
)
policy = _classification_from_raw(raw_payload)
@@ -865,6 +1217,10 @@ async def _classify_request(
policy.clarification_reason = "none"
policy.requires_user_clarification = False
policy.raw_secret_evidence = None
+ _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
+ policy.completion_contract_status = (
+ "present" if policy.completion_contract or policy.graded_completion_criteria() else "absent"
+ )
return policy
@@ -1335,6 +1691,7 @@ async def build_request_policy(
organization_id: str,
handler: Any,
active_criteria: list[CompletionCriterion] | None = None,
+ config: CopilotConfig | None = None,
) -> RequestPolicy:
policy = await _classify_request(
user_message,
@@ -1343,6 +1700,7 @@ async def build_request_policy(
global_llm_context,
handler,
active_criteria=active_criteria,
+ config=config,
)
policy.raw_secret_detected = policy.raw_secret_detected or policy.credential_input_kind == "raw_secret"
policy.existing_workflow_credential_ids = sorted(workflow_credential_ids(workflow_yaml))
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index b2569f8f5..799450bc5 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+ from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
@@ -308,6 +309,10 @@ class AgentContext:
# render the current tool result from structured product text.
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
tool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)
+ # Latest edited-schema-incompatibility terminal outcome, set when an edited
+ # extraction_schema declares fields that map to no output the block produces.
+ # Surfaced into the persisted TurnOutcome so a later turn can report it.
+ latest_schema_incompatibility: SchemaIncompatibility | None = None
def mcp_to_copilot(mcp_result: dict[str, Any]) -> dict[str, Any]:
diff --git a/skyvern/forge/sdk/copilot/schema_incompatibility.py b/skyvern/forge/sdk/copilot/schema_incompatibility.py
new file mode 100644
index 000000000..aaa8143ba
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/schema_incompatibility.py
@@ -0,0 +1,145 @@
+"""Typed terminal outcome for an edited extraction schema whose fields map to no
+output the workflow produces.
+
+When a user edits a code block's confirmed ``extraction_schema`` to add fields that
+overlap none of the block's known output contract (its top-level return keys plus
+confirmed ``goal_value_paths``), re-authoring cannot reconcile the mismatch: there is
+nothing on the page or in the return for the new field to bind to. Surfacing it as a
+typed, non-repairable schema-incompatibility outcome — rather than letting the agent
+churn until the repair ceiling — preserves the existing draft and tells the user
+exactly which fields do not map.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ CopilotToolBlockerSignal,
+ assert_clean_user_facing_text,
+)
+
+SCHEMA_INCOMPATIBILITY_REASON_CODE = "schema_incompatibility"
+SCHEMA_INCOMPATIBILITY_FAILURE_TYPE = "schema_incompatibility"
+SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL = "update_and_run_blocks"
+
+_DEFAULT_NEXT_ACTIONS: tuple[str, ...] = (
+ "Map the field to a value this workflow already produces.",
+ "Remove the field from the extraction schema.",
+ "Describe what the field should capture so a step can be added to produce it.",
+)
+
+_GENERIC_USER_REASON = (
+ "I couldn't apply the edited extraction schema because some of its fields don't match "
+ "any value this workflow currently produces. Tell me what they should map to and I'll try again. "
+ "Your current draft is unchanged."
+)
+
+
+class SchemaIncompatibility(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ block_label: str
+ incompatible_paths: tuple[str, ...]
+ known_output_paths: tuple[str, ...]
+ edited_schema_summary: str = ""
+ preserves_workflow_draft: bool = True
+ next_actions: tuple[str, ...] = Field(default=_DEFAULT_NEXT_ACTIONS)
+
+ def to_summary_dict(self) -> dict[str, Any]:
+ return {
+ "block_label": self.block_label,
+ "incompatible_paths": list(self.incompatible_paths),
+ "known_output_paths": list(self.known_output_paths),
+ "edited_schema_summary": self.edited_schema_summary,
+ "preserves_workflow_draft": self.preserves_workflow_draft,
+ "next_actions": list(self.next_actions),
+ }
+
+
+def merge_schema_incompatibilities(items: list[SchemaIncompatibility]) -> SchemaIncompatibility | None:
+ """Fold per-block incompatibilities into a single terminal record. The merged
+ paths are de-duplicated and ordered; the first block label anchors the record."""
+ real = [item for item in items if item is not None]
+ if not real:
+ return None
+ if len(real) == 1:
+ return real[0]
+ incompatible: list[str] = []
+ known: list[str] = []
+ summaries: list[str] = []
+ for item in real:
+ for path in item.incompatible_paths:
+ if path not in incompatible:
+ incompatible.append(path)
+ for path in item.known_output_paths:
+ if path not in known:
+ known.append(path)
+ if item.edited_schema_summary and item.edited_schema_summary not in summaries:
+ summaries.append(item.edited_schema_summary)
+ return SchemaIncompatibility(
+ block_label=real[0].block_label,
+ incompatible_paths=tuple(incompatible),
+ known_output_paths=tuple(known),
+ edited_schema_summary="; ".join(summaries),
+ preserves_workflow_draft=all(item.preserves_workflow_draft for item in real),
+ )
+
+
+def _field_list_phrase(paths: tuple[str, ...]) -> str:
+ quoted = [f"`{path}`" for path in paths]
+ if len(quoted) == 1:
+ return quoted[0]
+ if len(quoted) == 2:
+ return f"{quoted[0]} and {quoted[1]}"
+ return ", ".join(quoted[:-1]) + f", and {quoted[-1]}"
+
+
+def render_schema_incompatibility_user_reason(incompat: SchemaIncompatibility) -> str:
+ """Product-language reply rendered from the structured incompatibility. Falls back
+ to a field-free message if an exotic field name trips the user-facing safety gate."""
+ fields = _field_list_phrase(incompat.incompatible_paths)
+ verb = "doesn't" if len(incompat.incompatible_paths) == 1 else "don't"
+ sentences = [
+ f"I couldn't apply the edited extraction schema: the field {fields} {verb} match any value this workflow currently produces."
+ ]
+ if incompat.known_output_paths:
+ outputs = ", ".join(incompat.known_output_paths)
+ sentences.append(f"This workflow's data currently covers {outputs}.")
+ sentences.append(
+ "Tell me which existing output it should map to, or remove it, and I'll try again. Your current draft is unchanged."
+ )
+ candidate = " ".join(sentences)
+ try:
+ assert_clean_user_facing_text(candidate, blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL)
+ except ValueError:
+ return _GENERIC_USER_REASON
+ return candidate
+
+
+def render_schema_incompatibility_agent_steer(incompat: SchemaIncompatibility) -> str:
+ incompatible = ", ".join(incompat.incompatible_paths) or "(unknown)"
+ known = ", ".join(incompat.known_output_paths) or "(none recorded)"
+ return (
+ f"STOP: the edited extraction_schema declares field(s) [{incompatible}] that map to no output block "
+ f"`{incompat.block_label}` produces [{known}]. This is not repairable by re-authoring the same draft. "
+ "Report the mismatch to the user and ask which existing output the field should map to. "
+ "The prior draft is preserved; do not rerun the blocks."
+ )
+
+
+def build_schema_incompatibility_blocker_signal(incompat: SchemaIncompatibility) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=render_schema_incompatibility_agent_steer(incompat),
+ user_facing_reason=render_schema_incompatibility_user_reason(incompat),
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=incompat.preserves_workflow_draft,
+ renders_final_reply=True,
+ internal_reason_code=SCHEMA_INCOMPATIBILITY_REASON_CODE,
+ blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL,
+ extra={"schema_incompatibility": incompat.to_summary_dict()},
+ )
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 47dbbc7aa..2ef497508 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -7,7 +7,7 @@ import keyword
import re
import textwrap
import tokenize
-from collections.abc import Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Annotated, Any, Literal, NamedTuple
@@ -78,9 +78,19 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
code_is_download_intent,
)
from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteraction
+from skyvern.forge.sdk.copilot.schema_incompatibility import (
+ SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,
+ SchemaIncompatibility,
+ build_schema_incompatibility_blocker_signal,
+ merge_schema_incompatibilities,
+ render_schema_incompatibility_agent_steer,
+)
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.copilot.turn_halt import blocker_signal_is_genuinely_terminal
+from skyvern.forge.sdk.copilot.turn_halt import (
+ blocker_signal_is_genuinely_terminal,
+ stash_turn_halt_from_blocker_signal,
+)
from skyvern.forge.sdk.copilot.workflow_credential_utils import credential_params, parse_workflow_yaml, workflow_blocks
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected
@@ -360,6 +370,7 @@ class CodeArtifactNormalization(NamedTuple):
error: str | None
violations: list[str]
offending_labels: list[str]
+ schema_incompatibilities: list[SchemaIncompatibility] = []
def _normalize_code_artifact_metadata(
@@ -399,6 +410,7 @@ def _normalize_code_artifact_metadata_detailed(
trajectory: list[ScoutedInteraction] = scout_trajectory or []
violations: list[str] = []
offending_labels: list[str] = []
+ schema_incompatibilities: list[SchemaIncompatibility] = []
anchored: list[dict[str, Any]] = []
unanchored: list[dict[str, Any]] = []
seen_labels: set[str] = set()
@@ -494,6 +506,10 @@ def _normalize_code_artifact_metadata_detailed(
schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)
if schema_conformance_error is not None:
item_violations.append(schema_conformance_error)
+ schema_incompatibility = _extraction_schema_incompatibility(label, dumped, block_code)
+ if schema_incompatibility is not None:
+ schema_incompatibilities.append(schema_incompatibility)
+ item_violations.append(render_schema_incompatibility_agent_steer(schema_incompatibility))
download_shape_error = _download_return_shape_error(label, dumped, block_code)
if download_shape_error is not None:
item_violations.append(download_shape_error)
@@ -507,9 +523,13 @@ def _normalize_code_artifact_metadata_detailed(
normalized[label] = _imposed_artifact_skeleton(label, code_blocks[label], trajectory)
if violations:
return CodeArtifactNormalization(
- normalized, _format_code_artifact_violations(violations), violations, offending_labels
+ normalized,
+ _format_code_artifact_violations(violations),
+ violations,
+ offending_labels,
+ schema_incompatibilities,
)
- return CodeArtifactNormalization(normalized, None, [], [])
+ return CodeArtifactNormalization(normalized, None, [], [], [])
def _artifact_label_fragment(label: str) -> str:
@@ -3107,6 +3127,81 @@ def _extraction_schema_conformance_error(label: str, artifact: Mapping[str, Any]
return None
+def _top_level_path_segment(path: str) -> str:
+ head = path.strip()
+ for separator in (".", "[", "/"):
+ index = head.find(separator)
+ if index > 0:
+ head = head[:index]
+ return head.strip()
+
+
+def _known_output_contract_paths(artifact: Mapping[str, Any], code: str) -> set[str]:
+ """Top-level field names the block is known to produce: the snippet's return-dict
+ keys plus the confirmed `goal_value_paths`' top-level segments. Empty when neither
+ is statically determinable, so the incompatibility check stays tolerant."""
+ paths: set[str] = set()
+ return_keys = _top_level_return_dict_keys(code)
+ if return_keys:
+ paths |= return_keys
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(artifact.get(field_name)):
+ for path in _artifact_goal_value_paths(row.get("goal_value_paths")):
+ segment = _top_level_path_segment(path)
+ if segment:
+ paths.add(segment)
+ return paths
+
+
+def _schema_property_summary(schema: Mapping[str, Any]) -> str:
+ names = _schema_object_property_names(schema)
+ if names is None:
+ return ""
+ property_names, _required = names
+ return ", ".join(sorted(property_names))
+
+
+def _extraction_schema_incompatibility(
+ label: str, artifact: Mapping[str, Any], code: str
+) -> SchemaIncompatibility | None:
+ """Detect an edited `extraction_schema` whose object property names overlap NONE of
+ the block's known output contract. Unlike `_extraction_schema_conformance_error`,
+ this fires even when `required` is empty: an optional-only field that maps to nothing
+ the block produces is a non-repairable mismatch, not a tolerated gap. Stays tolerant
+ when the contract or property names are not statically determinable."""
+ if _is_download_intent(artifact, code) or not code.strip():
+ return None
+ known_paths = _known_output_contract_paths(artifact, code)
+ if not known_paths:
+ return None
+ incompatible: set[str] = set()
+ summaries: list[str] = []
+ for value in _artifact_extraction_schema_values(artifact):
+ if isinstance(value, str) and _is_unfilled_artifact_placeholder(value.strip()):
+ continue
+ parsed = _parse_extraction_schema(value)
+ if parsed is None or not validate_schema(parsed):
+ continue
+ names = _schema_object_property_names(parsed)
+ if names is None:
+ continue
+ property_names, _required = names
+ if not property_names or property_names & known_paths:
+ continue
+ incompatible |= property_names
+ summary = _schema_property_summary(parsed)
+ if summary and summary not in summaries:
+ summaries.append(summary)
+ if not incompatible:
+ return None
+ return SchemaIncompatibility(
+ block_label=label,
+ incompatible_paths=tuple(sorted(incompatible)),
+ known_output_paths=tuple(sorted(known_paths)),
+ edited_schema_summary="; ".join(summaries),
+ )
+
+
_EXPECT_DOWNLOAD_ATTR = "expect_download"
@@ -3507,6 +3602,29 @@ def _credentialed_code_block_scout_gate_errors(
return errors
+def _reject_schema_incompatibility(
+ ctx: AgentContext,
+ incompatibility: SchemaIncompatibility,
+ reject: Callable[..., dict[str, Any]],
+) -> dict[str, Any]:
+ """Emit the typed, non-repairable schema-incompatibility outcome: stash the terminal
+ blocker + turn halt so enforcement renders a product-language reply (instead of
+ falling through to repair churn), record it on the context, and reject without
+ persisting the incompatible draft."""
+ signal = build_schema_incompatibility_blocker_signal(incompatibility)
+ stash_blocker_signal(ctx, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
+ ctx.latest_schema_incompatibility = incompatibility
+ return reject(
+ error=render_schema_incompatibility_agent_steer(incompatibility),
+ user_facing_summary=signal.user_facing_reason,
+ data={
+ "failure_type": SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,
+ "schema_incompatibility": incompatibility.to_summary_dict(),
+ },
+ )
+
+
async def _update_workflow(
params: dict[str, Any],
ctx: AgentContext,
@@ -3600,6 +3718,10 @@ async def _update_workflow(
code_artifact_metadata_error = normalization.error
if code_artifact_metadata_error is not None:
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
+ if normalization.schema_incompatibilities:
+ incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)
+ if incompatibility is not None:
+ return _reject_schema_incompatibility(ctx, incompatibility, reject)
prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)
code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
code_authoring_repair_context = (
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index 2267989b1..7a712d79d 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
+from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_REASON_CODE
LOG = structlog.get_logger()
@@ -26,6 +27,7 @@ class TurnHaltKind(StrEnum):
ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"
PROBABLE_SITE_BLOCK = "probable_site_block"
REPAIR_CEILING_REACHED = "repair_ceiling_reached"
+ SCHEMA_INCOMPATIBILITY = "schema_incompatibility"
class TurnHaltVerdict(StrEnum):
@@ -53,12 +55,16 @@ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(
}
)
_PROBABLE_SITE_BLOCK_REASON_CODES = frozenset({"probable_site_block_stop"})
+_SCHEMA_INCOMPATIBILITY_REASON_CODES = frozenset({SCHEMA_INCOMPATIBILITY_REASON_CODE})
# A held blocker whose reason code is in this set must win both the rendered
# reply and the typed halt kind over a later non-terminal trip (e.g. the
# code-authoring churn backstop), which defers entirely when one is present.
GENUINELY_TERMINAL_BLOCKER_REASON_CODES = (
- _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES | _PROBABLE_SITE_BLOCK_REASON_CODES | frozenset({"repair_ceiling_reached"})
+ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES
+ | _PROBABLE_SITE_BLOCK_REASON_CODES
+ | _SCHEMA_INCOMPATIBILITY_REASON_CODES
+ | frozenset({"repair_ceiling_reached"})
)
@@ -74,10 +80,14 @@ _INVOLUNTARY_TURN_HALT_KINDS = frozenset(
TurnHaltKind.LOOP_DETECTED,
TurnHaltKind.PROBABLE_SITE_BLOCK,
TurnHaltKind.REPAIR_CEILING_REACHED,
+ TurnHaltKind.SCHEMA_INCOMPATIBILITY,
}
)
_INVOLUNTARY_BLOCKER_REASON_CODES = (
- _LOOP_TERMINAL_REASON_CODES | _PROBABLE_SITE_BLOCK_REASON_CODES | frozenset({REPAIR_CEILING_REASON_CODE})
+ _LOOP_TERMINAL_REASON_CODES
+ | _PROBABLE_SITE_BLOCK_REASON_CODES
+ | _SCHEMA_INCOMPATIBILITY_REASON_CODES
+ | frozenset({REPAIR_CEILING_REASON_CODE})
)
@@ -105,6 +115,8 @@ def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind |
return TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
if reason in _PROBABLE_SITE_BLOCK_REASON_CODES:
return TurnHaltKind.PROBABLE_SITE_BLOCK
+ if reason in _SCHEMA_INCOMPATIBILITY_REASON_CODES:
+ return TurnHaltKind.SCHEMA_INCOMPATIBILITY
return None
diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.py
index 42ad6555d..3452bbf01 100644
--- a/skyvern/forge/sdk/copilot/turn_outcome.py
+++ b/skyvern/forge/sdk/copilot/turn_outcome.py
@@ -167,7 +167,16 @@ def _string_or_none(value: object) -> str | None:
return value if isinstance(value, str) and value else None
-def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | None]:
+def _schema_incompatibility_summary(ctx: Any) -> dict[str, Any] | None:
+ incompatibility = getattr(ctx, "latest_schema_incompatibility", None)
+ to_summary = getattr(incompatibility, "to_summary_dict", None)
+ if not callable(to_summary):
+ return None
+ summary = to_summary()
+ return summary if isinstance(summary, dict) else None
+
+
+def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, Any]:
turn_halt_kind = getattr(getattr(ctx, "turn_halt", None), "kind", None)
turn_halt_kind_value = getattr(turn_halt_kind, "value", turn_halt_kind)
pending_capability = _string_or_none(getattr(ctx, "code_native_pending_capability", None))
@@ -177,6 +186,7 @@ def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | Non
),
"copilot_repair_ceiling_hit": turn_halt_kind_value == "repair_ceiling_reached",
"copilot_pending_capability": pending_capability,
+ "copilot_schema_incompatibility": _schema_incompatibility_summary(ctx),
}
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index b52dc73ac..23b55a413 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -39,3 +39,8 @@ class TurnOutcome(BaseModel):
copilot_repair_ceiling_hit: bool = False
copilot_pending_capability: str | None = None
copilot_turn_id: str | None = None
+ # Structured summary of an edited-schema-incompatibility terminal outcome
+ # (incompatible_paths, known_output_paths, next_actions, ...), persisted so a
+ # later "what was the problem?" turn can report it. None unless the turn ended
+ # on a schema-incompatibility halt.
+ copilot_schema_incompatibility: dict[str, Any] | None = None
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 4f6b5401f..b97524b06 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -1249,6 +1249,7 @@ class TestRequestPolicyInputGuardrail:
organization_id="org-1",
handler=policy_inputs.handler,
active_criteria=None,
+ config=None,
)
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_code_mode_opt_out.py b/tests/unit/test_copilot_code_mode_opt_out.py
index 8fbac65e5..af21e77d8 100644
--- a/tests/unit/test_copilot_code_mode_opt_out.py
+++ b/tests/unit/test_copilot_code_mode_opt_out.py
@@ -262,4 +262,5 @@ def test_derive_copilot_code_mode_diagnostics_uses_context_state() -> None:
"copilot_last_code_build_failed": True,
"copilot_repair_ceiling_hit": True,
"copilot_pending_capability": "credential-typed code synthesis",
+ "copilot_schema_incompatibility": None,
}
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 0291595c0..12e142e47 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -4,6 +4,7 @@ of the judge-unavailable success bypass."""
from __future__ import annotations
+import asyncio
from unittest.mock import MagicMock
from skyvern.forge.sdk.copilot.agent import (
@@ -28,7 +29,9 @@ from skyvern.forge.sdk.copilot.completion_verification import (
CriterionVerdict,
RunEvidenceSnapshot,
_coerce_result,
+ _render_criteria,
combine_verification_results,
+ evaluate_completion_criteria,
grade_definition_criteria,
grade_present_value_criteria,
run_plane_all_no_evidence,
@@ -59,8 +62,8 @@ from skyvern.forge.sdk.copilot.tools.completion import (
)
-def _criterion(cid: str, outcome: str, *, level: str = "run") -> CompletionCriterion:
- return CompletionCriterion(id=cid, outcome=outcome, level=level) # type: ignore[arg-type]
+def _criterion(cid: str, outcome: str, *, level: str = "run", output_path: str | None = None) -> CompletionCriterion:
+ return CompletionCriterion(id=cid, outcome=outcome, level=level, output_path=output_path) # type: ignore[arg-type]
def _stored(
@@ -154,6 +157,63 @@ def test_reconcile_non_subset_supersedes_wholesale() -> None:
assert decision.superseded_set_id == "wccs_old"
+def test_reconcile_keeps_stored_requested_output_when_fresh_is_generic_rephrase() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_existing",
+ goal_epoch=1,
+ criteria=(
+ _criterion("c0", "The profile details are captured."),
+ _criterion("c1", "The returned record includes ID.", output_path="output.id"),
+ ),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [_criterion("c0", "The full profile information is extracted.")],
+ actionable=True,
+ )
+
+ assert decision.action == "adopt_stored"
+ assert decision.reason == "kept"
+ assert decision.criteria == stored.criteria
+
+
+def test_reconcile_keeps_grounded_stored_requested_output_when_fresh_rephrases_path() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_existing",
+ goal_epoch=1,
+ criteria=(_criterion("c0", "The returned record includes ID.", output_path="output.id"),),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [_criterion("c1", "The final response captures the ID.", output_path="output.id")],
+ actionable=True,
+ )
+
+ assert decision.action == "adopt_stored"
+ assert decision.reason == "kept"
+ assert decision.criteria == stored.criteria
+
+
+def test_reconcile_keeps_grounded_stored_requested_output_when_fresh_id_is_ungrounded() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_existing",
+ goal_epoch=1,
+ criteria=(_criterion("c0", "The returned record includes ID.", output_path="output.id"),),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [_criterion("c1", "The returned record includes ID.")],
+ actionable=True,
+ )
+
+ assert decision.action == "adopt_stored"
+ assert decision.reason == "kept"
+ assert decision.criteria == stored.criteria
+
+
def test_reconcile_epoch_stays_monotonic_after_supersede_chain() -> None:
# A superseded latest row still advances next_epoch, so a new set never
# reuses an epoch number.
@@ -189,7 +249,12 @@ def test_reconcile_no_criteria_anywhere_is_noop() -> None:
def test_criteria_json_round_trip_preserves_level_and_flags() -> None:
criteria = (
- CompletionCriterion(id="c0", outcome="the item is in the cart", implicit=True),
+ CompletionCriterion(
+ id="c0",
+ outcome="the returned record includes ID",
+ implicit=True,
+ output_path="output.id",
+ ),
CompletionCriterion(id="c1", outcome="inputs are reusable", method_mandated=True, level="definition"),
)
assert criteria_from_json(criteria_to_json(criteria)) == criteria
@@ -308,25 +373,97 @@ def test_coerce_result_tristate_mapping() -> None:
assert result.verdict_state_counts() == {"satisfied": 1, "unsatisfied": 3, "unknown": 2}
+def test_explicit_output_path_criterion_prevents_full_verification_when_absent() -> None:
+ id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", id_criterion.id],
+ verdicts=[
+ _verdict("c0", "satisfied", "evidence_confirms"),
+ _verdict(id_criterion.id, "unsatisfied", "no_evidence"),
+ ],
+ )
+
+ assert id_criterion.output_path == "output.id"
+ assert id_criterion.outcome == "The returned record includes ID."
+ assert result.is_fully_satisfied() is False
+
+
+def test_completion_verifier_render_includes_required_output_path() -> None:
+ rendered = _render_criteria([_criterion("c0", "The returned record includes ID.", output_path="output.id")])
+
+ assert "required_output_path=output.id" in rendered
+
+
+def test_required_output_path_missing_is_not_satisfied_by_evidence_text() -> None:
+ id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")
+ criteria = [
+ _criterion(
+ "c0",
+ "The returned record includes status.",
+ output_path="output.status",
+ ),
+ id_criterion,
+ ]
+ snapshot = RunEvidenceSnapshot(
+ workflow_run_id="wr_test",
+ block_outputs={
+ "extract_profile": {
+ "output": {
+ "id": None,
+ "evidence_text": "Search result text mentions ID 1457803926.",
+ "status": "active",
+ }
+ }
+ },
+ )
+
+ async def _handler(**kwargs: object) -> dict:
+ prompt = str(kwargs["prompt"])
+ assert "required_output_path=output.id" in prompt
+ assert '"id": null' in prompt
+ assert "evidence_text" in prompt
+ assert "1457803926" in prompt
+ return {
+ "verdicts": [
+ {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c1",
+ "satisfied": False,
+ "reason_code": "no_evidence",
+ "missing_evidence": "output.id is missing/null.",
+ },
+ ]
+ }
+
+ result = asyncio.run(evaluate_completion_criteria(criteria, snapshot, _handler))
+
+ assert id_criterion.output_path == "output.id"
+ assert grade_present_value_criteria([id_criterion], snapshot) == []
+ assert result.verdicts[1].criterion_id == "c1"
+ assert result.verdicts[1].state == "unsatisfied"
+ assert result.is_fully_satisfied() is False
+
+
_PARAMETERIZED_YAML = """\
-title: provider lookup
+title: account lookup
workflow_definition:
parameters:
- parameter_type: workflow
key: first_name
- parameter_type: workflow
- key: npi_number
+ key: account_id
blocks:
- block_type: goto_url
label: open_portal
- url: https://registry.example.com/search
+ url: https://example.com/search
- block_type: navigation
label: fill_search
- navigation_goal: "Search for {{ first_name }} with NPI {{ npi_number }}"
+ navigation_goal: "Search for {{ first_name }} with account ID {{ account_id }}"
"""
_UNREFERENCED_YAML = """\
-title: provider lookup
+title: account lookup
workflow_definition:
parameters:
- parameter_type: workflow
@@ -334,29 +471,33 @@ workflow_definition:
blocks:
- block_type: goto_url
label: open_portal
- url: https://registry.example.com/search
+ url: https://example.com/search
"""
_NO_PARAMS_YAML = """\
-title: provider lookup
+title: account lookup
workflow_definition:
parameters: []
blocks:
- block_type: goto_url
label: open_portal
- url: https://registry.example.com/search
+ url: https://example.com/search
"""
def test_definition_grading_satisfied_when_parameters_exist_and_are_referenced() -> None:
- criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")]
+ criteria = [
+ _criterion("c0", "the workflow accepts first name and account ID as reusable inputs", level="definition")
+ ]
(verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)
assert verdict.state == "satisfied"
assert verdict.reason_code == "definition_parameters_referenced"
def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced() -> None:
- criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")]
+ criteria = [
+ _criterion("c0", "the workflow accepts first name and account ID as reusable inputs", level="definition")
+ ]
(missing,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)
assert missing.state == "unsatisfied"
assert missing.reason_code == "definition_parameters_missing"
@@ -379,19 +520,19 @@ def test_definition_grading_requires_every_named_input_to_match() -> None:
criteria = [
_criterion(
"c0",
- "the workflow accepts first name, last name, and NPI number as reusable inputs",
+ "the workflow accepts first name, last name, and account ID as reusable inputs",
level="definition",
)
]
- # _PARAMETERIZED_YAML defines first_name + npi_number but not last_name:
+ # _PARAMETERIZED_YAML defines first_name + account_id but not last_name:
# a multi-input ask must not be satisfied by a partial parameter set.
(verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)
assert verdict.state == "unknown"
assert verdict.reason_code == "definition_parameters_unmatched"
full_yaml = _PARAMETERIZED_YAML.replace(
- " - parameter_type: workflow\n key: npi_number",
- " - parameter_type: workflow\n key: last_name\n - parameter_type: workflow\n key: npi_number",
+ " - parameter_type: workflow\n key: account_id",
+ " - parameter_type: workflow\n key: last_name\n - parameter_type: workflow\n key: account_id",
)
(full,) = grade_definition_criteria(criteria, full_yaml)
assert full.state == "satisfied"
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 03d024319..4759b72b2 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -234,6 +234,34 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes
assert getattr(ctx, "blocker_signal", None) is None
+def test_schema_incompatibility_failure_type_stops_without_repair() -> None:
+ # SKY-11380: the typed schema-incompatibility reject must route to STOP, never repair,
+ # so the agent reports the mismatch instead of churning toward repair_ceiling_reached.
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "STOP: the edited extraction_schema declares field(s) [shoebox] that map to no output.",
+ "data": {
+ "failure_type": "schema_incompatibility",
+ "workflow_updated": False,
+ "schema_incompatibility": {
+ "block_label": "capture_row",
+ "incompatible_paths": ["shoebox"],
+ "known_output_paths": ["order_date", "order_total"],
+ },
+ },
+ },
+ ctx=_ctx(),
+ workflow_updated=False,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+ assert contract.repair_decision.next_action != RepairNextAction.REPAIR
+ assert contract.repair_decision.target_blocks == []
+
+
def test_judge_confirmed_suspicious_success_forces_no_change() -> None:
ctx = _ctx()
ctx.last_test_suspicious_success = True
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
new file mode 100644
index 000000000..a1c927302
--- /dev/null
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -0,0 +1,404 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+import pytest
+
+from skyvern.forge.sdk.copilot.completion_criteria_store import (
+ StoredCriteriaSet,
+ StoredCriteriaSnapshot,
+ criteria_to_json,
+ reconcile_completion_criteria,
+)
+from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ RequestPolicy,
+ _apply_requested_output_completion_criteria,
+ _classify_request,
+)
+
+
+async def _policy_for_message(
+ user_message: str,
+ criteria: list[dict[str, Any]],
+ *,
+ config: CopilotConfig | None = None,
+) -> RequestPolicy:
+ async def _handler(**_: Any) -> dict[str, Any]:
+ return {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": criteria,
+ }
+
+ return await _classify_request(
+ user_message,
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=_handler,
+ config=config,
+ )
+
+
+def _stored(*criteria: CompletionCriterion) -> StoredCriteriaSet:
+ return StoredCriteriaSet(set_id="wccs_existing", goal_epoch=1, criteria=tuple(criteria))
+
+
+def _criterion(
+ cid: str,
+ outcome: str,
+ *,
+ level: str = "run",
+ output_path: str | None = None,
+ method_mandated: bool = False,
+) -> CompletionCriterion:
+ return CompletionCriterion(
+ id=cid,
+ outcome=outcome,
+ level=level, # type: ignore[arg-type]
+ output_path=output_path,
+ method_mandated=method_mandated,
+ )
+
+
+def _outcomes(policy: RequestPolicy) -> list[str]:
+ return [criterion.outcome for criterion in policy.completion_criteria]
+
+
+def _criteria_for_path(policy: RequestPolicy, output_path: str) -> list[CompletionCriterion]:
+ return [criterion for criterion in policy.completion_criteria if criterion.output_path == output_path]
+
+
+def _criteria_fingerprint(criteria: list[CompletionCriterion]) -> str:
+ payload = json.dumps(criteria_to_json(criteria), sort_keys=True, separators=(",", ":"))
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[str]) -> list[CompletionCriterion]:
+ return [
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.level == "run"
+ and not criterion.method_mandated
+ and criterion.output_path in requested_output_paths
+ ]
+
+
+async def test_classifier_output_is_augmented_with_generic_requested_outputs() -> None:
+ policy = await _policy_for_message(
+ "Build a registry lookup. Return a final result record with customer name, record id, and status.",
+ [{"outcome": "The profile details are captured."}],
+ )
+
+ rendered = "\n".join(_outcomes(policy))
+ assert "customer name" in rendered
+ assert "record id" in rendered
+ assert "status" in rendered
+ assert _criteria_for_path(policy, "output.customer_name")
+ assert _criteria_for_path(policy, "output.record_id")
+ assert _criteria_for_path(policy, "output.status")
+
+
+async def test_lowercase_record_id_is_augmented_as_requested_output() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with record id.",
+ [],
+ )
+
+ assert _outcomes(policy) == ["The returned record includes record id."]
+ assert policy.completion_criteria[0].output_path == "output.record_id"
+
+
+async def test_leading_output_verb_does_not_enter_requested_output_slug() -> None:
+ policy = await _policy_for_message(
+ "Return a final record. Capture the identifier.",
+ [],
+ )
+
+ assert _outcomes(policy) == ["The returned record includes identifier."]
+ assert policy.completion_criteria[0].output_path == "output.identifier"
+
+
+async def test_possessive_requested_output_slug_is_canonicalized() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with each location's status.",
+ [],
+ )
+
+ assert _outcomes(policy) == ["The returned record includes location status."]
+ assert policy.completion_criteria[0].output_path == "output.location_status"
+
+
+async def test_combined_classifier_output_splits_requested_output_grounding() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with record id and status.",
+ [{"outcome": "The returned record includes record id and status."}],
+ )
+
+ assert _criteria_for_path(policy, "output.record_id")
+ assert _criteria_for_path(policy, "output.status")
+
+
+async def test_schema_derived_output_path_precedes_config_alias_and_slug_fallback() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with tracking number.",
+ [
+ {
+ "outcome": "The returned record includes tracking number.",
+ "output_path": "output.shipment.tracking_number",
+ }
+ ],
+ config=CopilotConfig(requested_output_path_aliases={"tracking number": "output.tracking_number"}),
+ )
+
+ assert _criteria_for_path(policy, "output.shipment.tracking_number")
+ assert not _criteria_for_path(policy, "output.tracking_number")
+
+
+async def test_config_alias_maps_generic_requested_output_field() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with reference number.",
+ [],
+ config=CopilotConfig(requested_output_path_aliases={"reference number": "output.reference_number"}),
+ )
+
+ assert _outcomes(policy) == ["The returned record includes reference number."]
+ assert policy.completion_criteria[0].output_path == "output.reference_number"
+
+
+async def test_classifier_drift_canonicalizes_to_stable_generic_requested_outputs() -> None:
+ prompt = "Return a final record with record id, status, and contact email."
+ classifier_shapes = [
+ [
+ {"outcome": "The complete profile details are visible."},
+ {"outcome": "The final evidence text includes the record id.", "output_path": "output.evidence_text"},
+ {"outcome": "The final record includes a contact email."},
+ {"outcome": "The workflow accepts lookup inputs.", "level": "definition"},
+ {
+ "outcome": "The workflow runs to its intended end state with the expected output.",
+ "method_mandated": True,
+ },
+ ],
+ [
+ {"outcome": "The extracted profile contains record id, status, and contact email."},
+ {"outcome": "The reusable inputs are defined.", "level": "definition"},
+ ],
+ [
+ {
+ "outcome": "Record id is copied from the search result evidence text.",
+ "output_path": "output.evidence_text",
+ },
+ {"outcome": "The returned record includes record id.", "output_path": "output.record_id"},
+ {"outcome": "The returned record includes status."},
+ {"outcome": "A different unbound narrative run criterion is present."},
+ ],
+ ]
+
+ policies = [await _policy_for_message(prompt, criteria) for criteria in classifier_shapes]
+ requested_output_paths = {
+ "output.record_id",
+ "output.status",
+ "output.contact_email",
+ }
+ canonical_subsets = [_requested_output_subset(policy, requested_output_paths) for policy in policies]
+ canonical_json = [criteria_to_json(subset) for subset in canonical_subsets]
+ fingerprints = [_criteria_fingerprint(subset) for subset in canonical_subsets]
+
+ assert canonical_json == [canonical_json[0], canonical_json[0], canonical_json[0]]
+ assert fingerprints == [fingerprints[0], fingerprints[0], fingerprints[0]]
+ assert {criterion.output_path for criterion in canonical_subsets[0]} == requested_output_paths
+ assert len(canonical_subsets[0]) == 3
+ assert all(_criteria_for_path(policy, "output.evidence_text") == [] for policy in policies)
+ assert any(criterion.level == "definition" for criterion in policies[0].completion_criteria)
+ assert any(
+ criterion.outcome == "A different unbound narrative run criterion is present."
+ for criterion in policies[2].completion_criteria
+ )
+
+
+async def test_method_and_setup_text_do_not_become_completion_criteria() -> None:
+ policy = await _policy_for_message(
+ "Build a lookup. Open Show Details, click Search, choose a plan, set the location, then output profile details.",
+ [],
+ )
+
+ rendered = "\n".join(_outcomes(policy)).lower()
+ assert "show details" not in rendered
+ assert "click search" not in rendered
+ assert "choose a plan" not in rendered
+ assert "location" not in rendered
+ assert policy.completion_criteria == []
+
+
+async def test_reusable_input_id_does_not_cover_requested_output_id() -> None:
+ policy = await _policy_for_message(
+ "Accept record id as a reusable input, search by that value, and return a final record with record id.",
+ [{"outcome": "The workflow accepts record id as a reusable input.", "level": "definition"}],
+ )
+
+ id_criteria = [criterion for criterion in policy.completion_criteria if "record id" in criterion.outcome]
+ assert [criterion.level for criterion in id_criteria] == ["definition", "run"]
+ assert [criterion.output_path for criterion in id_criteria] == [None, "output.record_id"]
+
+
+async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_gate_remains() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with record id, status, and phone.",
+ [
+ {"outcome": "The returned record narrative includes record id, status, and phone."},
+ {"outcome": "The portal session reaches the submitted results screen."},
+ ],
+ )
+
+ assert "The returned record narrative includes record id, status, and phone." not in _outcomes(policy)
+ assert "The portal session reaches the submitted results screen." in _outcomes(policy)
+ assert {
+ criterion.output_path
+ for criterion in _requested_output_subset(
+ policy,
+ {"output.record_id", "output.status", "output.phone"},
+ )
+ } == {"output.record_id", "output.status", "output.phone"}
+
+
+async def test_requested_output_coverage_uses_whole_token_sequence_matching() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with id.",
+ [{"outcome": "The returned record includes customer name."}],
+ )
+
+ rendered = "\n".join(_outcomes(policy)).lower()
+ assert "customer name" in rendered
+ assert "includes id" in rendered
+ assert _criteria_for_path(policy, "output.id")
+
+
+async def test_fresh_augmented_requested_output_supersedes_incomplete_stored_generic_criteria() -> None:
+ stored = _stored(_criterion("s0", "The profile details are captured."))
+ policy = await _policy_for_message(
+ "Return a final result record with customer name and record id.",
+ [{"outcome": "The profile details are captured."}],
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ policy.completion_criteria,
+ actionable=True,
+ )
+
+ assert decision.action == "create"
+ assert decision.superseded_set_id == stored.set_id
+ assert any(criterion.output_path == "output.record_id" for criterion in decision.criteria)
+
+
+def test_stored_complete_requested_output_survives_narrowed_generic_fresh_criteria() -> None:
+ stored = _stored(
+ _criterion("s0", "The profile details are captured."),
+ CompletionCriterion(id="s1", outcome="The returned record includes record id.", output_path="output.record_id"),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [_criterion("c0", "The profile details are captured.")],
+ actionable=True,
+ )
+
+ assert decision.action == "adopt_stored"
+ assert decision.criteria == stored.criteria
+
+
+async def test_requested_output_criteria_survive_cap_with_existing_and_generic_criteria() -> None:
+ criteria = [{"outcome": f"Specific retained outcome {index}."} for index in range(7)]
+ criteria.append(
+ {
+ "outcome": "The workflow runs to its intended end state with the expected output.",
+ "method_mandated": True,
+ }
+ )
+
+ policy = await _policy_for_message("Return a final record with record id.", criteria)
+
+ assert len(policy.completion_criteria) == 8
+ rendered = "\n".join(_outcomes(policy))
+ assert "record id" in rendered
+ assert "Specific retained outcome" in rendered
+ assert "intended end state" in rendered
+ assert _criteria_for_path(policy, "output.record_id")
+
+ criteria = [{"outcome": f"Specific retained outcome {index}."} for index in range(7)]
+ criteria.append({"outcome": "The returned record includes record id."})
+
+ policy = await _policy_for_message("Return a final record with record id and status.", criteria)
+
+ assert len(policy.completion_criteria) == 8
+ rendered = "\n".join(_outcomes(policy))
+ assert "record id" in rendered
+ assert "status" in rendered
+ assert "Specific retained outcome" in rendered
+ assert _criteria_for_path(policy, "output.record_id")
+ assert _criteria_for_path(policy, "output.status")
+
+
+def test_requested_output_criteria_survive_cap_when_already_present_before_augmentation() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id=f"c{index}", outcome=f"Specific retained outcome {index}.") for index in range(7)
+ ]
+ + [
+ CompletionCriterion(id="record_id", outcome="The returned record includes record id."),
+ CompletionCriterion(
+ id="floor",
+ outcome="The workflow runs to its intended end state with the expected output.",
+ method_mandated=True,
+ ),
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")
+
+ assert len(policy.completion_criteria) == 8
+ assert [criterion.output_path for criterion in _requested_output_subset(policy, {"output.record_id"})] == [
+ "output.record_id"
+ ]
+ assert any("intended end state" in criterion.outcome for criterion in policy.completion_criteria)
+
+
+def test_requested_output_criteria_can_exceed_cap_without_dropping_requested_fields() -> None:
+ policy = RequestPolicy()
+
+ _apply_requested_output_completion_criteria(
+ policy,
+ "Return a final record with name, record id, status, phone, email, license, taxonomy, specialty, and date.",
+ )
+
+ assert len(policy.completion_criteria) == 9
+ assert {criterion.output_path for criterion in policy.completion_criteria} == {
+ "output.name",
+ "output.record_id",
+ "output.status",
+ "output.phone",
+ "output.email",
+ "output.license",
+ "output.taxonomy",
+ "output.specialty",
+ "output.date",
+ }
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 87b8c82e3..a3e12764e 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -371,7 +371,7 @@ class TestActiveCriteriaPromptAnchor:
class TestClassifierFallbackCompletionCriteria:
@pytest.mark.parametrize(
- ("user_message", "expect_criteria"),
+ ("user_message", "expected_status", "expected_output_paths"),
[
(
(
@@ -380,7 +380,8 @@ class TestClassifierFallbackCompletionCriteria:
"and the result should come out as a record with the entity name, identifier, items, "
"and an overall status."
),
- True,
+ "present",
+ {"output.identifier", "output.location_status", "output.name", "output.overall_status"},
),
(
(
@@ -388,20 +389,29 @@ class TestClassifierFallbackCompletionCriteria:
"the entity name, identifier 1234567890, items, per-location status, overall status, and "
"no-results behavior."
),
- True,
+ "present",
+ {"output.name", "output.identifier", "output.per_location_status", "output.overall_status"},
),
- ("Extract the user's name, id, address, and status from the portal.", False),
- ("Open https://example.com and click the pricing link.", False),
+ (
+ "Extract the user's name, id, address, and status from the portal.",
+ "present",
+ {"output.user_name", "output.id", "output.address", "output.status"},
+ ),
+ ("Open https://example.com and click the pricing link.", "unknown", set()),
# Substring matches ("read" in "already", "name" in "filename", "id" in "decided")
# must not satisfy the whole-word gate.
- ("I already decided the filename and reviewed the status of locations.", False),
+ ("I already decided the filename and reviewed the status of locations.", "unknown", set()),
# Generic structural group term ("entries") satisfies the group gate.
- ("Return a record with the entity name, identifier 1234567890, status, and the grouped entries.", True),
+ (
+ "Return a record with the entity name, identifier 1234567890, status, and the grouped entries.",
+ "present",
+ {"output.name", "output.identifier", "output.status"},
+ ),
],
)
@pytest.mark.asyncio
async def test_classifier_fallback_structured_record_criteria(
- self, user_message: str, expect_criteria: bool
+ self, user_message: str, expected_status: str, expected_output_paths: set[str]
) -> None:
policy = await _classify_request(
user_message=user_message,
@@ -412,22 +422,21 @@ class TestClassifierFallbackCompletionCriteria:
)
assert policy.classifier_status == "fallback"
- assert policy.completion_contract_status == ("present" if expect_criteria else "unknown")
- if not expect_criteria:
+ assert policy.completion_contract_status == expected_status
+ if expected_status == "unknown":
assert policy.completion_criteria
assert all(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)
return
- assert not any(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)
- assert [criterion.id for criterion in policy.completion_criteria] == [
- "fallback_record_identity",
- "fallback_record_identifier",
- "fallback_record_groups",
- "fallback_record_status",
- ]
+ assert {
+ criterion.output_path for criterion in policy.completion_criteria if criterion.output_path
+ } == expected_output_paths
+ assert policy.graded_completion_criteria()
assert policy.requires_user_clarification is False
assert policy.user_response_policy == "proceed"
- assert all(criterion.implicit for criterion in policy.completion_criteria)
+ assert all(
+ criterion.implicit for criterion in policy.completion_criteria if criterion.id.startswith("fallback_")
+ )
assert all(criterion.level == "run" for criterion in policy.completion_criteria)
def test_classifier_fallback_logs_synthesized_structured_record_criteria(
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index f4db87ef7..7b6fa039f 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -27,6 +27,15 @@ from skyvern.forge import app
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.schema_incompatibility import (
+ SchemaIncompatibility,
+ render_schema_incompatibility_user_reason,
+)
+from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
+from skyvern.forge.sdk.copilot.turn_outcome import (
+ build_minimal_turn_outcome,
+ with_copilot_code_mode_diagnostics,
+)
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.routes.workflow_copilot import (
COPILOT_V2_FLAG_KEY,
@@ -34,7 +43,9 @@ from skyvern.forge.sdk.routes.workflow_copilot import (
workflow_copilot_chat_audio,
workflow_copilot_chat_post,
)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatMessage,
WorkflowCopilotChatRequest,
WorkflowCopilotChatSender,
WorkflowCopilotStreamErrorUpdate,
@@ -1374,3 +1385,74 @@ async def test_persist_state_keeps_verified_review_tested_proposal(monkeypatch:
persisted = calls[0].kwargs["proposed_workflow"]
assert persisted is not None
assert persisted.get("_copilot_unvalidated") is not True
+
+
+def _schema_incompatibility_ctx() -> SimpleNamespace:
+ incompat = SchemaIncompatibility(
+ block_label="capture_row",
+ incompatible_paths=("shoebox",),
+ known_output_paths=("order_date", "order_total"),
+ )
+ return SimpleNamespace(
+ latest_schema_incompatibility=incompat,
+ turn_halt=SimpleNamespace(kind=TurnHaltKind.SCHEMA_INCOMPATIBILITY),
+ code_native_pending_capability=None,
+ last_test_ok=None,
+ last_failed_workflow_yaml=None,
+ )
+
+
+def test_schema_incompatibility_turn_outcome_is_not_repair_ceiling() -> None:
+ # SKY-11380: the schema-incompatibility halt is a distinct typed outcome; it must
+ # not masquerade as the repair-ceiling diagnostic on the persisted turn.
+ ctx = _schema_incompatibility_ctx()
+ reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)
+ outcome = with_copilot_code_mode_diagnostics(
+ build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),
+ ctx,
+ )
+
+ assert outcome.copilot_repair_ceiling_hit is False
+ assert outcome.copilot_schema_incompatibility is not None
+ assert outcome.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]
+ assert outcome.copilot_schema_incompatibility["known_output_paths"] == ["order_date", "order_total"]
+
+
+def test_schema_incompatibility_persists_and_recalls_for_followup_turn() -> None:
+ # The follow-up "what was the problem?" turn reads the prior assistant outcome from
+ # chat history; the structured incompatibility survives the round-trip so it can be reported.
+ ctx = _schema_incompatibility_ctx()
+ reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)
+ outcome = with_copilot_code_mode_diagnostics(
+ build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),
+ ctx,
+ )
+ now = datetime.now(timezone.utc)
+ messages = [
+ WorkflowCopilotChatMessage(
+ workflow_copilot_chat_message_id="m1",
+ workflow_copilot_chat_id="c1",
+ sender=WorkflowCopilotChatSender.USER,
+ content="add a shoebox field to the extraction",
+ created_at=now,
+ modified_at=now,
+ ),
+ WorkflowCopilotChatMessage(
+ workflow_copilot_chat_message_id="m2",
+ workflow_copilot_chat_id="c1",
+ sender=WorkflowCopilotChatSender.AI,
+ content=reply,
+ turn_outcome=outcome,
+ created_at=now,
+ modified_at=now,
+ ),
+ ]
+
+ recalled = workflow_copilot_route._latest_assistant_turn_outcome(messages)
+
+ assert recalled is not None
+ assert recalled.terminal_reason == "turn_halt:schema_incompatibility"
+ assert recalled.copilot_schema_incompatibility is not None
+ assert recalled.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]
+ # The persisted reply reports the problem in product language.
+ assert "shoebox" in messages[1].content