Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11464 Stabilize Copilot completion criteria outputs (#6814)
Andrew Neilson80c40f0bdf78
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 35b9e2fc6..aeb2c31e4 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -404,10 +404,14 @@ def _reconcile_completion_criteria_on_context(snapshot = policy_inputs.stored_completion_criteriaif snapshot is None:return+ requested_output_path_aliases = (+ ctx.copilot_config.requested_output_path_aliases if ctx.copilot_config is not None else None+ )decision = reconcile_completion_criteria(snapshot,list(policy.completion_criteria),actionable=policy.user_response_policy != "ask_clarification",+ requested_output_path_aliases=requested_output_path_aliases,)if decision.action == "adopt_stored":policy.completion_criteria = list(decision.criteria)@@ -3105,6 +3109,7 @@ def _build_copilot_input_guardrails(organization_id=policy_inputs.organization_id,handler=policy_inputs.handler,active_criteria=_stored_active_completion_criteria(policy_inputs),+ config=getattr(ctx, "copilot_config", None) if isinstance(ctx, CopilotContext) else None,)if isinstance(ctx, CopilotContext):turn_intent_classifier_result = None@@ -3551,6 +3556,7 @@ async def _run_copilot_turn_impl(prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,block_authoring_policy=copilot_config.block_authoring_policy,impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,+ copilot_config=copilot_config,target_block_label=getattr(chat_request, "target_block_label", None),)# Fail loud if a future caller skips the kwarg and gets a fresh UUID fromdiff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex f677522f7..1a987da0d 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,is_fallback_floor_criterion,normalized_criterion_outcome_key,+ requested_output_path_for_field,)ReconcileAction = Literal["create", "adopt_stored", "none"]@@ -112,6 +113,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"implicit": criterion.implicit,"method_mandated": criterion.method_mandated,"level": criterion.level,+ "output_path": criterion.output_path,}for criterion in criteria]@@ -129,6 +131,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:if not isinstance(criterion_id, str) or not isinstance(outcome, str) or not outcome.strip():continuelevel = item.get("level")+ output_path = item.get("output_path")criteria.append(CompletionCriterion(id=criterion_id,@@ -136,13 +139,120 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:implicit=bool(item.get("implicit")),method_mandated=bool(item.get("method_mandated")),level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]+ output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,))return tuple(criteria)+def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:+ if criterion.output_path:+ return f"output_path:{criterion.output_path}"+ return f"outcome:{normalized_criterion_outcome_key(criterion.outcome)}"++def _outcome_key_set(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:- return {normalized_criterion_outcome_key(criterion.outcome) for criterion in criteria}+ return {_criterion_reconcile_key(criterion) for criterion in criteria}+++def _word_tokens(text: str) -> list[str]:+ return "".join(char if char.isalnum() else " " for char in text.casefold()).split()+++_REQUESTED_OUTPUT_WORDS = frozenset(+ "capture captured extract extracted final include included includes output read record result return returned".split()+)+_REQUESTED_OUTPUT_FIELD_TOKENS = frozenset(+ "address addresses date dates email emails id identifier identifiers license licenses location locations name names "+ "npi number numbers phone phones result specialties specialty status statuses taxonomy".split()+)+_GENERIC_PROFILE_MARKERS = (+ "profile details",+ "profile information",+ "profile is captured",+ "profile is extracted",+ "intended end state",+ "expected output",+)+++def _requested_output_tokens(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:+ tokens: set[str] = set()+ for criterion in criteria:+ if criterion.level == "definition" or criterion.method_mandated:+ continue+ outcome_tokens = _word_tokens(criterion.outcome)+ if not any(word in _REQUESTED_OUTPUT_WORDS for word in outcome_tokens):+ continue+ for token in outcome_tokens:+ if token in _REQUESTED_OUTPUT_FIELD_TOKENS:+ tokens.add(token)+ return tokens+++def _requested_output_paths(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> set[str]:+ return {+ criterion.output_path+ for criterion in criteria+ if criterion.output_path and criterion.level != "definition" and not criterion.method_mandated+ }+++def _criterion_mentions_output_path(+ criterion: CompletionCriterion,+ output_path: str,+ aliases: dict[str, str] | None = None,+) -> bool:+ outcome_tokens = _word_tokens(criterion.outcome)+ max_span_len = min(4, len(outcome_tokens))+ for span_len in range(max_span_len, 0, -1):+ for start in range(len(outcome_tokens) - span_len + 1):+ field_name = " ".join(outcome_tokens[start : start + span_len])+ if requested_output_path_for_field(field_name, aliases) == output_path:+ return True+ return False+++def _is_generic_profile_criterion(criterion: CompletionCriterion) -> bool:+ key = normalized_criterion_outcome_key(criterion.outcome)+ return is_fallback_floor_criterion(criterion) or any(marker in key for marker in _GENERIC_PROFILE_MARKERS)+++def _fresh_generic_rephrase_lacks_stored_requested_outputs(+ stored: tuple[CompletionCriterion, ...],+ fresh: list[CompletionCriterion],+ *,+ requested_output_path_aliases: dict[str, str] | None = None,+) -> bool:+ stored_requested_criteria = tuple(+ criterion+ for criterion in stored+ if criterion.output_path and criterion.level != "definition" and not criterion.method_mandated+ )+ stored_requested_paths = {+ output_path for criterion in stored_requested_criteria if (output_path := criterion.output_path) is not None+ }+ if stored_requested_paths:+ fresh_requested_paths = _requested_output_paths(fresh)+ missing_paths = stored_requested_paths - fresh_requested_paths+ if not missing_paths:+ return False+ if fresh and all(_is_generic_profile_criterion(criterion) for criterion in fresh):+ return True+ return any(+ criterion.output_path is None+ and _criterion_mentions_output_path(criterion, output_path, requested_output_path_aliases)+ for criterion in fresh+ for output_path in missing_paths+ )++ stored_requested_tokens = _requested_output_tokens(stored)+ if not stored_requested_tokens:+ return False+ if not fresh or not all(_is_generic_profile_criterion(criterion) for criterion in fresh):+ return False+ fresh_tokens = set().union(*(_word_tokens(criterion.outcome) for criterion in fresh))+ return bool(stored_requested_tokens - fresh_tokens)def reconcile_completion_criteria(@@ -150,6 +260,7 @@ def reconcile_completion_criteria(fresh: list[CompletionCriterion],*,actionable: bool,+ requested_output_path_aliases: dict[str, str] | None = None,) -> ReconcileDecision:"""Decide once per turn whether the stored set survives or a new epoch begins.@@ -176,6 +287,14 @@ def reconcile_completion_criteria(return ReconcileDecision(action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria)+ if _fresh_generic_rephrase_lacks_stored_requested_outputs(+ stored.criteria,+ fresh,+ requested_output_path_aliases=requested_output_path_aliases,+ ):+ return ReconcileDecision(+ action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria+ )if not actionable:return ReconcileDecision(action="adopt_stored", reason="not_actionable", epoch=stored.goal_epoch, criteria=stored.criteriadiff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex ebd8191be..6ff5f1109 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -252,7 +252,8 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:flags += " (implicit)"if criterion.method_mandated:flags += " (method_mandated)"- parts.append(f"- {criterion.id}: {criterion.outcome}{flags}")+ output_path = f" [required_output_path={criterion.output_path}]" if criterion.output_path else ""+ parts.append(f"- {criterion.id}: {criterion.outcome}{output_path}{flags}")return "\n".join(parts)diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex 4398a6617..61c385fef 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -366,6 +366,7 @@ class CopilotConfig:fallback_llm_key: str | None = field(default_factory=_default_fallback_llm_key)block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARDimpose_synthesized_code_block: bool = False+ requested_output_path_aliases: dict[str, str] = field(default_factory=dict)def nudge(self, key: str) -> str:return self.enforcement_nudges.get(key, DEFAULT_ENFORCEMENT_NUDGES[key])diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 9b600d355..9f18f2fa2 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -11,7 +11,7 @@ from pydantic import BaseModel, Fieldfrom typing_extensions import NotRequired, TypedDictfrom skyvern.forge.sdk.copilot.build_phase import BuildPhase-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidencefrom skyvern.forge.sdk.workflow.models.workflow import Workflow@@ -95,6 +95,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContractfrom skyvern.forge.sdk.copilot.narration import NarratorStatefrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy+ from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibilityfrom skyvern.forge.sdk.copilot.turn_context import TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltfrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent@@ -386,6 +387,7 @@ class CopilotContext(AgentContext):block_goal_main_goal: str = ""allow_untested_workflow_draft: bool = Falserequest_policy: RequestPolicy | None = None+ copilot_config: CopilotConfig | None = Noneblock_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARDimpose_synthesized_code_block: bool = Falsetarget_block_label: str | None = None@@ -406,6 +408,7 @@ class CopilotContext(AgentContext):latest_tool_blocker_signal: CopilotToolBlockerSignal | None = Nonetool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)turn_halt: TurnHalt | None = None+ latest_schema_incompatibility: SchemaIncompatibility | None = None# ``None`` until usage is observed; ``0`` only when a provider explicitly# reported zero. Distinct values let cost grading flag missing telemetry.diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex ffa79d7bf..6bf184051 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (from skyvern.forge.sdk.copilot.output_policy import url_originfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name+from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_FAILURE_TYPEfrom skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE@@ -50,6 +51,7 @@ class DiagnosisFailureType(StrEnum):REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"ACTIVE_RUN_TERMINAL_EVIDENCE = "active_run_terminal_evidence"UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"+ SCHEMA_INCOMPATIBILITY = "schema_incompatibility"UNKNOWN = "unknown"@@ -235,6 +237,11 @@ def build_diagnosis_repair_contract("Stop the current retry loop: the active run reached the requested browser state, ""but the reusable workflow is not verified end-to-end.")+ elif failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:+ decision_summary = (+ "Stop re-authoring: the edited extraction schema declares fields that map to no output the "+ "workflow produces, so the mismatch is not repairable without user input."+ )completion_check = {RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",@@ -482,6 +489,8 @@ def _failure_type(return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INITif _safe_str(data.get("failure_type")) == "missing_credential_or_init":return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ if _safe_str(data.get("failure_type")) == SCHEMA_INCOMPATIBILITY_FAILURE_TYPE:+ return DiagnosisFailureType.SCHEMA_INCOMPATIBILITYif repair_context is not None:return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILUREif outcome_verified:@@ -511,6 +520,8 @@ def _next_action(return RepairNextAction.STOPif failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:return RepairNextAction.STOP+ if failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY:+ return RepairNextAction.STOPif ctx.last_test_non_retriable_nav_error:return RepairNextAction.STOPif _last_test_anti_bot_is_terminal(ctx, data) and failure_type in {diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 9b1703475..bbc848627 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -13,6 +13,7 @@ import structlogfrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import StructuredContextfrom skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_errorfrom skyvern.forge.sdk.copilot.output_utils import parse_final_response@@ -144,6 +145,7 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "_MAX_COMPLETION_CRITERIA = 8_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200+_REQUESTED_OUTPUT_CRITERION_ID_PREFIX = "__copilot_requested_output__"FALLBACK_FLOOR_CRITERION_ID_PREFIX = "__copilot_fallback_floor__"_FALLBACK_FLOOR_BASE_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}run"@@ -154,6 +156,40 @@ _FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and reCriterionLevel = Literal["definition", "run"]_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})+_OUTPUT_INTENT_RE = re.compile(+ r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"+ r"final\s+(?:extracted\s+)?fields?|result\s+records?|returned\s+records?)\b",+ re.I,+)+_OUTPUT_SPAN_END_RE = re.compile(r"[\n.!?]")+_OUTPUT_METHOD_TAIL_RE = re.compile(+ r"\b(?:by|via|using|after|before|then|click(?:ing)?|open(?:ing)?|select(?:ing)?|"+ r"choose|choosing|search(?:ing)?|navigate|go\s+to)\b.*",+ re.I,+)+_OUTPUT_SPLIT_RE = re.compile(r",|;|\band\b|\bplus\b|&", re.I)+_OUTPUT_FIELD_CONNECTOR_RE = re.compile(+ r"\b(?:with|including|include|includes|containing|contains|fields?|result)\b[:\s]+",+ re.I,+)+_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")+_OUTPUT_FIELD_WORDS = frozenset(+ "address addresses date dates email emails id identifier identifiers license licenses location locations "+ "name names number numbers phone phones specialties specialty status statuses taxonomy".split()+)+_OUTPUT_GENERIC_WORDS = frozenset(+ "a all an data detail details each entity final for information its of output outputs profile record records "+ "result results structured the value values".split()+)+_OUTPUT_METHOD_WORDS = frozenset("choose click open plan search select setup show".split())+_OUTPUT_LEADING_FIELD_WORDS = frozenset(+ "capture captured extract extracted include included includes output read return returned".split()+)+_OUTPUT_INPUT_ONLY_WORDS = frozenset({"input", "inputs", "parameter", "parameters", "reusable"})+_OUTPUT_OUTCOME_WORDS = frozenset(+ "capture captured extract extracted final include included includes output read record result return returned".split()+)+@dataclass(frozen=True)class CompletionCriterion:@@ -164,6 +200,7 @@ class CompletionCriterion:# "definition": a property of the workflow definition itself, graded against the# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".level: CriterionLevel = "run"+ output_path: str | None = None@dataclass@@ -528,7 +565,9 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:outcome = " ".join(outcome_raw.split())[:_COMPLETION_CRITERION_OUTCOME_MAX_CHARS].strip()if not outcome:continue- key = normalized_criterion_outcome_key(outcome)+ output_path_raw = item.get("output_path")+ output_path = output_path_raw.strip() if isinstance(output_path_raw, str) and output_path_raw.strip() else None+ key = output_path or normalized_criterion_outcome_key(outcome)if key in seen:continueseen.add(key)@@ -542,6 +581,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:level=cast(CriterionLevel, level_raw)if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELSelse "run",+ output_path=output_path,))if len(criteria) >= _MAX_COMPLETION_CRITERIA:@@ -554,20 +594,321 @@ def normalized_criterion_outcome_key(outcome: str) -> str:return collapsed.strip().lower().rstrip(".!?;:,").strip()+def _output_intent_spans(user_message: str) -> list[str]:+ message = user_message or ""+ spans: list[str] = []+ for match in _OUTPUT_INTENT_RE.finditer(message):+ tail = message[match.start() :]+ end_match = _OUTPUT_SPAN_END_RE.search(tail)+ span = tail[: end_match.start()] if end_match is not None else tail+ span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")+ if span:+ spans.append(span)+ return spans+++def _normalize_requested_output_aliases(aliases: dict[str, str] | None) -> dict[str, str]:+ normalized: dict[str, str] = {}+ for field_name, output_path in (aliases or {}).items():+ if not isinstance(field_name, str) or not isinstance(output_path, str):+ continue+ path = output_path.strip()+ if not path.startswith("output."):+ continue+ key = " ".join(_word_tokens(field_name))+ if key:+ normalized[key] = path+ return normalized+++def _lookup_requested_output_path_alias(field_name: str, aliases: dict[str, str] | None) -> str | None:+ normalized_aliases = _normalize_requested_output_aliases(aliases)+ field_key = " ".join(_word_tokens(field_name))+ if not field_key:+ return None+ if field_key in normalized_aliases:+ return normalized_aliases[field_key]+ field_words = field_key.split()+ for alias_key, output_path in normalized_aliases.items():+ alias_words = alias_key.split()+ if _tokens_contain_sequence(field_words, alias_words) or _tokens_contain_sequence(alias_words, field_words):+ return output_path+ return None+++def _schema_output_path_aliases_from_criteria(criteria: list[CompletionCriterion]) -> dict[str, str]:+ aliases: dict[str, str] = {}+ for criterion in criteria:+ if criterion.level == "definition" or criterion.method_mandated or not criterion.output_path:+ continue+ path_tokens = _word_tokens(criterion.output_path.removeprefix("output."))+ keep_generic_words = (+ {"record"} if any(token in {"id", "identifier", "identifiers"} for token in path_tokens) else set()+ )+ path_phrase = " ".join(+ token for token in path_tokens if token not in _OUTPUT_GENERIC_WORDS or token in keep_generic_words+ )+ if path_phrase:+ aliases.setdefault(path_phrase, criterion.output_path)+ for token in path_tokens:+ if token and token not in _OUTPUT_GENERIC_WORDS:+ aliases.setdefault(token, criterion.output_path)+ return aliases+++def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | None = None) -> str | None:+ candidate = _OUTPUT_METHOD_TAIL_RE.sub("", segment).strip(" :-")+ if not candidate:+ return None++ connector_matches = list(_OUTPUT_FIELD_CONNECTOR_RE.finditer(candidate))+ if connector_matches:+ candidate = candidate[connector_matches[-1].end() :]+ candidate = re.sub(r"\b([A-Za-z0-9]+)'s\b", r"\1", candidate)+ candidate = " ".join(re.sub(r"[^A-Za-z0-9 _/-]+", " ", candidate).split()).strip(" :-")+ if not candidate:+ return None+ normalized_aliases = _normalize_requested_output_aliases(aliases)+ if " ".join(_word_tokens(candidate)) in normalized_aliases:+ return candidate+ candidate_words = candidate.split()+ for word in candidate_words:+ if " ".join(_word_tokens(word)) in normalized_aliases:+ return word+ if _lookup_requested_output_path_alias(candidate, aliases) is not None:+ return candidate++ acronyms = [+ token+ for token in _OUTPUT_ACRONYM_RE.findall(candidate)+ if token.casefold() not in _OUTPUT_METHOD_WORDS+ and token.casefold() not in _OUTPUT_GENERIC_WORDS+ and _lookup_requested_output_path_alias(token, aliases) is not None+ ]+ if acronyms:+ return acronyms[0]++ words = candidate.split()+ normalized_words = [word.casefold().strip("_-/") for word in words if word.strip("_-/")]+ if not normalized_words:+ return None+ while normalized_words and normalized_words[0] in _OUTPUT_LEADING_FIELD_WORDS:+ words = words[1:]+ normalized_words = normalized_words[1:]+ if not normalized_words:+ return None+ if any(word in _OUTPUT_METHOD_WORDS for word in normalized_words):+ return None+ if all(word in _OUTPUT_GENERIC_WORDS for word in normalized_words):+ return None+ field_indexes = [i for i, word in enumerate(normalized_words) if word in _OUTPUT_FIELD_WORDS]+ if not field_indexes:+ return None+ status_indexes = [i for i, word in enumerate(normalized_words) if word in {"status", "statuses"}]+ if status_indexes:+ field_indexes = status_indexes+ last_field_index = field_indexes[-1]+ start = max(0, last_field_index - 2)+ keep_generic_words = (+ {"record"} if normalized_words[last_field_index] in {"id", "identifier", "identifiers"} else set()+ )+ phrase_words = [+ word+ for word in words[start : last_field_index + 1]+ if word.casefold().strip("_-/") not in _OUTPUT_GENERIC_WORDS+ or word.casefold().strip("_-/") in keep_generic_words+ ]+ if not phrase_words:+ return None+ return " ".join(phrase_words)+++def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:+ fields: list[str] = []+ seen: set[str] = set()+ for span in _output_intent_spans(user_message):+ for segment in _OUTPUT_SPLIT_RE.split(span):+ field_name = _clean_requested_output_candidate(segment, aliases)+ if field_name is None:+ continue+ key = field_name.casefold()+ if key in seen:+ continue+ seen.add(key)+ fields.append(field_name)+ return fields+++def requested_output_path_for_field(field_name: str, aliases: dict[str, str] | None = None) -> str:+ alias_path = _lookup_requested_output_path_alias(field_name, aliases)+ if alias_path is not None:+ return alias_path+ words = _word_tokens(field_name)+ slug = "_".join(words) or "field"+ return f"output.{slug}"+++def _requested_output_field_label(field_name: str, output_path: str) -> str:+ stripped = field_name.strip()+ if stripped.isupper():+ return stripped+ return " ".join(_word_tokens(field_name))+++def _tokens_contain_sequence(haystack: list[str], needle: list[str]) -> bool:+ if not needle or len(needle) > len(haystack):+ return False+ for index in range(len(haystack) - len(needle) + 1):+ if haystack[index : index + len(needle)] == needle:+ return True+ return False+++def _criterion_text_covers_requested_output(criterion: CompletionCriterion, field_name: str) -> bool:+ if criterion.level == "definition" or criterion.method_mandated:+ return False+ outcome_words = _word_tokens(criterion.outcome)+ field_words = _word_tokens(field_name)+ if not _tokens_contain_sequence(outcome_words, field_words):+ return False+ if any(word in _OUTPUT_INPUT_ONLY_WORDS for word in outcome_words) and not any(+ word in _OUTPUT_OUTCOME_WORDS for word in outcome_words+ ):+ return False+ return True+++def _criterion_covers_requested_output(+ criterion: CompletionCriterion, field_name: str, aliases: dict[str, str] | None = None+) -> bool:+ if criterion.level == "definition" or criterion.method_mandated:+ return False+ output_path = requested_output_path_for_field(field_name, aliases)+ return criterion.output_path == output_path or _criterion_text_covers_requested_output(criterion, field_name)+++def _criterion_text_covers_any_requested_output(criterion: CompletionCriterion, requested_fields: list[str]) -> bool:+ return any(_criterion_text_covers_requested_output(criterion, field_name) for field_name in requested_fields)+++def _requested_output_criterion_id(output_path: str) -> str:+ slug = "_".join(_word_tokens(output_path)) or "field"+ return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"+++def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:+ key = normalized_criterion_outcome_key(criterion.outcome)+ if is_fallback_floor_criterion(criterion):+ return True+ generic_markers = (+ "intended end state",+ "expected output",+ "profile details",+ "profile information",+ "requested identifier-like value",+ "target entity",+ "grouped row entries",+ )+ return any(marker in key for marker in generic_markers)+++def _criterion_drop_priority(criterion: CompletionCriterion, requested_output_paths: set[str]) -> int:+ if criterion.output_path in requested_output_paths:+ return 0+ if criterion.level == "definition":+ return 0+ if is_fallback_floor_criterion(criterion):+ return 4+ if criterion.method_mandated:+ return 0+ if _generic_completion_criterion(criterion):+ return 3+ return 2+++def _cap_completion_criteria(+ criteria: list[CompletionCriterion], requested_output_paths: set[str]+) -> list[CompletionCriterion]:+ capped = list(criteria)+ while len(capped) > _MAX_COMPLETION_CRITERIA:+ drop_index = max(+ range(len(capped)),+ key=lambda index: (_criterion_drop_priority(capped[index], requested_output_paths), index),+ )+ if _criterion_drop_priority(capped[drop_index], requested_output_paths) == 0:+ break+ del capped[drop_index]+ return capped+++def _apply_requested_output_completion_criteria(+ policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None+) -> None:+ schema_aliases = _schema_output_path_aliases_from_criteria(policy.completion_criteria)+ config_aliases = _normalize_requested_output_aliases(aliases)+ schema_aliases = _normalize_requested_output_aliases(schema_aliases)+ detection_aliases = {**config_aliases, **schema_aliases}+ requested_fields = _requested_output_fields(user_message, detection_aliases)+ if not requested_fields:+ return++ requested_specs: list[tuple[str, str, str]] = []+ requested_output_paths: set[str] = set()+ for field_name in requested_fields:+ output_path = (+ _lookup_requested_output_path_alias(field_name, schema_aliases)+ or _lookup_requested_output_path_alias(field_name, config_aliases)+ or requested_output_path_for_field(field_name)+ )+ field_label = _requested_output_field_label(field_name, output_path)+ if output_path in requested_output_paths:+ continue+ requested_output_paths.add(output_path)+ requested_specs.append((field_name, output_path, field_label))++ if not requested_output_paths:+ return++ preserved_criteria = [+ criterion+ for criterion in policy.completion_criteria+ if criterion.level == "definition"+ or criterion.method_mandated+ or (+ criterion.output_path not in requested_output_paths+ and not _criterion_text_covers_any_requested_output(criterion, requested_fields)+ )+ ]+ canonical_requested_criteria = [+ CompletionCriterion(+ id=_requested_output_criterion_id(output_path),+ outcome=f"The returned record includes {field_label}.",+ level="run",+ output_path=output_path,+ )+ for _field_name, output_path, field_label in requested_specs+ ]+ policy.completion_criteria = _cap_completion_criteria(+ preserved_criteria + canonical_requested_criteria,+ requested_output_paths,+ )++def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | None) -> str:if not criteria:return ""- return json.dumps(- [- {- "outcome": criterion.outcome,- "implicit": criterion.implicit,- "method_mandated": criterion.method_mandated,- "level": criterion.level,- }- for criterion in criteria- ]- )+ rendered: list[dict[str, Any]] = []+ for criterion in criteria:+ item: dict[str, Any] = {+ "outcome": criterion.outcome,+ "implicit": criterion.implicit,+ "method_mandated": criterion.method_mandated,+ "level": criterion.level,+ }+ if criterion.output_path:+ item["output_path"] = criterion.output_path+ rendered.append(item)+ return json.dumps(rendered)def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:@@ -604,6 +945,7 @@ def _classifier_fallback_policy(failure_kind: str,retry_count: int = 0,user_message: str = "",+ requested_output_path_aliases: dict[str, str] | None = None,) -> RequestPolicy:if failure_kind not in _CLASSIFIER_FAILURE_KINDS:failure_kind = "provider_error"@@ -626,7 +968,7 @@ def _classifier_fallback_policy(classifier_failure_kind=failure_kind,completion_criterion_ids=[criterion.id for criterion in fallback_criteria],)- return RequestPolicy(+ policy = RequestPolicy(credential_input_kind="credential_id" if ids else "none",credential_refs=ids,completion_criteria=fallback_criteria or build_classifier_fallback_floor(ids),@@ -635,6 +977,10 @@ def _classifier_fallback_policy(classifier_retry_count=retry_count,completion_contract_status="present" if fallback_criteria else "unknown",)+ _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)+ if policy.graded_completion_criteria():+ policy.completion_contract_status = "present"+ return policydef _word_tokens(text: str) -> list[str]:@@ -754,7 +1100,9 @@ async def _classify_request(handler: Any,*,active_criteria: list[CompletionCriterion] | None = None,+ config: CopilotConfig | None = None,) -> RequestPolicy:+ requested_output_path_aliases = config.requested_output_path_aliases if config is not None else {}ids = _credential_ids(user_message)raw_secret_present = _raw_secret_detected(user_message)if raw_secret_present and handler is None:@@ -763,6 +1111,7 @@ async def _classify_request(raw_secret_present=True,failure_kind="raw_secret_no_handler",user_message=user_message,+ requested_output_path_aliases=requested_output_path_aliases,)structural_reason = _structural_clarification_reason(user_message)if structural_reason != "none" and not raw_secret_present:@@ -778,6 +1127,7 @@ async def _classify_request(raw_secret_present=False,failure_kind="missing_handler",user_message=user_message,+ requested_output_path_aliases=requested_output_path_aliases,)# Raw-secret turns intentionally reach the classifier when available so it@@ -806,6 +1156,7 @@ async def _classify_request(failure_kind=failure_kind,retry_count=retry_count,user_message=user_message,+ requested_output_path_aliases=requested_output_path_aliases,)raw_payload = _coerce_classifier_payload(raw)@@ -817,6 +1168,7 @@ async def _classify_request(failure_kind="provider_error",retry_count=retry_count,user_message=user_message,+ requested_output_path_aliases=requested_output_path_aliases,)policy = _classification_from_raw(raw_payload)@@ -865,6 +1217,10 @@ async def _classify_request(policy.clarification_reason = "none"policy.requires_user_clarification = Falsepolicy.raw_secret_evidence = None+ _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)+ policy.completion_contract_status = (+ "present" if policy.completion_contract or policy.graded_completion_criteria() else "absent"+ )return policy@@ -1335,6 +1691,7 @@ async def build_request_policy(organization_id: str,handler: Any,active_criteria: list[CompletionCriterion] | None = None,+ config: CopilotConfig | None = None,) -> RequestPolicy:policy = await _classify_request(user_message,@@ -1343,6 +1700,7 @@ async def build_request_policy(global_llm_context,handler,active_criteria=active_criteria,+ config=config,)policy.raw_secret_detected = policy.raw_secret_detected or policy.credential_input_kind == "raw_secret"policy.existing_workflow_credential_ids = sorted(workflow_credential_ids(workflow_yaml))diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex b2569f8f5..799450bc5 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -41,6 +41,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+ from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibilityfrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltfrom skyvern.forge.sdk.routes.event_source_stream import EventSourceStreamfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession@@ -308,6 +309,10 @@ class AgentContext:# render the current tool result from structured product text.latest_tool_blocker_signal: CopilotToolBlockerSignal | None = Nonetool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)+ # Latest edited-schema-incompatibility terminal outcome, set when an edited+ # extraction_schema declares fields that map to no output the block produces.+ # Surfaced into the persisted TurnOutcome so a later turn can report it.+ latest_schema_incompatibility: SchemaIncompatibility | None = Nonedef mcp_to_copilot(mcp_result: dict[str, Any]) -> dict[str, Any]:diff --git a/skyvern/forge/sdk/copilot/schema_incompatibility.py b/skyvern/forge/sdk/copilot/schema_incompatibility.pynew file mode 100644index 000000000..aaa8143ba--- /dev/null+++ b/skyvern/forge/sdk/copilot/schema_incompatibility.py@@ -0,0 +1,145 @@+"""Typed terminal outcome for an edited extraction schema whose fields map to no+output the workflow produces.++When a user edits a code block's confirmed ``extraction_schema`` to add fields that+overlap none of the block's known output contract (its top-level return keys plus+confirmed ``goal_value_paths``), re-authoring cannot reconcile the mismatch: there is+nothing on the page or in the return for the new field to bind to. Surfacing it as a+typed, non-repairable schema-incompatibility outcome — rather than letting the agent+churn until the repair ceiling — preserves the existing draft and tells the user+exactly which fields do not map.+"""++from __future__ import annotations++from typing import Any++from pydantic import BaseModel, ConfigDict, Field++from skyvern.forge.sdk.copilot.blocker_signal import (+ CopilotToolBlockerSignal,+ assert_clean_user_facing_text,+)++SCHEMA_INCOMPATIBILITY_REASON_CODE = "schema_incompatibility"+SCHEMA_INCOMPATIBILITY_FAILURE_TYPE = "schema_incompatibility"+SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL = "update_and_run_blocks"++_DEFAULT_NEXT_ACTIONS: tuple[str, ...] = (+ "Map the field to a value this workflow already produces.",+ "Remove the field from the extraction schema.",+ "Describe what the field should capture so a step can be added to produce it.",+)++_GENERIC_USER_REASON = (+ "I couldn't apply the edited extraction schema because some of its fields don't match "+ "any value this workflow currently produces. Tell me what they should map to and I'll try again. "+ "Your current draft is unchanged."+)+++class SchemaIncompatibility(BaseModel):+ model_config = ConfigDict(frozen=True)++ block_label: str+ incompatible_paths: tuple[str, ...]+ known_output_paths: tuple[str, ...]+ edited_schema_summary: str = ""+ preserves_workflow_draft: bool = True+ next_actions: tuple[str, ...] = Field(default=_DEFAULT_NEXT_ACTIONS)++ def to_summary_dict(self) -> dict[str, Any]:+ return {+ "block_label": self.block_label,+ "incompatible_paths": list(self.incompatible_paths),+ "known_output_paths": list(self.known_output_paths),+ "edited_schema_summary": self.edited_schema_summary,+ "preserves_workflow_draft": self.preserves_workflow_draft,+ "next_actions": list(self.next_actions),+ }+++def merge_schema_incompatibilities(items: list[SchemaIncompatibility]) -> SchemaIncompatibility | None:+ """Fold per-block incompatibilities into a single terminal record. The merged+ paths are de-duplicated and ordered; the first block label anchors the record."""+ real = [item for item in items if item is not None]+ if not real:+ return None+ if len(real) == 1:+ return real[0]+ incompatible: list[str] = []+ known: list[str] = []+ summaries: list[str] = []+ for item in real:+ for path in item.incompatible_paths:+ if path not in incompatible:+ incompatible.append(path)+ for path in item.known_output_paths:+ if path not in known:+ known.append(path)+ if item.edited_schema_summary and item.edited_schema_summary not in summaries:+ summaries.append(item.edited_schema_summary)+ return SchemaIncompatibility(+ block_label=real[0].block_label,+ incompatible_paths=tuple(incompatible),+ known_output_paths=tuple(known),+ edited_schema_summary="; ".join(summaries),+ preserves_workflow_draft=all(item.preserves_workflow_draft for item in real),+ )+++def _field_list_phrase(paths: tuple[str, ...]) -> str:+ quoted = [f"`{path}`" for path in paths]+ if len(quoted) == 1:+ return quoted[0]+ if len(quoted) == 2:+ return f"{quoted[0]} and {quoted[1]}"+ return ", ".join(quoted[:-1]) + f", and {quoted[-1]}"+++def render_schema_incompatibility_user_reason(incompat: SchemaIncompatibility) -> str:+ """Product-language reply rendered from the structured incompatibility. Falls back+ to a field-free message if an exotic field name trips the user-facing safety gate."""+ fields = _field_list_phrase(incompat.incompatible_paths)+ verb = "doesn't" if len(incompat.incompatible_paths) == 1 else "don't"+ sentences = [+ f"I couldn't apply the edited extraction schema: the field {fields} {verb} match any value this workflow currently produces."+ ]+ if incompat.known_output_paths:+ outputs = ", ".join(incompat.known_output_paths)+ sentences.append(f"This workflow's data currently covers {outputs}.")+ sentences.append(+ "Tell me which existing output it should map to, or remove it, and I'll try again. Your current draft is unchanged."+ )+ candidate = " ".join(sentences)+ try:+ assert_clean_user_facing_text(candidate, blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL)+ except ValueError:+ return _GENERIC_USER_REASON+ return candidate+++def render_schema_incompatibility_agent_steer(incompat: SchemaIncompatibility) -> str:+ incompatible = ", ".join(incompat.incompatible_paths) or "(unknown)"+ known = ", ".join(incompat.known_output_paths) or "(none recorded)"+ return (+ f"STOP: the edited extraction_schema declares field(s) [{incompatible}] that map to no output block "+ f"`{incompat.block_label}` produces [{known}]. This is not repairable by re-authoring the same draft. "+ "Report the mismatch to the user and ask which existing output the field should map to. "+ "The prior draft is preserved; do not rerun the blocks."+ )+++def build_schema_incompatibility_blocker_signal(incompat: SchemaIncompatibility) -> CopilotToolBlockerSignal:+ return CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text=render_schema_incompatibility_agent_steer(incompat),+ user_facing_reason=render_schema_incompatibility_user_reason(incompat),+ recovery_hint="report_blocker_to_user",+ cleared_by_tools=frozenset(),+ preserves_workflow_draft=incompat.preserves_workflow_draft,+ renders_final_reply=True,+ internal_reason_code=SCHEMA_INCOMPATIBILITY_REASON_CODE,+ blocked_tool=SCHEMA_INCOMPATIBILITY_BLOCKED_TOOL,+ extra={"schema_incompatibility": incompat.to_summary_dict()},+ )diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 47dbbc7aa..2ef497508 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -7,7 +7,7 @@ import keywordimport reimport textwrapimport tokenize-from collections.abc import Iterator, Mapping+from collections.abc import Callable, Iterator, Mappingfrom dataclasses import dataclassfrom dataclasses import field as dataclass_fieldfrom typing import Annotated, Any, Literal, NamedTuple@@ -78,9 +78,19 @@ from skyvern.forge.sdk.copilot.reached_download_target import (code_is_download_intent,)from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteraction+from skyvern.forge.sdk.copilot.schema_incompatibility import (+ SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,+ SchemaIncompatibility,+ build_schema_incompatibility_blocker_signal,+ merge_schema_incompatibilities,+ render_schema_incompatibility_agent_steer,+)from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_endfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span-from skyvern.forge.sdk.copilot.turn_halt import blocker_signal_is_genuinely_terminal+from skyvern.forge.sdk.copilot.turn_halt import (+ blocker_signal_is_genuinely_terminal,+ stash_turn_halt_from_blocker_signal,+)from skyvern.forge.sdk.copilot.workflow_credential_utils import credential_params, parse_workflow_yaml, workflow_blocksfrom skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected@@ -360,6 +370,7 @@ class CodeArtifactNormalization(NamedTuple):error: str | Noneviolations: list[str]offending_labels: list[str]+ schema_incompatibilities: list[SchemaIncompatibility] = []def _normalize_code_artifact_metadata(@@ -399,6 +410,7 @@ def _normalize_code_artifact_metadata_detailed(trajectory: list[ScoutedInteraction] = scout_trajectory or []violations: list[str] = []offending_labels: list[str] = []+ schema_incompatibilities: list[SchemaIncompatibility] = []anchored: list[dict[str, Any]] = []unanchored: list[dict[str, Any]] = []seen_labels: set[str] = set()@@ -494,6 +506,10 @@ def _normalize_code_artifact_metadata_detailed(schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)if schema_conformance_error is not None:item_violations.append(schema_conformance_error)+ schema_incompatibility = _extraction_schema_incompatibility(label, dumped, block_code)+ if schema_incompatibility is not None:+ schema_incompatibilities.append(schema_incompatibility)+ item_violations.append(render_schema_incompatibility_agent_steer(schema_incompatibility))download_shape_error = _download_return_shape_error(label, dumped, block_code)if download_shape_error is not None:item_violations.append(download_shape_error)@@ -507,9 +523,13 @@ def _normalize_code_artifact_metadata_detailed(normalized[label] = _imposed_artifact_skeleton(label, code_blocks[label], trajectory)if violations:return CodeArtifactNormalization(- normalized, _format_code_artifact_violations(violations), violations, offending_labels+ normalized,+ _format_code_artifact_violations(violations),+ violations,+ offending_labels,+ schema_incompatibilities,)- return CodeArtifactNormalization(normalized, None, [], [])+ return CodeArtifactNormalization(normalized, None, [], [], [])def _artifact_label_fragment(label: str) -> str:@@ -3107,6 +3127,81 @@ def _extraction_schema_conformance_error(label: str, artifact: Mapping[str, Any]return None+def _top_level_path_segment(path: str) -> str:+ head = path.strip()+ for separator in (".", "[", "/"):+ index = head.find(separator)+ if index > 0:+ head = head[:index]+ return head.strip()+++def _known_output_contract_paths(artifact: Mapping[str, Any], code: str) -> set[str]:+ """Top-level field names the block is known to produce: the snippet's return-dict+ keys plus the confirmed `goal_value_paths`' top-level segments. Empty when neither+ is statically determinable, so the incompatibility check stays tolerant."""+ paths: set[str] = set()+ return_keys = _top_level_return_dict_keys(code)+ if return_keys:+ paths |= return_keys+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):+ for row in _artifact_rows(artifact.get(field_name)):+ for path in _artifact_goal_value_paths(row.get("goal_value_paths")):+ segment = _top_level_path_segment(path)+ if segment:+ paths.add(segment)+ return paths+++def _schema_property_summary(schema: Mapping[str, Any]) -> str:+ names = _schema_object_property_names(schema)+ if names is None:+ return ""+ property_names, _required = names+ return ", ".join(sorted(property_names))+++def _extraction_schema_incompatibility(+ label: str, artifact: Mapping[str, Any], code: str+) -> SchemaIncompatibility | None:+ """Detect an edited `extraction_schema` whose object property names overlap NONE of+ the block's known output contract. Unlike `_extraction_schema_conformance_error`,+ this fires even when `required` is empty: an optional-only field that maps to nothing+ the block produces is a non-repairable mismatch, not a tolerated gap. Stays tolerant+ when the contract or property names are not statically determinable."""+ if _is_download_intent(artifact, code) or not code.strip():+ return None+ known_paths = _known_output_contract_paths(artifact, code)+ if not known_paths:+ return None+ incompatible: set[str] = set()+ summaries: list[str] = []+ for value in _artifact_extraction_schema_values(artifact):+ if isinstance(value, str) and _is_unfilled_artifact_placeholder(value.strip()):+ continue+ parsed = _parse_extraction_schema(value)+ if parsed is None or not validate_schema(parsed):+ continue+ names = _schema_object_property_names(parsed)+ if names is None:+ continue+ property_names, _required = names+ if not property_names or property_names & known_paths:+ continue+ incompatible |= property_names+ summary = _schema_property_summary(parsed)+ if summary and summary not in summaries:+ summaries.append(summary)+ if not incompatible:+ return None+ return SchemaIncompatibility(+ block_label=label,+ incompatible_paths=tuple(sorted(incompatible)),+ known_output_paths=tuple(sorted(known_paths)),+ edited_schema_summary="; ".join(summaries),+ )++_EXPECT_DOWNLOAD_ATTR = "expect_download"@@ -3507,6 +3602,29 @@ def _credentialed_code_block_scout_gate_errors(return errors+def _reject_schema_incompatibility(+ ctx: AgentContext,+ incompatibility: SchemaIncompatibility,+ reject: Callable[..., dict[str, Any]],+) -> dict[str, Any]:+ """Emit the typed, non-repairable schema-incompatibility outcome: stash the terminal+ blocker + turn halt so enforcement renders a product-language reply (instead of+ falling through to repair churn), record it on the context, and reject without+ persisting the incompatible draft."""+ signal = build_schema_incompatibility_blocker_signal(incompatibility)+ stash_blocker_signal(ctx, signal)+ stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")+ ctx.latest_schema_incompatibility = incompatibility+ return reject(+ error=render_schema_incompatibility_agent_steer(incompatibility),+ user_facing_summary=signal.user_facing_reason,+ data={+ "failure_type": SCHEMA_INCOMPATIBILITY_FAILURE_TYPE,+ "schema_incompatibility": incompatibility.to_summary_dict(),+ },+ )++async def _update_workflow(params: dict[str, Any],ctx: AgentContext,@@ -3600,6 +3718,10 @@ async def _update_workflow(code_artifact_metadata_error = normalization.errorif code_artifact_metadata_error is not None:record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)+ if normalization.schema_incompatibilities:+ incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)+ if incompatibility is not None:+ return _reject_schema_incompatibility(ctx, incompatibility, reject)prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)code_authoring_repair_context = (diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex 2267989b1..7a712d79d 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_datafrom skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE+from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILITY_REASON_CODELOG = structlog.get_logger()@@ -26,6 +27,7 @@ class TurnHaltKind(StrEnum):ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"PROBABLE_SITE_BLOCK = "probable_site_block"REPAIR_CEILING_REACHED = "repair_ceiling_reached"+ SCHEMA_INCOMPATIBILITY = "schema_incompatibility"class TurnHaltVerdict(StrEnum):@@ -53,12 +55,16 @@ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(})_PROBABLE_SITE_BLOCK_REASON_CODES = frozenset({"probable_site_block_stop"})+_SCHEMA_INCOMPATIBILITY_REASON_CODES = frozenset({SCHEMA_INCOMPATIBILITY_REASON_CODE})# A held blocker whose reason code is in this set must win both the rendered# reply and the typed halt kind over a later non-terminal trip (e.g. the# code-authoring churn backstop), which defers entirely when one is present.GENUINELY_TERMINAL_BLOCKER_REASON_CODES = (- _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES | _PROBABLE_SITE_BLOCK_REASON_CODES | frozenset({"repair_ceiling_reached"})+ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES+ | _PROBABLE_SITE_BLOCK_REASON_CODES+ | _SCHEMA_INCOMPATIBILITY_REASON_CODES+ | frozenset({"repair_ceiling_reached"}))@@ -74,10 +80,14 @@ _INVOLUNTARY_TURN_HALT_KINDS = frozenset(TurnHaltKind.LOOP_DETECTED,TurnHaltKind.PROBABLE_SITE_BLOCK,TurnHaltKind.REPAIR_CEILING_REACHED,+ TurnHaltKind.SCHEMA_INCOMPATIBILITY,})_INVOLUNTARY_BLOCKER_REASON_CODES = (- _LOOP_TERMINAL_REASON_CODES | _PROBABLE_SITE_BLOCK_REASON_CODES | frozenset({REPAIR_CEILING_REASON_CODE})+ _LOOP_TERMINAL_REASON_CODES+ | _PROBABLE_SITE_BLOCK_REASON_CODES+ | _SCHEMA_INCOMPATIBILITY_REASON_CODES+ | frozenset({REPAIR_CEILING_REASON_CODE}))@@ -105,6 +115,8 @@ def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind |return TurnHaltKind.ACTIVE_TERMINAL_CHALLENGEif reason in _PROBABLE_SITE_BLOCK_REASON_CODES:return TurnHaltKind.PROBABLE_SITE_BLOCK+ if reason in _SCHEMA_INCOMPATIBILITY_REASON_CODES:+ return TurnHaltKind.SCHEMA_INCOMPATIBILITYreturn Nonediff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.pyindex 42ad6555d..3452bbf01 100644--- a/skyvern/forge/sdk/copilot/turn_outcome.py+++ b/skyvern/forge/sdk/copilot/turn_outcome.py@@ -167,7 +167,16 @@ def _string_or_none(value: object) -> str | None:return value if isinstance(value, str) and value else None-def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | None]:+def _schema_incompatibility_summary(ctx: Any) -> dict[str, Any] | None:+ incompatibility = getattr(ctx, "latest_schema_incompatibility", None)+ to_summary = getattr(incompatibility, "to_summary_dict", None)+ if not callable(to_summary):+ return None+ summary = to_summary()+ return summary if isinstance(summary, dict) else None+++def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, Any]:turn_halt_kind = getattr(getattr(ctx, "turn_halt", None), "kind", None)turn_halt_kind_value = getattr(turn_halt_kind, "value", turn_halt_kind)pending_capability = _string_or_none(getattr(ctx, "code_native_pending_capability", None))@@ -177,6 +186,7 @@ def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | Non),"copilot_repair_ceiling_hit": turn_halt_kind_value == "repair_ceiling_reached","copilot_pending_capability": pending_capability,+ "copilot_schema_incompatibility": _schema_incompatibility_summary(ctx),}diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.pyindex b52dc73ac..23b55a413 100644--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py@@ -39,3 +39,8 @@ class TurnOutcome(BaseModel):copilot_repair_ceiling_hit: bool = Falsecopilot_pending_capability: str | None = Nonecopilot_turn_id: str | None = None+ # Structured summary of an edited-schema-incompatibility terminal outcome+ # (incompatible_paths, known_output_paths, next_actions, ...), persisted so a+ # later "what was the problem?" turn can report it. None unless the turn ended+ # on a schema-incompatibility halt.+ copilot_schema_incompatibility: dict[str, Any] | None = Nonediff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 4f6b5401f..b97524b06 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -1249,6 +1249,7 @@ class TestRequestPolicyInputGuardrail:organization_id="org-1",handler=policy_inputs.handler,active_criteria=None,+ config=None,)@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_code_mode_opt_out.py b/tests/unit/test_copilot_code_mode_opt_out.pyindex 8fbac65e5..af21e77d8 100644--- a/tests/unit/test_copilot_code_mode_opt_out.py+++ b/tests/unit/test_copilot_code_mode_opt_out.py@@ -262,4 +262,5 @@ def test_derive_copilot_code_mode_diagnostics_uses_context_state() -> None:"copilot_last_code_build_failed": True,"copilot_repair_ceiling_hit": True,"copilot_pending_capability": "credential-typed code synthesis",+ "copilot_schema_incompatibility": None,}diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex 0291595c0..12e142e47 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -4,6 +4,7 @@ of the judge-unavailable success bypass."""from __future__ import annotations+import asynciofrom unittest.mock import MagicMockfrom skyvern.forge.sdk.copilot.agent import (@@ -28,7 +29,9 @@ from skyvern.forge.sdk.copilot.completion_verification import (CriterionVerdict,RunEvidenceSnapshot,_coerce_result,+ _render_criteria,combine_verification_results,+ evaluate_completion_criteria,grade_definition_criteria,grade_present_value_criteria,run_plane_all_no_evidence,@@ -59,8 +62,8 @@ from skyvern.forge.sdk.copilot.tools.completion import ()-def _criterion(cid: str, outcome: str, *, level: str = "run") -> CompletionCriterion:- return CompletionCriterion(id=cid, outcome=outcome, level=level) # type: ignore[arg-type]+def _criterion(cid: str, outcome: str, *, level: str = "run", output_path: str | None = None) -> CompletionCriterion:+ return CompletionCriterion(id=cid, outcome=outcome, level=level, output_path=output_path) # type: ignore[arg-type]def _stored(@@ -154,6 +157,63 @@ def test_reconcile_non_subset_supersedes_wholesale() -> None:assert decision.superseded_set_id == "wccs_old"+def test_reconcile_keeps_stored_requested_output_when_fresh_is_generic_rephrase() -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_existing",+ goal_epoch=1,+ criteria=(+ _criterion("c0", "The profile details are captured."),+ _criterion("c1", "The returned record includes ID.", output_path="output.id"),+ ),+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ [_criterion("c0", "The full profile information is extracted.")],+ actionable=True,+ )++ assert decision.action == "adopt_stored"+ assert decision.reason == "kept"+ assert decision.criteria == stored.criteria+++def test_reconcile_keeps_grounded_stored_requested_output_when_fresh_rephrases_path() -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_existing",+ goal_epoch=1,+ criteria=(_criterion("c0", "The returned record includes ID.", output_path="output.id"),),+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ [_criterion("c1", "The final response captures the ID.", output_path="output.id")],+ actionable=True,+ )++ assert decision.action == "adopt_stored"+ assert decision.reason == "kept"+ assert decision.criteria == stored.criteria+++def test_reconcile_keeps_grounded_stored_requested_output_when_fresh_id_is_ungrounded() -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_existing",+ goal_epoch=1,+ criteria=(_criterion("c0", "The returned record includes ID.", output_path="output.id"),),+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ [_criterion("c1", "The returned record includes ID.")],+ actionable=True,+ )++ assert decision.action == "adopt_stored"+ assert decision.reason == "kept"+ assert decision.criteria == stored.criteria++def test_reconcile_epoch_stays_monotonic_after_supersede_chain() -> None:# A superseded latest row still advances next_epoch, so a new set never# reuses an epoch number.@@ -189,7 +249,12 @@ def test_reconcile_no_criteria_anywhere_is_noop() -> None:def test_criteria_json_round_trip_preserves_level_and_flags() -> None:criteria = (- CompletionCriterion(id="c0", outcome="the item is in the cart", implicit=True),+ CompletionCriterion(+ id="c0",+ outcome="the returned record includes ID",+ implicit=True,+ output_path="output.id",+ ),CompletionCriterion(id="c1", outcome="inputs are reusable", method_mandated=True, level="definition"),)assert criteria_from_json(criteria_to_json(criteria)) == criteria@@ -308,25 +373,97 @@ def test_coerce_result_tristate_mapping() -> None:assert result.verdict_state_counts() == {"satisfied": 1, "unsatisfied": 3, "unknown": 2}+def test_explicit_output_path_criterion_prevents_full_verification_when_absent() -> None:+ id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0", id_criterion.id],+ verdicts=[+ _verdict("c0", "satisfied", "evidence_confirms"),+ _verdict(id_criterion.id, "unsatisfied", "no_evidence"),+ ],+ )++ assert id_criterion.output_path == "output.id"+ assert id_criterion.outcome == "The returned record includes ID."+ assert result.is_fully_satisfied() is False+++def test_completion_verifier_render_includes_required_output_path() -> None:+ rendered = _render_criteria([_criterion("c0", "The returned record includes ID.", output_path="output.id")])++ assert "required_output_path=output.id" in rendered+++def test_required_output_path_missing_is_not_satisfied_by_evidence_text() -> None:+ id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")+ criteria = [+ _criterion(+ "c0",+ "The returned record includes status.",+ output_path="output.status",+ ),+ id_criterion,+ ]+ snapshot = RunEvidenceSnapshot(+ workflow_run_id="wr_test",+ block_outputs={+ "extract_profile": {+ "output": {+ "id": None,+ "evidence_text": "Search result text mentions ID 1457803926.",+ "status": "active",+ }+ }+ },+ )++ async def _handler(**kwargs: object) -> dict:+ prompt = str(kwargs["prompt"])+ assert "required_output_path=output.id" in prompt+ assert '"id": null' in prompt+ assert "evidence_text" in prompt+ assert "1457803926" in prompt+ return {+ "verdicts": [+ {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},+ {+ "criterion_id": "c1",+ "satisfied": False,+ "reason_code": "no_evidence",+ "missing_evidence": "output.id is missing/null.",+ },+ ]+ }++ result = asyncio.run(evaluate_completion_criteria(criteria, snapshot, _handler))++ assert id_criterion.output_path == "output.id"+ assert grade_present_value_criteria([id_criterion], snapshot) == []+ assert result.verdicts[1].criterion_id == "c1"+ assert result.verdicts[1].state == "unsatisfied"+ assert result.is_fully_satisfied() is False++_PARAMETERIZED_YAML = """\-title: provider lookup+title: account lookupworkflow_definition:parameters:- parameter_type: workflowkey: first_name- parameter_type: workflow- key: npi_number+ key: account_idblocks:- block_type: goto_urllabel: open_portal- url: https://registry.example.com/search+ url: https://example.com/search- block_type: navigationlabel: fill_search- navigation_goal: "Search for {{ first_name }} with NPI {{ npi_number }}"+ navigation_goal: "Search for {{ first_name }} with account ID {{ account_id }}""""_UNREFERENCED_YAML = """\-title: provider lookup+title: account lookupworkflow_definition:parameters:- parameter_type: workflow@@ -334,29 +471,33 @@ workflow_definition:blocks:- block_type: goto_urllabel: open_portal- url: https://registry.example.com/search+ url: https://example.com/search"""_NO_PARAMS_YAML = """\-title: provider lookup+title: account lookupworkflow_definition:parameters: []blocks:- block_type: goto_urllabel: open_portal- url: https://registry.example.com/search+ url: https://example.com/search"""def test_definition_grading_satisfied_when_parameters_exist_and_are_referenced() -> None:- criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")]+ criteria = [+ _criterion("c0", "the workflow accepts first name and account ID as reusable inputs", level="definition")+ ](verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)assert verdict.state == "satisfied"assert verdict.reason_code == "definition_parameters_referenced"def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced() -> None:- criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")]+ criteria = [+ _criterion("c0", "the workflow accepts first name and account ID as reusable inputs", level="definition")+ ](missing,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)assert missing.state == "unsatisfied"assert missing.reason_code == "definition_parameters_missing"@@ -379,19 +520,19 @@ def test_definition_grading_requires_every_named_input_to_match() -> None:criteria = [_criterion("c0",- "the workflow accepts first name, last name, and NPI number as reusable inputs",+ "the workflow accepts first name, last name, and account ID as reusable inputs",level="definition",)]- # _PARAMETERIZED_YAML defines first_name + npi_number but not last_name:+ # _PARAMETERIZED_YAML defines first_name + account_id but not last_name:# a multi-input ask must not be satisfied by a partial parameter set.(verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)assert verdict.state == "unknown"assert verdict.reason_code == "definition_parameters_unmatched"full_yaml = _PARAMETERIZED_YAML.replace(- " - parameter_type: workflow\n key: npi_number",- " - parameter_type: workflow\n key: last_name\n - parameter_type: workflow\n key: npi_number",+ " - parameter_type: workflow\n key: account_id",+ " - parameter_type: workflow\n key: last_name\n - parameter_type: workflow\n key: account_id",)(full,) = grade_definition_criteria(criteria, full_yaml)assert full.state == "satisfied"diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 03d024319..4759b72b2 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -234,6 +234,34 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changesassert getattr(ctx, "blocker_signal", None) is None+def test_schema_incompatibility_failure_type_stops_without_repair() -> None:+ # SKY-11380: the typed schema-incompatibility reject must route to STOP, never repair,+ # so the agent reports the mismatch instead of churning toward repair_ceiling_reached.+ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": False,+ "error": "STOP: the edited extraction_schema declares field(s) [shoebox] that map to no output.",+ "data": {+ "failure_type": "schema_incompatibility",+ "workflow_updated": False,+ "schema_incompatibility": {+ "block_label": "capture_row",+ "incompatible_paths": ["shoebox"],+ "known_output_paths": ["order_date", "order_total"],+ },+ },+ },+ ctx=_ctx(),+ workflow_updated=False,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SCHEMA_INCOMPATIBILITY+ assert contract.repair_decision.next_action == RepairNextAction.STOP+ assert contract.repair_decision.next_action != RepairNextAction.REPAIR+ assert contract.repair_decision.target_blocks == []++def test_judge_confirmed_suspicious_success_forces_no_change() -> None:ctx = _ctx()ctx.last_test_suspicious_success = Truediff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pynew file mode 100644index 000000000..a1c927302--- /dev/null+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -0,0 +1,404 @@+from __future__ import annotations++import hashlib+import json+from typing import Any++import pytest++from skyvern.forge.sdk.copilot.completion_criteria_store import (+ StoredCriteriaSet,+ StoredCriteriaSnapshot,+ criteria_to_json,+ reconcile_completion_criteria,+)+from skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.request_policy import (+ CompletionCriterion,+ RequestPolicy,+ _apply_requested_output_completion_criteria,+ _classify_request,+)+++async def _policy_for_message(+ user_message: str,+ criteria: list[dict[str, Any]],+ *,+ config: CopilotConfig | None = None,+) -> RequestPolicy:+ async def _handler(**_: Any) -> dict[str, Any]:+ return {+ "testing_intent": "require_test",+ "credential_input_kind": "none",+ "requires_user_clarification": False,+ "completion_criteria": criteria,+ }++ return await _classify_request(+ user_message,+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=_handler,+ config=config,+ )+++def _stored(*criteria: CompletionCriterion) -> StoredCriteriaSet:+ return StoredCriteriaSet(set_id="wccs_existing", goal_epoch=1, criteria=tuple(criteria))+++def _criterion(+ cid: str,+ outcome: str,+ *,+ level: str = "run",+ output_path: str | None = None,+ method_mandated: bool = False,+) -> CompletionCriterion:+ return CompletionCriterion(+ id=cid,+ outcome=outcome,+ level=level, # type: ignore[arg-type]+ output_path=output_path,+ method_mandated=method_mandated,+ )+++def _outcomes(policy: RequestPolicy) -> list[str]:+ return [criterion.outcome for criterion in policy.completion_criteria]+++def _criteria_for_path(policy: RequestPolicy, output_path: str) -> list[CompletionCriterion]:+ return [criterion for criterion in policy.completion_criteria if criterion.output_path == output_path]+++def _criteria_fingerprint(criteria: list[CompletionCriterion]) -> str:+ payload = json.dumps(criteria_to_json(criteria), sort_keys=True, separators=(",", ":"))+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()+++def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[str]) -> list[CompletionCriterion]:+ return [+ criterion+ for criterion in policy.completion_criteria+ if criterion.level == "run"+ and not criterion.method_mandated+ and criterion.output_path in requested_output_paths+ ]+++async def test_classifier_output_is_augmented_with_generic_requested_outputs() -> None:+ policy = await _policy_for_message(+ "Build a registry lookup. Return a final result record with customer name, record id, and status.",+ [{"outcome": "The profile details are captured."}],+ )++ rendered = "\n".join(_outcomes(policy))+ assert "customer name" in rendered+ assert "record id" in rendered+ assert "status" in rendered+ assert _criteria_for_path(policy, "output.customer_name")+ assert _criteria_for_path(policy, "output.record_id")+ assert _criteria_for_path(policy, "output.status")+++async def test_lowercase_record_id_is_augmented_as_requested_output() -> None:+ policy = await _policy_for_message(+ "Return a final record with record id.",+ [],+ )++ assert _outcomes(policy) == ["The returned record includes record id."]+ assert policy.completion_criteria[0].output_path == "output.record_id"+++async def test_leading_output_verb_does_not_enter_requested_output_slug() -> None:+ policy = await _policy_for_message(+ "Return a final record. Capture the identifier.",+ [],+ )++ assert _outcomes(policy) == ["The returned record includes identifier."]+ assert policy.completion_criteria[0].output_path == "output.identifier"+++async def test_possessive_requested_output_slug_is_canonicalized() -> None:+ policy = await _policy_for_message(+ "Return a final record with each location's status.",+ [],+ )++ assert _outcomes(policy) == ["The returned record includes location status."]+ assert policy.completion_criteria[0].output_path == "output.location_status"+++async def test_combined_classifier_output_splits_requested_output_grounding() -> None:+ policy = await _policy_for_message(+ "Return a final record with record id and status.",+ [{"outcome": "The returned record includes record id and status."}],+ )++ assert _criteria_for_path(policy, "output.record_id")+ assert _criteria_for_path(policy, "output.status")+++async def test_schema_derived_output_path_precedes_config_alias_and_slug_fallback() -> None:+ policy = await _policy_for_message(+ "Return a final record with tracking number.",+ [+ {+ "outcome": "The returned record includes tracking number.",+ "output_path": "output.shipment.tracking_number",+ }+ ],+ config=CopilotConfig(requested_output_path_aliases={"tracking number": "output.tracking_number"}),+ )++ assert _criteria_for_path(policy, "output.shipment.tracking_number")+ assert not _criteria_for_path(policy, "output.tracking_number")+++async def test_config_alias_maps_generic_requested_output_field() -> None:+ policy = await _policy_for_message(+ "Return a final record with reference number.",+ [],+ config=CopilotConfig(requested_output_path_aliases={"reference number": "output.reference_number"}),+ )++ assert _outcomes(policy) == ["The returned record includes reference number."]+ assert policy.completion_criteria[0].output_path == "output.reference_number"+++async def test_classifier_drift_canonicalizes_to_stable_generic_requested_outputs() -> None:+ prompt = "Return a final record with record id, status, and contact email."+ classifier_shapes = [+ [+ {"outcome": "The complete profile details are visible."},+ {"outcome": "The final evidence text includes the record id.", "output_path": "output.evidence_text"},+ {"outcome": "The final record includes a contact email."},+ {"outcome": "The workflow accepts lookup inputs.", "level": "definition"},+ {+ "outcome": "The workflow runs to its intended end state with the expected output.",+ "method_mandated": True,+ },+ ],+ [+ {"outcome": "The extracted profile contains record id, status, and contact email."},+ {"outcome": "The reusable inputs are defined.", "level": "definition"},+ ],+ [+ {+ "outcome": "Record id is copied from the search result evidence text.",+ "output_path": "output.evidence_text",+ },+ {"outcome": "The returned record includes record id.", "output_path": "output.record_id"},+ {"outcome": "The returned record includes status."},+ {"outcome": "A different unbound narrative run criterion is present."},+ ],+ ]++ policies = [await _policy_for_message(prompt, criteria) for criteria in classifier_shapes]+ requested_output_paths = {+ "output.record_id",+ "output.status",+ "output.contact_email",+ }+ canonical_subsets = [_requested_output_subset(policy, requested_output_paths) for policy in policies]+ canonical_json = [criteria_to_json(subset) for subset in canonical_subsets]+ fingerprints = [_criteria_fingerprint(subset) for subset in canonical_subsets]++ assert canonical_json == [canonical_json[0], canonical_json[0], canonical_json[0]]+ assert fingerprints == [fingerprints[0], fingerprints[0], fingerprints[0]]+ assert {criterion.output_path for criterion in canonical_subsets[0]} == requested_output_paths+ assert len(canonical_subsets[0]) == 3+ assert all(_criteria_for_path(policy, "output.evidence_text") == [] for policy in policies)+ assert any(criterion.level == "definition" for criterion in policies[0].completion_criteria)+ assert any(+ criterion.outcome == "A different unbound narrative run criterion is present."+ for criterion in policies[2].completion_criteria+ )+++async def test_method_and_setup_text_do_not_become_completion_criteria() -> None:+ policy = await _policy_for_message(+ "Build a lookup. Open Show Details, click Search, choose a plan, set the location, then output profile details.",+ [],+ )++ rendered = "\n".join(_outcomes(policy)).lower()+ assert "show details" not in rendered+ assert "click search" not in rendered+ assert "choose a plan" not in rendered+ assert "location" not in rendered+ assert policy.completion_criteria == []+++async def test_reusable_input_id_does_not_cover_requested_output_id() -> None:+ policy = await _policy_for_message(+ "Accept record id as a reusable input, search by that value, and return a final record with record id.",+ [{"outcome": "The workflow accepts record id as a reusable input.", "level": "definition"}],+ )++ id_criteria = [criterion for criterion in policy.completion_criteria if "record id" in criterion.outcome]+ assert [criterion.level for criterion in id_criteria] == ["definition", "run"]+ assert [criterion.output_path for criterion in id_criteria] == [None, "output.record_id"]+++async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_gate_remains() -> None:+ policy = await _policy_for_message(+ "Return a final record with record id, status, and phone.",+ [+ {"outcome": "The returned record narrative includes record id, status, and phone."},+ {"outcome": "The portal session reaches the submitted results screen."},+ ],+ )++ assert "The returned record narrative includes record id, status, and phone." not in _outcomes(policy)+ assert "The portal session reaches the submitted results screen." in _outcomes(policy)+ assert {+ criterion.output_path+ for criterion in _requested_output_subset(+ policy,+ {"output.record_id", "output.status", "output.phone"},+ )+ } == {"output.record_id", "output.status", "output.phone"}+++async def test_requested_output_coverage_uses_whole_token_sequence_matching() -> None:+ policy = await _policy_for_message(+ "Return a final record with id.",+ [{"outcome": "The returned record includes customer name."}],+ )++ rendered = "\n".join(_outcomes(policy)).lower()+ assert "customer name" in rendered+ assert "includes id" in rendered+ assert _criteria_for_path(policy, "output.id")+++async def test_fresh_augmented_requested_output_supersedes_incomplete_stored_generic_criteria() -> None:+ stored = _stored(_criterion("s0", "The profile details are captured."))+ policy = await _policy_for_message(+ "Return a final result record with customer name and record id.",+ [{"outcome": "The profile details are captured."}],+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ policy.completion_criteria,+ actionable=True,+ )++ assert decision.action == "create"+ assert decision.superseded_set_id == stored.set_id+ assert any(criterion.output_path == "output.record_id" for criterion in decision.criteria)+++def test_stored_complete_requested_output_survives_narrowed_generic_fresh_criteria() -> None:+ stored = _stored(+ _criterion("s0", "The profile details are captured."),+ CompletionCriterion(id="s1", outcome="The returned record includes record id.", output_path="output.record_id"),+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ [_criterion("c0", "The profile details are captured.")],+ actionable=True,+ )++ assert decision.action == "adopt_stored"+ assert decision.criteria == stored.criteria+++async def test_requested_output_criteria_survive_cap_with_existing_and_generic_criteria() -> None:+ criteria = [{"outcome": f"Specific retained outcome {index}."} for index in range(7)]+ criteria.append(+ {+ "outcome": "The workflow runs to its intended end state with the expected output.",+ "method_mandated": True,+ }+ )++ policy = await _policy_for_message("Return a final record with record id.", criteria)++ assert len(policy.completion_criteria) == 8+ rendered = "\n".join(_outcomes(policy))+ assert "record id" in rendered+ assert "Specific retained outcome" in rendered+ assert "intended end state" in rendered+ assert _criteria_for_path(policy, "output.record_id")++ criteria = [{"outcome": f"Specific retained outcome {index}."} for index in range(7)]+ criteria.append({"outcome": "The returned record includes record id."})++ policy = await _policy_for_message("Return a final record with record id and status.", criteria)++ assert len(policy.completion_criteria) == 8+ rendered = "\n".join(_outcomes(policy))+ assert "record id" in rendered+ assert "status" in rendered+ assert "Specific retained outcome" in rendered+ assert _criteria_for_path(policy, "output.record_id")+ assert _criteria_for_path(policy, "output.status")+++def test_requested_output_criteria_survive_cap_when_already_present_before_augmentation() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(id=f"c{index}", outcome=f"Specific retained outcome {index}.") for index in range(7)+ ]+ + [+ CompletionCriterion(id="record_id", outcome="The returned record includes record id."),+ CompletionCriterion(+ id="floor",+ outcome="The workflow runs to its intended end state with the expected output.",+ method_mandated=True,+ ),+ ]+ )++ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")++ assert len(policy.completion_criteria) == 8+ assert [criterion.output_path for criterion in _requested_output_subset(policy, {"output.record_id"})] == [+ "output.record_id"+ ]+ assert any("intended end state" in criterion.outcome for criterion in policy.completion_criteria)+++def test_requested_output_criteria_can_exceed_cap_without_dropping_requested_fields() -> None:+ policy = RequestPolicy()++ _apply_requested_output_completion_criteria(+ policy,+ "Return a final record with name, record id, status, phone, email, license, taxonomy, specialty, and date.",+ )++ assert len(policy.completion_criteria) == 9+ assert {criterion.output_path for criterion in policy.completion_criteria} == {+ "output.name",+ "output.record_id",+ "output.status",+ "output.phone",+ "output.email",+ "output.license",+ "output.taxonomy",+ "output.specialty",+ "output.date",+ }diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 87b8c82e3..a3e12764e 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -371,7 +371,7 @@ class TestActiveCriteriaPromptAnchor:class TestClassifierFallbackCompletionCriteria:@pytest.mark.parametrize(- ("user_message", "expect_criteria"),+ ("user_message", "expected_status", "expected_output_paths"),[((@@ -380,7 +380,8 @@ class TestClassifierFallbackCompletionCriteria:"and the result should come out as a record with the entity name, identifier, items, ""and an overall status."),- True,+ "present",+ {"output.identifier", "output.location_status", "output.name", "output.overall_status"},),((@@ -388,20 +389,29 @@ class TestClassifierFallbackCompletionCriteria:"the entity name, identifier 1234567890, items, per-location status, overall status, and ""no-results behavior."),- True,+ "present",+ {"output.name", "output.identifier", "output.per_location_status", "output.overall_status"},),- ("Extract the user's name, id, address, and status from the portal.", False),- ("Open https://example.com and click the pricing link.", False),+ (+ "Extract the user's name, id, address, and status from the portal.",+ "present",+ {"output.user_name", "output.id", "output.address", "output.status"},+ ),+ ("Open https://example.com and click the pricing link.", "unknown", set()),# Substring matches ("read" in "already", "name" in "filename", "id" in "decided")# must not satisfy the whole-word gate.- ("I already decided the filename and reviewed the status of locations.", False),+ ("I already decided the filename and reviewed the status of locations.", "unknown", set()),# Generic structural group term ("entries") satisfies the group gate.- ("Return a record with the entity name, identifier 1234567890, status, and the grouped entries.", True),+ (+ "Return a record with the entity name, identifier 1234567890, status, and the grouped entries.",+ "present",+ {"output.name", "output.identifier", "output.status"},+ ),],)@pytest.mark.asyncioasync def test_classifier_fallback_structured_record_criteria(- self, user_message: str, expect_criteria: bool+ self, user_message: str, expected_status: str, expected_output_paths: set[str]) -> None:policy = await _classify_request(user_message=user_message,@@ -412,22 +422,21 @@ class TestClassifierFallbackCompletionCriteria:)assert policy.classifier_status == "fallback"- assert policy.completion_contract_status == ("present" if expect_criteria else "unknown")- if not expect_criteria:+ assert policy.completion_contract_status == expected_status+ if expected_status == "unknown":assert policy.completion_criteriaassert all(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)return- assert not any(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)- assert [criterion.id for criterion in policy.completion_criteria] == [- "fallback_record_identity",- "fallback_record_identifier",- "fallback_record_groups",- "fallback_record_status",- ]+ assert {+ criterion.output_path for criterion in policy.completion_criteria if criterion.output_path+ } == expected_output_paths+ assert policy.graded_completion_criteria()assert policy.requires_user_clarification is Falseassert policy.user_response_policy == "proceed"- assert all(criterion.implicit for criterion in policy.completion_criteria)+ assert all(+ criterion.implicit for criterion in policy.completion_criteria if criterion.id.startswith("fallback_")+ )assert all(criterion.level == "run" for criterion in policy.completion_criteria)def test_classifier_fallback_logs_synthesized_structured_record_criteria(diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex f4db87ef7..7b6fa039f 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -27,6 +27,15 @@ from skyvern.forge import appfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.schema_incompatibility import (+ SchemaIncompatibility,+ render_schema_incompatibility_user_reason,+)+from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind+from skyvern.forge.sdk.copilot.turn_outcome import (+ build_minimal_turn_outcome,+ with_copilot_code_mode_diagnostics,+)from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_routefrom skyvern.forge.sdk.routes.workflow_copilot import (COPILOT_V2_FLAG_KEY,@@ -34,7 +43,9 @@ from skyvern.forge.sdk.routes.workflow_copilot import (workflow_copilot_chat_audio,workflow_copilot_chat_post,)+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKindfrom skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatMessage,WorkflowCopilotChatRequest,WorkflowCopilotChatSender,WorkflowCopilotStreamErrorUpdate,@@ -1374,3 +1385,74 @@ async def test_persist_state_keeps_verified_review_tested_proposal(monkeypatch:persisted = calls[0].kwargs["proposed_workflow"]assert persisted is not Noneassert persisted.get("_copilot_unvalidated") is not True+++def _schema_incompatibility_ctx() -> SimpleNamespace:+ incompat = SchemaIncompatibility(+ block_label="capture_row",+ incompatible_paths=("shoebox",),+ known_output_paths=("order_date", "order_total"),+ )+ return SimpleNamespace(+ latest_schema_incompatibility=incompat,+ turn_halt=SimpleNamespace(kind=TurnHaltKind.SCHEMA_INCOMPATIBILITY),+ code_native_pending_capability=None,+ last_test_ok=None,+ last_failed_workflow_yaml=None,+ )+++def test_schema_incompatibility_turn_outcome_is_not_repair_ceiling() -> None:+ # SKY-11380: the schema-incompatibility halt is a distinct typed outcome; it must+ # not masquerade as the repair-ceiling diagnostic on the persisted turn.+ ctx = _schema_incompatibility_ctx()+ reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)+ outcome = with_copilot_code_mode_diagnostics(+ build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),+ ctx,+ )++ assert outcome.copilot_repair_ceiling_hit is False+ assert outcome.copilot_schema_incompatibility is not None+ assert outcome.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]+ assert outcome.copilot_schema_incompatibility["known_output_paths"] == ["order_date", "order_total"]+++def test_schema_incompatibility_persists_and_recalls_for_followup_turn() -> None:+ # The follow-up "what was the problem?" turn reads the prior assistant outcome from+ # chat history; the structured incompatibility survives the round-trip so it can be reported.+ ctx = _schema_incompatibility_ctx()+ reply = render_schema_incompatibility_user_reason(ctx.latest_schema_incompatibility)+ outcome = with_copilot_code_mode_diagnostics(+ build_minimal_turn_outcome(reply, ResponseKind.DIAGNOSE, terminal_reason="turn_halt:schema_incompatibility"),+ ctx,+ )+ now = datetime.now(timezone.utc)+ messages = [+ WorkflowCopilotChatMessage(+ workflow_copilot_chat_message_id="m1",+ workflow_copilot_chat_id="c1",+ sender=WorkflowCopilotChatSender.USER,+ content="add a shoebox field to the extraction",+ created_at=now,+ modified_at=now,+ ),+ WorkflowCopilotChatMessage(+ workflow_copilot_chat_message_id="m2",+ workflow_copilot_chat_id="c1",+ sender=WorkflowCopilotChatSender.AI,+ content=reply,+ turn_outcome=outcome,+ created_at=now,+ modified_at=now,+ ),+ ]++ recalled = workflow_copilot_route._latest_assistant_turn_outcome(messages)++ assert recalled is not None+ assert recalled.terminal_reason == "turn_halt:schema_incompatibility"+ assert recalled.copilot_schema_incompatibility is not None+ assert recalled.copilot_schema_incompatibility["incompatible_paths"] == ["shoebox"]+ # The persisted reply reports the problem in product language.+ assert "shoebox" in messages[1].content