Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11365: Add terminal goal record verification (#6807)
Andrew Neilsone9d2638c45b3
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsxindex 0f289fdd4..fb1dc4844 100644--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx@@ -22,6 +22,7 @@ const tagKeys: Array<TagKey> = [];function renderControl(props: {+ exactValuesOnly?: boolean;value?: Array<TagFilterTerm>;onChange?: (terms: Array<TagFilterTerm>) => void;}) {@@ -32,6 +33,7 @@ function renderControl(props: {value={props.value ?? []}onChange={props.onChange ?? (() => {})}labelSuggestions={["adhoc"]}+ exactValuesOnly={props.exactValuesOnly}/></MemoryRouter>,);@@ -43,37 +45,32 @@ function openAndType(query: string) {fireEvent.change(input, { target: { value: query } });}-// All surfaces (workflows list + analytics dashboard) accept the same three-// term shapes: bare label, group:* (group-any), and exact group:value.-describe("TagFilterControl", () => {- it("offers a bare label as an addable filter", () => {- renderControl({});+describe("TagFilterControl exactValuesOnly", () => {+ it("does not offer a bare label as an addable filter", () => {+ renderControl({ exactValuesOnly: true });openAndType("prod");- expect(screen.getByText(/Filter by label/)).toBeTruthy();+ expect(screen.queryByText(/^Filter by/)).toBeNull();});- it("adds a bare label on selection", () => {- const onChange = vi.fn();- renderControl({ onChange });- openAndType("prod");- fireEvent.click(screen.getByText(/Filter by/));- expect(onChange).toHaveBeenCalledWith([{ key: null, value: "prod" }]);+ it("offers an exact group:value as an addable filter", () => {+ renderControl({ exactValuesOnly: true });+ openAndType("env:prod");+ expect(screen.getByText(/env: prod/)).toBeTruthy();});- it("offers and adds an exact group:value", () => {+ it("adds an exact term on selection", () => {const onChange = vi.fn();- renderControl({ onChange });+ renderControl({ exactValuesOnly: true, onChange });openAndType("env:prod");- expect(screen.getByText(/env: prod/)).toBeTruthy();fireEvent.click(screen.getByText(/Filter by/));expect(onChange).toHaveBeenCalledWith([{ key: "env", value: "prod" }]);});+});- it("adds a group-any term from a group suggestion", () => {- const onChange = vi.fn();- renderControl({ onChange });- openAndType("env");- fireEvent.click(screen.getByText(/: any/));- expect(onChange).toHaveBeenCalledWith([{ key: "env", value: null }]);+describe("TagFilterControl default mode", () => {+ it("still offers a bare label as an addable filter (workflows list parity)", () => {+ renderControl({ exactValuesOnly: false });+ openAndType("prod");+ expect(screen.getByText(/Filter by label/)).toBeTruthy();});});diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsxindex 968bc67d1..74ffc7dc8 100644--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx@@ -51,6 +51,11 @@ type Props = {// to this callback. Omit it for read-only surfaces (e.g. the analytics// dashboard filter) that must not expose destructive tag-key management.onDeleteKey?: (tagKey: TagKey) => void;+ // Restrict filtering to exact `group:value` terms only — no bare labels and+ // no group-any (`key:*`). Use on surfaces whose backend matches values+ // literally (the analytics summary endpoint), where those broader forms+ // would silently return unfiltered/empty data.+ exactValuesOnly?: boolean;// (key, value) -> palette color; only exact group:value chips are colored.// Omit on surfaces that don't load colors (chips stay neutral).colors?: TagColorMap;@@ -83,6 +88,7 @@ function TagFilterControl({valueSuggestionsByKey,triggerLabel = "Tags",onDeleteKey,+ exactValuesOnly = false,colors,}: Props) {const [open, setOpen] = React.useState(false);@@ -137,7 +143,12 @@ function TagFilterControl({const trimmedQuery = query.trim();const normalizedQuery = trimmedQuery.toLowerCase();const candidate = parseTagFilterTerm(query);- const candidateAddable = candidate !== null;+ // In exact mode only fully-specified group:value terms are addable; bare+ // labels and group-any (key:null) are rejected because the backend can't+ // honor them.+ const candidateAddable =+ candidate !== null &&+ (!exactValuesOnly || (candidate.key !== null && candidate.value !== null));const candidateExists =candidate !== null && value.some((term) => sameTerm(term, candidate));const showAdd = candidateAddable && !candidateExists;@@ -155,7 +166,7 @@ function TagFilterControl({.slice(0, MAX_AUTOCOMPLETE_SUGGESTIONS): [];const labelMatches =- typedKey === null+ typedKey === null && !exactValuesOnly? labelSuggestions.filter((label) => label.toLowerCase().includes(normalizedQuery)).filter(@@ -296,7 +307,11 @@ function TagFilterControl({) : null}<Command shouldFilter={false}><CommandInput- placeholder="Filter by label, group:*, or group:label…"+ placeholder={+ exactValuesOnly+ ? "Filter by group:value…"+ : "Filter by label or group:label…"+ }value={query}onValueChange={setQuery}onKeyDown={(event) => {@@ -307,7 +322,11 @@ function TagFilterControl({}}/><CommandList>- <CommandEmpty>Type a label, group:*, or group:label.</CommandEmpty>+ <CommandEmpty>+ {exactValuesOnly+ ? "Type group:value to filter."+ : "Type a label, group:*, or group:label."}+ </CommandEmpty>{showAdd && candidate ? (<CommandGroup><CommandItem@@ -326,13 +345,21 @@ function TagFilterControl({key={tagKey.key}value={`__group__,${tagKey.key}`}className="justify-between gap-2"- // Filter by group (any value in it).- onSelect={() => addTerm({ key: tagKey.key, value: null })}+ // Default: filter by group (any value in it). In exact mode+ // group-any isn't supported, so prefill `key:` and make the+ // user pick a concrete value instead.+ onSelect={() =>+ exactValuesOnly+ ? setQuery(`${tagKey.key}:`)+ : addTerm({ key: tagKey.key, value: null })+ }><div className="flex min-w-0 flex-col"><span className="truncate"><span className="font-medium">{tagKey.key}</span>- <span className="text-muted-foreground">: any</span>+ <span className="text-muted-foreground">+ {exactValuesOnly ? ": …" : ": any"}+ </span></span>{tagKey.description ? (<span className="truncate text-xs text-muted-foreground">diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsxindex 79acb664a..cfb6fcfea 100644--- a/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx@@ -36,30 +36,25 @@ vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({}));import { OnboardingProvider } from "./OnboardingProvider";-import type { OnboardingState, OnboardingStateResponse } from "./types";+import type { OnboardingStateResponse } from "./types";import { useOnboardingState } from "./useOnboardingState";-import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";const DISMISSED_AT = "2026-01-01T00:00:00.000Z";-function freshOnboardingState(): OnboardingState {- return {- tour_completed_at: null,- modal_dismissed_at: null,- first_save_at: null,- first_run_at: null,- ab_variant: null,- user_intent: null,- seen_canvas: null,- seen_node_adder: null,- seen_sidebar: null,- seen_save_run: null,- };-}-function freshResponse(): OnboardingStateResponse {return {- onboarding_state: freshOnboardingState(),+ onboarding_state: {+ tour_completed_at: null,+ modal_dismissed_at: null,+ first_save_at: null,+ first_run_at: null,+ ab_variant: null,+ user_intent: null,+ seen_canvas: null,+ seen_node_adder: null,+ seen_sidebar: null,+ seen_save_run: null,+ },launch_date_at_signup: "2025-01-01T00:00:00Z",};}@@ -125,7 +120,7 @@ describe("OnboardingProvider write resilience", () => {const dismissedResponse: OnboardingStateResponse = {...freshResponse(),onboarding_state: {- ...freshOnboardingState(),+ ...freshResponse().onboarding_state,modal_dismissed_at: DISMISSED_AT,},};@@ -147,25 +142,3 @@ describe("OnboardingProvider write resilience", () => {expect(screen.getByTestId("dismissed").textContent).toBe(DISMISSED_AT);});});--describe("OnboardingProvider missing onboarding_state", () => {- it("renders without throwing when the response omits onboarding_state", async () => {- // A signed-out / transitional /users/me/onboarding body can lack- // onboarding_state; the provider must not dereference it (SKY-11430).- // The key is absent (not explicitly undefined) to mirror the real JSON.- const responseWithoutState: OnboardingStateResponse = {- launch_date_at_signup: null,- };- mockGet.mockResolvedValue({ data: responseWithoutState });-- renderProvider();-- await waitFor(() => expect(mockGet).toHaveBeenCalledTimes(1));- // state resolves to null, so the consumer reads an undefined field rather- // than crashing on a missing onboarding_state.- await waitFor(() =>- expect(screen.getByTestId("dismissed").textContent).toBe("undefined"),- );- expect(OnboardingTelemetry.registerVariant).not.toHaveBeenCalled();- });-});diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsxindex 395cb5e51..69fb16cd6 100644--- a/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx@@ -49,7 +49,7 @@ function OnboardingProvider({ children }: Readonly<Props>) {await queryClient.cancelQueries({ queryKey: QUERY_KEY });const previous =queryClient.getQueryData<OnboardingStateResponse>(QUERY_KEY);- if (previous?.onboarding_state) {+ if (previous) {queryClient.setQueryData<OnboardingStateResponse>(QUERY_KEY, {...previous,onboarding_state: { ...previous.onboarding_state, ...patch },@@ -70,7 +70,7 @@ function OnboardingProvider({ children }: Readonly<Props>) {data?.launch_date_at_signup != null &&new Date(data.launch_date_at_signup) >= new Date(PRODUCT_LAUNCH_DATE);- const abVariant = data?.onboarding_state?.ab_variant ?? null;+ const abVariant = data?.onboarding_state.ab_variant ?? null;useEffect(() => {if (abVariant) {diff --git a/skyvern-frontend/src/store/onboarding/types.ts b/skyvern-frontend/src/store/onboarding/types.tsindex 67798c1f9..11912d185 100644--- a/skyvern-frontend/src/store/onboarding/types.ts+++ b/skyvern-frontend/src/store/onboarding/types.ts@@ -15,7 +15,7 @@ export type OnboardingState = {};export type OnboardingStateResponse = {- onboarding_state?: OnboardingState;+ onboarding_state: OnboardingState;launch_date_at_signup: string | null;};diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex f35d4b0ae..ebd8191be 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -639,6 +639,209 @@ def grade_structured_record_criteria(return best_verdicts+_TERMINAL_ACTION_KEY_TOKENS = (+ (("submitted",), "submission"),+ (("request", "submitted"), "request"),+ (("application", "submitted"), "application"),+ (("form", "submitted"), "form"),+ (("order", "placed"), "order"),+)+_TERMINAL_ARTIFACT_KEY_TOKENS = (+ (("confirmation", "number"), "confirmation"),+ (("confirmation", "id"), "confirmation"),+ (("request", "number"), "request"),+ (("request", "id"), "request"),+ (("submission", "id"), "submission"),+ (("submission", "number"), "submission"),+ (("application", "number"), "application"),+ (("application", "id"), "application"),+ (("order", "number"), "order"),+ (("order", "id"), "order"),+ (("receipt", "number"), "receipt"),+ (("receipt", "id"), "receipt"),+ (("reference", "number"), "reference"),+ (("reference", "id"), "reference"),+)+_TERMINAL_CRITERION_VERBS = frozenset({"submit", "submitted", "submission", "place", "placed"})+_TERMINAL_CRITERION_ABSTAIN_TOKENS = frozenset(+ {+ "lookup",+ "looked",+ "retrieve",+ "retrieved",+ "retrieval",+ "return",+ "returned",+ "status",+ }+)+_NEGATIVE_GUARD_TOKENS = frozenset({"blocker", "error", "failure", "failed", "challenge"})+_NEGATIVE_TERMINAL_STATUS_VALUES = frozenset(+ {+ "blocked",+ "cancelled",+ "canceled",+ "captcha required",+ "denied",+ "error",+ "failed",+ "failure",+ "incomplete",+ "not submitted",+ "timeout",+ "unable",+ }+)+_TERMINAL_RECORD_FAMILY_ARTIFACTS = {+ "request": frozenset({"confirmation", "reference", "request", "submission"}),+ "application": frozenset({"application", "confirmation", "reference", "submission"}),+ "form": frozenset({"confirmation", "reference", "submission"}),+ "order": frozenset({"order", "receipt"}),+}+_TERMINAL_RECORD_FAMILY_ACTIONS = {+ "request": frozenset({"request", "submission"}),+ "application": frozenset({"application", "submission"}),+ "form": frozenset({"form", "submission"}),+ "order": frozenset({"order"}),+}+++def grade_terminal_goal_record_criteria(+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot+) -> list[CriterionVerdict]:+ verdicts: list[CriterionVerdict] = []+ eligible_criteria = [+ (criterion, family)+ for criterion in criteria+ if (family := _terminal_goal_criterion_family(criterion.outcome)) is not None+ ]+ if not eligible_criteria:+ return []+ for label, payload in snapshot.block_outputs.items():+ record = _structured_record_payload(payload)+ if record is None:+ continue+ for criterion, family in eligible_criteria:+ if _terminal_goal_record_confirmed(record, family):+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref=f"block_outputs:{label}",+ )+ )+ if verdicts:+ return verdicts+ return []+++def _terminal_goal_criterion_family(outcome: str) -> str | None:+ if _extract_present_value_literals(outcome):+ return None+ tokens = set(_status_word_tokens(outcome))+ if tokens & _TERMINAL_CRITERION_ABSTAIN_TOKENS:+ return None+ if not tokens & _TERMINAL_CRITERION_VERBS:+ return None+ families = [family for family in ("request", "application", "form", "order") if family in tokens]+ if len(families) != 1:+ return None+ family = families[0]+ if family == "order" and "placed" not in tokens:+ return None+ return family+++def _terminal_goal_record_confirmed(record: dict[str, Any], family: str) -> bool:+ if _terminal_goal_record_has_negative_guard(record):+ return False+ if _structured_record_contradiction(record):+ return False+ action_families = [+ action_family+ for key, value in _walk_record_scalars(record)+ if (action_family := _terminal_action_family_for_key(key)) is not None and value is True+ ]+ if any(+ value is False+ for key, value in _walk_record_scalars(record)+ if _terminal_action_family_for_key(key) is not None+ ):+ return False+ has_family_artifact = _terminal_goal_record_has_artifact_for_family(record, family)+ if not action_families:+ return has_family_artifact and not _terminal_goal_record_has_generic_success_claim(record)+ if family not in _TERMINAL_RECORD_FAMILY_ACTIONS:+ return False+ if not any(action_family in _TERMINAL_RECORD_FAMILY_ACTIONS[family] for action_family in action_families):+ return False+ return has_family_artifact+++def _terminal_action_family_for_key(key: str) -> str | None:+ leaf_tokens = _record_key_leaf_tokens(key)+ for key_tokens, family in _TERMINAL_ACTION_KEY_TOKENS:+ if leaf_tokens == key_tokens:+ return family+ return None+++def _terminal_goal_record_has_generic_success_claim(record: dict[str, Any]) -> bool:+ generic_success_keys = {+ ("completed",),+ ("succeeded",),+ ("success",),+ }+ for key, value in _walk_record_scalars(record):+ leaf_tokens = _record_key_leaf_tokens(key)+ if value is True and _terminal_action_family_for_key(key) is None and leaf_tokens:+ if leaf_tokens[-1] in _TERMINAL_CRITERION_VERBS:+ return True+ if leaf_tokens in generic_success_keys and value is True:+ return True+ if "status" in _key_word_tokens(key) and isinstance(value, str):+ polarity = _status_polarity(value)+ if polarity is not None and not polarity[0]:+ return True+ return False+++def _terminal_goal_record_has_negative_guard(record: dict[str, Any]) -> bool:+ for key, value in _walk_record_scalars(record):+ tokens = _key_word_tokens(key)+ if tokens & _NEGATIVE_GUARD_TOKENS and _is_meaningful_record_value(value) and value is not False:+ return True+ if "status" in tokens and isinstance(value, str):+ normalized_status = _normalize_present_value(value)+ if normalized_status in _NEGATIVE_TERMINAL_STATUS_VALUES or any(+ phrase in normalized_status for phrase in ("captcha required", "not submitted", "unable")+ ):+ return True+ polarity = _status_polarity(value)+ if polarity is not None and polarity[0]:+ return True+ return False+++def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: str) -> bool:+ allowed_artifacts = _TERMINAL_RECORD_FAMILY_ARTIFACTS[family]+ for key, value in _walk_record_scalars(record):+ if isinstance(value, bool) or not _is_meaningful_record_value(value):+ continue+ leaf_tokens = _record_key_leaf_tokens(key)+ for key_tokens, artifact_family in _TERMINAL_ARTIFACT_KEY_TOKENS:+ if leaf_tokens == key_tokens and artifact_family in allowed_artifacts:+ return True+ return False+++def _record_key_leaf_tokens(key: str) -> tuple[str, ...]:+ leaf = re.sub(r"\[\d+\]$", "", key.rsplit(".", 1)[-1])+ spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", leaf).casefold()+ return tuple(re.findall(r"[a-z0-9]+", spaced))++def _structured_record_satisfied(criterion_id: str, label: str) -> CriterionVerdict:return CriterionVerdict(criterion_id=criterion_id,diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 0d15dece4..6903d4878 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_present_value_criteria,grade_record_semantic_consistency,grade_structured_record_criteria,+ grade_terminal_goal_record_criteria,summarize_unsatisfied_outcomes,verdict_missing_evidence,)@@ -437,6 +438,8 @@ def _apply_present_value_upgrades(run_result: CompletionVerificationResult,run_criteria: list[CompletionCriterion],snapshot: RunEvidenceSnapshot,+ *,+ include_terminal_goal_records: bool = False,) -> CompletionVerificationResult:"""Upgrade a ``no_evidence``/``unknown`` run verdict to a deterministic present-value``satisfied``. An ``evidence_contradicts`` verdict is left to the judge, a judge@@ -448,6 +451,10 @@ def _apply_present_value_upgrades(upgrades.update({verdict.criterion_id: verdict for verdict in grade_structured_record_criteria(run_criteria, snapshot)})+ if include_terminal_goal_records:+ upgrades.update(+ {verdict.criterion_id: verdict for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot)}+ )semantic_verdicts = {verdict.criterion_id: verdict for verdict in grade_record_semantic_consistency(run_criteria, snapshot)}@@ -497,6 +504,8 @@ def _deterministic_run_verification_result(deterministic_by_id[verdict.criterion_id] = verdictfor verdict in grade_structured_record_criteria(run_criteria, snapshot):deterministic_by_id[verdict.criterion_id] = verdict+ for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):+ deterministic_by_id[verdict.criterion_id] = verdictfor verdict in semantic_verdicts:deterministic_by_id[verdict.criterion_id] = verdict@@ -581,21 +590,21 @@ async def _maybe_run_completion_verification(],)else:- handler = await _completion_verification_handler(copilot_ctx)- if handler is None:- return CompletionVerificationResult(status="unavailable")- # Too little budget to verify a candidate run: fail closed (unavailable)- # rather than let the run-status proxy claim an unverified outcome as success.- remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)- if (- remaining- <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS- ):- return CompletionVerificationResult(status="unavailable")deterministic_result, remaining_criteria = _deterministic_run_verification_result(run_criteria, snapshot)if deterministic_result is not None and not remaining_criteria:run_result = deterministic_resultelse:+ handler = await _completion_verification_handler(copilot_ctx)+ if handler is None:+ return CompletionVerificationResult(status="unavailable")+ # Too little budget to verify a candidate run: fail closed (unavailable)+ # rather than let the run-status proxy claim an unverified outcome as success.+ remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)+ if (+ remaining+ <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS+ ):+ return CompletionVerificationResult(status="unavailable")judged_result = await evaluate_completion_criteria(remaining_criteria, snapshot, handler)if judged_result.status != "evaluated":run_result = judged_result@@ -609,7 +618,12 @@ async def _maybe_run_completion_verification(criterion_ids=[criterion.id for criterion in run_criteria],verdicts=verdicts,)- run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)+ run_result = _apply_present_value_upgrades(+ run_result,+ run_criteria,+ snapshot,+ include_terminal_goal_records=True,+ )return combine_verification_results(criterion_ids, run_result, definition_verdicts)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex cb451d015..47dbbc7aa 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -965,6 +965,51 @@ def _code_block_authoring_repair_context(return None+def _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml: str) -> str:+ parsed = parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, dict):+ return workflow_yaml+ declared_string_keys = _declared_string_workflow_parameter_keys(parsed)+ if not declared_string_keys:+ return workflow_yaml+ available_binding_keys_by_label = _code_block_available_binding_keys_by_label(workflow_yaml)+ adopted_by_label: dict[str, list[str]] = {}+ for block in _workflow_code_blocks(parsed):+ label = str(block.get("label") or "").strip()+ if not label:+ continue+ code = str(block.get("code") or "")+ if not code.strip():+ continue+ parameter_keys = _code_block_parameter_keys(block)+ available_declared_keys = set(available_binding_keys_by_label.get(label, [])) & declared_string_keys+ if not available_declared_keys:+ continue+ diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)+ if diagnostic is None or not diagnostic.unresolved_names:+ continue+ adopted_keys = [+ name+ for name in diagnostic.unresolved_names+ if name in available_declared_keys and name not in parameter_keys+ ]+ if not adopted_keys:+ continue+ raw_keys = block.get("parameter_keys")+ merged_keys = (+ [str(key) for key in raw_keys if isinstance(key, str) and key] if isinstance(raw_keys, list) else []+ )+ for key in adopted_keys:+ if key not in merged_keys:+ merged_keys.append(key)+ block["parameter_keys"] = merged_keys+ adopted_by_label[label] = adopted_keys+ if not adopted_by_label:+ return workflow_yaml+ LOG.info("copilot adopted exact declared parameter keys for unresolved names", adopted_by_label=adopted_by_label)+ return yaml.safe_dump(parsed, sort_keys=False)++def _repair_context_log_values(values: list[str], *, max_items: int = 20) -> list[str]:cleaned: list[str] = []for raw_value in values[:max_items]:@@ -3523,6 +3568,8 @@ async def _update_workflow(user_facing_summary=_compiled_authoring_user_summary(),data=_code_repair_progress_data(),)+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ workflow_yaml = _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)params["workflow_yaml"] = workflow_yamlparameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)if parameter_contract_error is not None:diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 9a9e718f9..c9fa8344a 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -738,6 +738,64 @@ class TestCodeSafetySeam:assert accepted["ok"] is Trueassert ctx.last_code_authoring_repair_context is None+ @pytest.mark.asyncio+ async def test_code_only_exact_declared_string_parameters_are_adopted_for_unresolved_names(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ submitted = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample}+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: City}+ blocks:+ - block_type: code+ label: search_registry+ code: |+ await page.locator("#query").fill(str(provider_query))+ await page.locator("#location").fill(str(search_location))+ """+ )++ accepted = await _update_workflow({"workflow_yaml": submitted}, ctx)++ assert accepted["ok"] is True+ parsed = parse_workflow_yaml(ctx.workflow_yaml)+ assert isinstance(parsed, dict)+ block = _single_code_block(parsed)+ assert block["parameter_keys"] == ["provider_query", "search_location"]++ @pytest.mark.asyncio+ async def test_code_only_partial_declared_parameter_name_still_rejects(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ submitted = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample}+ blocks:+ - block_type: code+ label: search_registry+ code: |+ await page.locator("#query").fill(str(provider_name))+ """+ )++ rejected = await _update_workflow({"workflow_yaml": submitted}, ctx)++ assert rejected["ok"] is False+ result_context = rejected["data"]["authoring_repair_context"]+ assert result_context["unresolved_names"] == ["provider_name"]+ assert result_context["available_parameter_keys"] == ["provider_query"]+ assert result_context["binding_candidates"] == ["provider_query", "provider_name"]+@pytest.mark.asyncioasync def test_unresolved_name_repair_context_includes_existing_workflow_binding_candidates(self, monkeypatch: pytest.MonkeyPatchdiff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex ab0f8e835..4a0d9a846 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -22,8 +22,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (_structured_record_has_identifier,evaluate_completion_criteria,grade_definition_criteria,+ grade_present_value_criteria,grade_record_semantic_consistency,grade_structured_record_criteria,+ grade_terminal_goal_record_criteria,+ structured_record_has_goal_content,structured_record_has_identity,summarize_unsatisfied_outcomes,)@@ -136,6 +139,21 @@ def _record_payload(**overrides: Any) -> dict[str, Any]:return payload+def _terminal_goal_payload(**overrides: Any) -> dict[str, Any]:+ payload = {+ "submitted": True,+ "blocker": None,+ "confirmation_number": "WTR-1842-DEMO",+ "account_number": "100245",+ "selected_start_date": "2026-06-22",+ "deposit_amount": "$41.00 plus initiation fee",+ "next_owner": "Provider",+ "evidence_text": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",+ }+ payload.update(overrides)+ return payload++def _status_snapshot(status: str,*,@@ -439,6 +457,187 @@ def test_structured_record_criteria_satisfy_structured_record_outputs(block_outpassert _satisfied_criterion_ids(verdicts) == _STRUCTURED_RECORD_CRITERION_IDS+def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})++ verdicts = grade_terminal_goal_record_criteria(+ [_criterion("c0", "a commercial water service request is submitted")], snapshot+ )++ assert verdicts == [+ CriterionVerdict(+ criterion_id="c0",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:submit_water_request",+ )+ ]+++def test_terminal_goal_record_accepts_family_artifact_without_self_asserted_boolean() -> None:+ payload = _terminal_goal_payload(submitted=None)+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})++ verdicts = grade_terminal_goal_record_criteria(+ [_criterion("c0", "a commercial water service request is submitted")], snapshot+ )++ assert _satisfied_criterion_ids(verdicts) == {"c0"}+++ "payload",+ [+ _terminal_goal_payload(confirmation_number=None),+ _terminal_goal_payload(confirmation_number=True),+ _terminal_goal_payload(confirmation_number="", account_number="100245"),+ _terminal_goal_payload(confirmation_number=None, record_number="RN-100245"),+ _terminal_goal_payload(confirmation_number=None, customer_id="cus_123456"),+ ],+)+def test_terminal_goal_record_rejects_ordinary_identifiers(payload: dict[str, Any]) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})++ assert (+ grade_terminal_goal_record_criteria(+ [_criterion("c0", "a commercial water service request is submitted")], snapshot+ )+ == []+ )+++ ("payload", "outcome"),+ [+ (+ _terminal_goal_payload(confirmation_number="WTR-1842-DEMO"),+ "a commercial water service request is submitted",+ ),+ (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "an order is placed"),+ (+ _terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),+ "an application is submitted",+ ),+ (+ _terminal_goal_payload(form_submitted=True, submitted=None, submission_id="SUB-1842"),+ "the form is submitted",+ ),+ (+ _terminal_goal_payload(request_submitted=True, submitted=None, request_id="REQ-1842"),+ "a service request is submitted",+ ),+ ],+)+def test_terminal_goal_record_accepts_narrow_terminal_artifacts(payload: dict[str, Any], outcome: str) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ verdicts = grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot)++ assert _satisfied_criterion_ids(verdicts) == {"c0"}+++ ("payload", "outcome"),+ [+ (_terminal_goal_payload(), "an order is placed"),+ (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "a request is submitted"),+ (+ _terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),+ "an order is placed",+ ),+ ],+)+def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []+++ "payload",+ [+ _terminal_goal_payload(submitted=None, completed=True),+ _terminal_goal_payload(submitted=None, succeeded=True),+ _terminal_goal_payload(submitted=None, success=True),+ _terminal_goal_payload(submitted=None, status="completed"),+ ],+)+def test_terminal_goal_record_rejects_generic_success_synonyms(payload: dict[str, Any]) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []+++ "outcome",+ [+ "the account status is retrieved",+ "the record lookup result is returned",+ "the water request status is shown",+ ],+)+def test_terminal_goal_record_abstains_for_lookup_and_status_criteria(outcome: str) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": _terminal_goal_payload()})++ assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []+++ "payload",+ [+ _terminal_goal_payload(blocker="provider requires a phone call"),+ _terminal_goal_payload(error="submission failed"),+ _terminal_goal_payload(failure_reason="network failure"),+ _terminal_goal_payload(challenge_detected=True),+ _terminal_goal_payload(submitted=False),+ _terminal_goal_payload(status="failed"),+ _terminal_goal_payload(status="denied"),+ _terminal_goal_payload(status="cancelled"),+ _terminal_goal_payload(status="canceled"),+ _terminal_goal_payload(status="incomplete"),+ _terminal_goal_payload(status="timeout"),+ _terminal_goal_payload(status="captcha required"),+ _terminal_goal_payload(status="not submitted"),+ _terminal_goal_payload(status="unable to submit"),+ ],+)+def test_terminal_goal_record_negative_guards_abstain(payload: dict[str, Any]) -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert (+ grade_terminal_goal_record_criteria(+ [_criterion("c0", "a commercial water service request is submitted")], snapshot+ )+ == []+ )++[email protected]("key", ["not_submitted", "previously_submitted"])+def test_terminal_goal_record_rejects_negated_or_temporal_action_keys(key: str) -> None:+ payload = _terminal_goal_payload(submitted=None)+ payload[key] = True+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert (+ grade_terminal_goal_record_criteria(+ [_criterion("c0", "a commercial water service request is submitted")], snapshot+ )+ == []+ )+++def test_terminal_goal_record_does_not_take_literal_criteria_from_present_value() -> None:+ criteria = [_criterion("c0", "the confirmation number WTR-1842-DEMO is reported")]+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": _terminal_goal_payload()})++ assert grade_terminal_goal_record_criteria(criteria, snapshot) == []+ assert _satisfied_criterion_ids(grade_present_value_criteria(criteria, snapshot)) == {"c0"}+++def test_structured_record_goal_content_remains_strict_for_flat_terminal_payload() -> None:+ assert structured_record_has_goal_content(_terminal_goal_payload()) is False++def test_structured_record_partial_matches_do_not_combine_across_blocks() -> None:snapshot = RunEvidenceSnapshot(block_outputs={@@ -842,6 +1041,26 @@ def _structured_record_top_level_output_result() -> dict:}+def _terminal_goal_output_result(**payload_overrides: Any) -> dict:+ return {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_terminal_goal",+ "overall_status": "completed",+ "executed_block_labels": ["submit_water_request"],+ "current_url": "https://example.test/confirmation",+ "blocks": [+ {+ "label": "submit_water_request",+ "block_type": "CODE",+ "status": "completed",+ "extracted_data": _terminal_goal_payload(**payload_overrides),+ }+ ],+ },+ }++def _run_ctx() -> CopilotContext:ctx = CopilotContext(organization_id="o",@@ -1669,6 +1888,46 @@ async def test_page_observation_verification_recognizes_budgeted_outcome(assert "SKU-12345 is present in the cart" in seen_prompt["prompt"]+async def test_page_observation_verification_does_not_apply_terminal_goal_record_upgrade(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {+ "verdicts": [+ {+ "criterion_id": "c0",+ "satisfied": False,+ "reason_code": "no_evidence",+ }+ ]+ }++ _patch_completion_handler(monkeypatch, handler)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ )+ ctx.last_test_ok = False+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"+ ctx.copilot_run_start_monotonic = time.monotonic()+ _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url="https://example.com/confirmation",+ observed_data=_terminal_goal_payload(),+ )++ result = await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url="https://example.com/confirmation",+ observed_data=_terminal_goal_payload(),+ )++ assert result is not None+ assert result.is_fully_satisfied() is False++@pytest.mark.asyncioasync def test_page_observation_verification_does_not_overwrite_satisfied_verdict(monkeypatch: pytest.MonkeyPatch,@@ -2049,6 +2308,97 @@ async def test_maybe_run_completion_verification_treats_fallback_record_as_criteassert handler_lookup_calls == 0+async def test_maybe_run_completion_verification_terminal_goal_bypasses_judge(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler_lookup_calls = 0++ async def fail_handler(**_: object) -> object:+ raise AssertionError("deterministically covered terminal goal must not call the judge")++ async def handler_lookup(_ctx: object) -> object:+ nonlocal handler_lookup_calls+ handler_lookup_calls += 1+ return fail_handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",+ handler_lookup,+ )+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "submit_water_request")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ )++ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"+ assert handler_lookup_calls == 0+++async def test_maybe_run_completion_verification_terminal_goal_without_boolean_bypasses_judge(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("artifact-backed terminal goal must not call the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "submit_water_request")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ )++ verification = await _maybe_run_completion_verification(+ ctx, _terminal_goal_output_result(submitted=None), time.monotonic()+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"+++async def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_judge_unknown(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {+ "verdicts": [+ {+ "criterion_id": "c1",+ "satisfied": True,+ "reason_code": "evidence_confirms",+ "evidence_ref": "block_outputs:submit_water_request",+ }+ ]+ }++ _patch_completion_handler(monkeypatch, handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "submit_water_request")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion("c0", "a commercial water service request is submitted"),+ _criterion("c1", "the selected start date is reported"),+ ]+ )++ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert {verdict.criterion_id: verdict.reason_code for verdict in verification.verdicts} == {+ "c0": "evidence_confirms",+ "c1": "evidence_confirms",+ }++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_fails_closed_on_no_judge_for_judge_needed_criteria(monkeypatch: pytest.MonkeyPatch,