Skip to content

Automate browser based workflows with AI

SKY-11365: Add terminal goal record verification (#6807)

Andrew Neilsone9d2638c45b3
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx
index 0f289fdd4..fb1dc4844 100644
--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.test.tsx
@@ -22,6 +22,7 @@ const tagKeys: Array<TagKey> = [
];
function renderControl(props: {
+ exactValuesOnly?: boolean;
value?: Array<TagFilterTerm>;
onChange?: (terms: Array<TagFilterTerm>) => void;
}) {
@@ -32,6 +33,7 @@ function renderControl(props: {
value={props.value ?? []}
onChange={props.onChange ?? (() => {})}
labelSuggestions={["adhoc"]}
+ exactValuesOnly={props.exactValuesOnly}
/>
</MemoryRouter>,
);
@@ -43,37 +45,32 @@ function openAndType(query: string) {
fireEvent.change(input, { target: { value: query } });
}
-// All surfaces (workflows list + analytics dashboard) accept the same three
-// term shapes: bare label, group:* (group-any), and exact group:value.
-describe("TagFilterControl", () => {
- it("offers a bare label as an addable filter", () => {
- renderControl({});
+describe("TagFilterControl exactValuesOnly", () => {
+ it("does not offer a bare label as an addable filter", () => {
+ renderControl({ exactValuesOnly: true });
openAndType("prod");
- expect(screen.getByText(/Filter by label/)).toBeTruthy();
+ expect(screen.queryByText(/^Filter by/)).toBeNull();
});
- it("adds a bare label on selection", () => {
- const onChange = vi.fn();
- renderControl({ onChange });
- openAndType("prod");
- fireEvent.click(screen.getByText(/Filter by/));
- expect(onChange).toHaveBeenCalledWith([{ key: null, value: "prod" }]);
+ it("offers an exact group:value as an addable filter", () => {
+ renderControl({ exactValuesOnly: true });
+ openAndType("env:prod");
+ expect(screen.getByText(/env: prod/)).toBeTruthy();
});
- it("offers and adds an exact group:value", () => {
+ it("adds an exact term on selection", () => {
const onChange = vi.fn();
- renderControl({ onChange });
+ renderControl({ exactValuesOnly: true, onChange });
openAndType("env:prod");
- expect(screen.getByText(/env: prod/)).toBeTruthy();
fireEvent.click(screen.getByText(/Filter by/));
expect(onChange).toHaveBeenCalledWith([{ key: "env", value: "prod" }]);
});
+});
- it("adds a group-any term from a group suggestion", () => {
- const onChange = vi.fn();
- renderControl({ onChange });
- openAndType("env");
- fireEvent.click(screen.getByText(/: any/));
- expect(onChange).toHaveBeenCalledWith([{ key: "env", value: null }]);
+describe("TagFilterControl default mode", () => {
+ it("still offers a bare label as an addable filter (workflows list parity)", () => {
+ renderControl({ exactValuesOnly: false });
+ openAndType("prod");
+ expect(screen.getByText(/Filter by label/)).toBeTruthy();
});
});
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx
index 968bc67d1..74ffc7dc8 100644
--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagFilterControl.tsx
@@ -51,6 +51,11 @@ type Props = {
// to this callback. Omit it for read-only surfaces (e.g. the analytics
// dashboard filter) that must not expose destructive tag-key management.
onDeleteKey?: (tagKey: TagKey) => void;
+ // Restrict filtering to exact `group:value` terms only — no bare labels and
+ // no group-any (`key:*`). Use on surfaces whose backend matches values
+ // literally (the analytics summary endpoint), where those broader forms
+ // would silently return unfiltered/empty data.
+ exactValuesOnly?: boolean;
// (key, value) -> palette color; only exact group:value chips are colored.
// Omit on surfaces that don't load colors (chips stay neutral).
colors?: TagColorMap;
@@ -83,6 +88,7 @@ function TagFilterControl({
valueSuggestionsByKey,
triggerLabel = "Tags",
onDeleteKey,
+ exactValuesOnly = false,
colors,
}: Props) {
const [open, setOpen] = React.useState(false);
@@ -137,7 +143,12 @@ function TagFilterControl({
const trimmedQuery = query.trim();
const normalizedQuery = trimmedQuery.toLowerCase();
const candidate = parseTagFilterTerm(query);
- const candidateAddable = candidate !== null;
+ // In exact mode only fully-specified group:value terms are addable; bare
+ // labels and group-any (key:null) are rejected because the backend can't
+ // honor them.
+ const candidateAddable =
+ candidate !== null &&
+ (!exactValuesOnly || (candidate.key !== null && candidate.value !== null));
const candidateExists =
candidate !== null && value.some((term) => sameTerm(term, candidate));
const showAdd = candidateAddable && !candidateExists;
@@ -155,7 +166,7 @@ function TagFilterControl({
.slice(0, MAX_AUTOCOMPLETE_SUGGESTIONS)
: [];
const labelMatches =
- typedKey === null
+ typedKey === null && !exactValuesOnly
? labelSuggestions
.filter((label) => label.toLowerCase().includes(normalizedQuery))
.filter(
@@ -296,7 +307,11 @@ function TagFilterControl({
) : null}
<Command shouldFilter={false}>
<CommandInput
- placeholder="Filter by label, group:*, or group:label…"
+ placeholder={
+ exactValuesOnly
+ ? "Filter by group:value…"
+ : "Filter by label or group:label…"
+ }
value={query}
onValueChange={setQuery}
onKeyDown={(event) => {
@@ -307,7 +322,11 @@ function TagFilterControl({
}}
/>
<CommandList>
- <CommandEmpty>Type a label, group:*, or group:label.</CommandEmpty>
+ <CommandEmpty>
+ {exactValuesOnly
+ ? "Type group:value to filter."
+ : "Type a label, group:*, or group:label."}
+ </CommandEmpty>
{showAdd && candidate ? (
<CommandGroup>
<CommandItem
@@ -326,13 +345,21 @@ function TagFilterControl({
key={tagKey.key}
value={`__group__,${tagKey.key}`}
className="justify-between gap-2"
- // Filter by group (any value in it).
- onSelect={() => addTerm({ key: tagKey.key, value: null })}
+ // Default: filter by group (any value in it). In exact mode
+ // group-any isn't supported, so prefill `key:` and make the
+ // user pick a concrete value instead.
+ onSelect={() =>
+ exactValuesOnly
+ ? setQuery(`${tagKey.key}:`)
+ : addTerm({ key: tagKey.key, value: null })
+ }
>
<div className="flex min-w-0 flex-col">
<span className="truncate">
<span className="font-medium">{tagKey.key}</span>
- <span className="text-muted-foreground">: any</span>
+ <span className="text-muted-foreground">
+ {exactValuesOnly ? ": …" : ": any"}
+ </span>
</span>
{tagKey.description ? (
<span className="truncate text-xs text-muted-foreground">
diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx
index 79acb664a..cfb6fcfea 100644
--- a/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx
+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx
@@ -36,30 +36,25 @@ vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({
}));
import { OnboardingProvider } from "./OnboardingProvider";
-import type { OnboardingState, OnboardingStateResponse } from "./types";
+import type { OnboardingStateResponse } from "./types";
import { useOnboardingState } from "./useOnboardingState";
-import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
const DISMISSED_AT = "2026-01-01T00:00:00.000Z";
-function freshOnboardingState(): OnboardingState {
- return {
- tour_completed_at: null,
- modal_dismissed_at: null,
- first_save_at: null,
- first_run_at: null,
- ab_variant: null,
- user_intent: null,
- seen_canvas: null,
- seen_node_adder: null,
- seen_sidebar: null,
- seen_save_run: null,
- };
-}
-
function freshResponse(): OnboardingStateResponse {
return {
- onboarding_state: freshOnboardingState(),
+ onboarding_state: {
+ tour_completed_at: null,
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: null,
+ user_intent: null,
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+ },
launch_date_at_signup: "2025-01-01T00:00:00Z",
};
}
@@ -125,7 +120,7 @@ describe("OnboardingProvider write resilience", () => {
const dismissedResponse: OnboardingStateResponse = {
...freshResponse(),
onboarding_state: {
- ...freshOnboardingState(),
+ ...freshResponse().onboarding_state,
modal_dismissed_at: DISMISSED_AT,
},
};
@@ -147,25 +142,3 @@ describe("OnboardingProvider write resilience", () => {
expect(screen.getByTestId("dismissed").textContent).toBe(DISMISSED_AT);
});
});
-
-describe("OnboardingProvider missing onboarding_state", () => {
- it("renders without throwing when the response omits onboarding_state", async () => {
- // A signed-out / transitional /users/me/onboarding body can lack
- // onboarding_state; the provider must not dereference it (SKY-11430).
- // The key is absent (not explicitly undefined) to mirror the real JSON.
- const responseWithoutState: OnboardingStateResponse = {
- launch_date_at_signup: null,
- };
- mockGet.mockResolvedValue({ data: responseWithoutState });
-
- renderProvider();
-
- await waitFor(() => expect(mockGet).toHaveBeenCalledTimes(1));
- // state resolves to null, so the consumer reads an undefined field rather
- // than crashing on a missing onboarding_state.
- await waitFor(() =>
- expect(screen.getByTestId("dismissed").textContent).toBe("undefined"),
- );
- expect(OnboardingTelemetry.registerVariant).not.toHaveBeenCalled();
- });
-});
diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx
index 395cb5e51..69fb16cd6 100644
--- a/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx
+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx
@@ -49,7 +49,7 @@ function OnboardingProvider({ children }: Readonly<Props>) {
await queryClient.cancelQueries({ queryKey: QUERY_KEY });
const previous =
queryClient.getQueryData<OnboardingStateResponse>(QUERY_KEY);
- if (previous?.onboarding_state) {
+ if (previous) {
queryClient.setQueryData<OnboardingStateResponse>(QUERY_KEY, {
...previous,
onboarding_state: { ...previous.onboarding_state, ...patch },
@@ -70,7 +70,7 @@ function OnboardingProvider({ children }: Readonly<Props>) {
data?.launch_date_at_signup != null &&
new Date(data.launch_date_at_signup) >= new Date(PRODUCT_LAUNCH_DATE);
- const abVariant = data?.onboarding_state?.ab_variant ?? null;
+ const abVariant = data?.onboarding_state.ab_variant ?? null;
useEffect(() => {
if (abVariant) {
diff --git a/skyvern-frontend/src/store/onboarding/types.ts b/skyvern-frontend/src/store/onboarding/types.ts
index 67798c1f9..11912d185 100644
--- a/skyvern-frontend/src/store/onboarding/types.ts
+++ b/skyvern-frontend/src/store/onboarding/types.ts
@@ -15,7 +15,7 @@ export type OnboardingState = {
};
export type OnboardingStateResponse = {
- onboarding_state?: OnboardingState;
+ onboarding_state: OnboardingState;
launch_date_at_signup: string | null;
};
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index f35d4b0ae..ebd8191be 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -639,6 +639,209 @@ def grade_structured_record_criteria(
return best_verdicts
+_TERMINAL_ACTION_KEY_TOKENS = (
+ (("submitted",), "submission"),
+ (("request", "submitted"), "request"),
+ (("application", "submitted"), "application"),
+ (("form", "submitted"), "form"),
+ (("order", "placed"), "order"),
+)
+_TERMINAL_ARTIFACT_KEY_TOKENS = (
+ (("confirmation", "number"), "confirmation"),
+ (("confirmation", "id"), "confirmation"),
+ (("request", "number"), "request"),
+ (("request", "id"), "request"),
+ (("submission", "id"), "submission"),
+ (("submission", "number"), "submission"),
+ (("application", "number"), "application"),
+ (("application", "id"), "application"),
+ (("order", "number"), "order"),
+ (("order", "id"), "order"),
+ (("receipt", "number"), "receipt"),
+ (("receipt", "id"), "receipt"),
+ (("reference", "number"), "reference"),
+ (("reference", "id"), "reference"),
+)
+_TERMINAL_CRITERION_VERBS = frozenset({"submit", "submitted", "submission", "place", "placed"})
+_TERMINAL_CRITERION_ABSTAIN_TOKENS = frozenset(
+ {
+ "lookup",
+ "looked",
+ "retrieve",
+ "retrieved",
+ "retrieval",
+ "return",
+ "returned",
+ "status",
+ }
+)
+_NEGATIVE_GUARD_TOKENS = frozenset({"blocker", "error", "failure", "failed", "challenge"})
+_NEGATIVE_TERMINAL_STATUS_VALUES = frozenset(
+ {
+ "blocked",
+ "cancelled",
+ "canceled",
+ "captcha required",
+ "denied",
+ "error",
+ "failed",
+ "failure",
+ "incomplete",
+ "not submitted",
+ "timeout",
+ "unable",
+ }
+)
+_TERMINAL_RECORD_FAMILY_ARTIFACTS = {
+ "request": frozenset({"confirmation", "reference", "request", "submission"}),
+ "application": frozenset({"application", "confirmation", "reference", "submission"}),
+ "form": frozenset({"confirmation", "reference", "submission"}),
+ "order": frozenset({"order", "receipt"}),
+}
+_TERMINAL_RECORD_FAMILY_ACTIONS = {
+ "request": frozenset({"request", "submission"}),
+ "application": frozenset({"application", "submission"}),
+ "form": frozenset({"form", "submission"}),
+ "order": frozenset({"order"}),
+}
+
+
+def grade_terminal_goal_record_criteria(
+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot
+) -> list[CriterionVerdict]:
+ verdicts: list[CriterionVerdict] = []
+ eligible_criteria = [
+ (criterion, family)
+ for criterion in criteria
+ if (family := _terminal_goal_criterion_family(criterion.outcome)) is not None
+ ]
+ if not eligible_criteria:
+ return []
+ for label, payload in snapshot.block_outputs.items():
+ record = _structured_record_payload(payload)
+ if record is None:
+ continue
+ for criterion, family in eligible_criteria:
+ if _terminal_goal_record_confirmed(record, family):
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref=f"block_outputs:{label}",
+ )
+ )
+ if verdicts:
+ return verdicts
+ return []
+
+
+def _terminal_goal_criterion_family(outcome: str) -> str | None:
+ if _extract_present_value_literals(outcome):
+ return None
+ tokens = set(_status_word_tokens(outcome))
+ if tokens & _TERMINAL_CRITERION_ABSTAIN_TOKENS:
+ return None
+ if not tokens & _TERMINAL_CRITERION_VERBS:
+ return None
+ families = [family for family in ("request", "application", "form", "order") if family in tokens]
+ if len(families) != 1:
+ return None
+ family = families[0]
+ if family == "order" and "placed" not in tokens:
+ return None
+ return family
+
+
+def _terminal_goal_record_confirmed(record: dict[str, Any], family: str) -> bool:
+ if _terminal_goal_record_has_negative_guard(record):
+ return False
+ if _structured_record_contradiction(record):
+ return False
+ action_families = [
+ action_family
+ for key, value in _walk_record_scalars(record)
+ if (action_family := _terminal_action_family_for_key(key)) is not None and value is True
+ ]
+ if any(
+ value is False
+ for key, value in _walk_record_scalars(record)
+ if _terminal_action_family_for_key(key) is not None
+ ):
+ return False
+ has_family_artifact = _terminal_goal_record_has_artifact_for_family(record, family)
+ if not action_families:
+ return has_family_artifact and not _terminal_goal_record_has_generic_success_claim(record)
+ if family not in _TERMINAL_RECORD_FAMILY_ACTIONS:
+ return False
+ if not any(action_family in _TERMINAL_RECORD_FAMILY_ACTIONS[family] for action_family in action_families):
+ return False
+ return has_family_artifact
+
+
+def _terminal_action_family_for_key(key: str) -> str | None:
+ leaf_tokens = _record_key_leaf_tokens(key)
+ for key_tokens, family in _TERMINAL_ACTION_KEY_TOKENS:
+ if leaf_tokens == key_tokens:
+ return family
+ return None
+
+
+def _terminal_goal_record_has_generic_success_claim(record: dict[str, Any]) -> bool:
+ generic_success_keys = {
+ ("completed",),
+ ("succeeded",),
+ ("success",),
+ }
+ for key, value in _walk_record_scalars(record):
+ leaf_tokens = _record_key_leaf_tokens(key)
+ if value is True and _terminal_action_family_for_key(key) is None and leaf_tokens:
+ if leaf_tokens[-1] in _TERMINAL_CRITERION_VERBS:
+ return True
+ if leaf_tokens in generic_success_keys and value is True:
+ return True
+ if "status" in _key_word_tokens(key) and isinstance(value, str):
+ polarity = _status_polarity(value)
+ if polarity is not None and not polarity[0]:
+ return True
+ return False
+
+
+def _terminal_goal_record_has_negative_guard(record: dict[str, Any]) -> bool:
+ for key, value in _walk_record_scalars(record):
+ tokens = _key_word_tokens(key)
+ if tokens & _NEGATIVE_GUARD_TOKENS and _is_meaningful_record_value(value) and value is not False:
+ return True
+ if "status" in tokens and isinstance(value, str):
+ normalized_status = _normalize_present_value(value)
+ if normalized_status in _NEGATIVE_TERMINAL_STATUS_VALUES or any(
+ phrase in normalized_status for phrase in ("captcha required", "not submitted", "unable")
+ ):
+ return True
+ polarity = _status_polarity(value)
+ if polarity is not None and polarity[0]:
+ return True
+ return False
+
+
+def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: str) -> bool:
+ allowed_artifacts = _TERMINAL_RECORD_FAMILY_ARTIFACTS[family]
+ for key, value in _walk_record_scalars(record):
+ if isinstance(value, bool) or not _is_meaningful_record_value(value):
+ continue
+ leaf_tokens = _record_key_leaf_tokens(key)
+ for key_tokens, artifact_family in _TERMINAL_ARTIFACT_KEY_TOKENS:
+ if leaf_tokens == key_tokens and artifact_family in allowed_artifacts:
+ return True
+ return False
+
+
+def _record_key_leaf_tokens(key: str) -> tuple[str, ...]:
+ leaf = re.sub(r"\[\d+\]$", "", key.rsplit(".", 1)[-1])
+ spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", leaf).casefold()
+ return tuple(re.findall(r"[a-z0-9]+", spaced))
+
+
def _structured_record_satisfied(criterion_id: str, label: str) -> CriterionVerdict:
return CriterionVerdict(
criterion_id=criterion_id,
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 0d15dece4..6903d4878 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_present_value_criteria,
grade_record_semantic_consistency,
grade_structured_record_criteria,
+ grade_terminal_goal_record_criteria,
summarize_unsatisfied_outcomes,
verdict_missing_evidence,
)
@@ -437,6 +438,8 @@ def _apply_present_value_upgrades(
run_result: CompletionVerificationResult,
run_criteria: list[CompletionCriterion],
snapshot: RunEvidenceSnapshot,
+ *,
+ include_terminal_goal_records: bool = False,
) -> CompletionVerificationResult:
"""Upgrade a ``no_evidence``/``unknown`` run verdict to a deterministic present-value
``satisfied``. An ``evidence_contradicts`` verdict is left to the judge, a judge
@@ -448,6 +451,10 @@ def _apply_present_value_upgrades(
upgrades.update(
{verdict.criterion_id: verdict for verdict in grade_structured_record_criteria(run_criteria, snapshot)}
)
+ if include_terminal_goal_records:
+ upgrades.update(
+ {verdict.criterion_id: verdict for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot)}
+ )
semantic_verdicts = {
verdict.criterion_id: verdict for verdict in grade_record_semantic_consistency(run_criteria, snapshot)
}
@@ -497,6 +504,8 @@ def _deterministic_run_verification_result(
deterministic_by_id[verdict.criterion_id] = verdict
for verdict in grade_structured_record_criteria(run_criteria, snapshot):
deterministic_by_id[verdict.criterion_id] = verdict
+ for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):
+ deterministic_by_id[verdict.criterion_id] = verdict
for verdict in semantic_verdicts:
deterministic_by_id[verdict.criterion_id] = verdict
@@ -581,21 +590,21 @@ async def _maybe_run_completion_verification(
],
)
else:
- handler = await _completion_verification_handler(copilot_ctx)
- if handler is None:
- return CompletionVerificationResult(status="unavailable")
- # Too little budget to verify a candidate run: fail closed (unavailable)
- # rather than let the run-status proxy claim an unverified outcome as success.
- remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
- if (
- remaining
- <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
- ):
- return CompletionVerificationResult(status="unavailable")
deterministic_result, remaining_criteria = _deterministic_run_verification_result(run_criteria, snapshot)
if deterministic_result is not None and not remaining_criteria:
run_result = deterministic_result
else:
+ handler = await _completion_verification_handler(copilot_ctx)
+ if handler is None:
+ return CompletionVerificationResult(status="unavailable")
+ # Too little budget to verify a candidate run: fail closed (unavailable)
+ # rather than let the run-status proxy claim an unverified outcome as success.
+ remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
+ if (
+ remaining
+ <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
+ ):
+ return CompletionVerificationResult(status="unavailable")
judged_result = await evaluate_completion_criteria(remaining_criteria, snapshot, handler)
if judged_result.status != "evaluated":
run_result = judged_result
@@ -609,7 +618,12 @@ async def _maybe_run_completion_verification(
criterion_ids=[criterion.id for criterion in run_criteria],
verdicts=verdicts,
)
- run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)
+ run_result = _apply_present_value_upgrades(
+ run_result,
+ run_criteria,
+ snapshot,
+ include_terminal_goal_records=True,
+ )
return combine_verification_results(criterion_ids, run_result, definition_verdicts)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index cb451d015..47dbbc7aa 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -965,6 +965,51 @@ def _code_block_authoring_repair_context(
return None
+def _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml: str) -> str:
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return workflow_yaml
+ declared_string_keys = _declared_string_workflow_parameter_keys(parsed)
+ if not declared_string_keys:
+ return workflow_yaml
+ available_binding_keys_by_label = _code_block_available_binding_keys_by_label(workflow_yaml)
+ adopted_by_label: dict[str, list[str]] = {}
+ for block in _workflow_code_blocks(parsed):
+ label = str(block.get("label") or "").strip()
+ if not label:
+ continue
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ parameter_keys = _code_block_parameter_keys(block)
+ available_declared_keys = set(available_binding_keys_by_label.get(label, [])) & declared_string_keys
+ if not available_declared_keys:
+ continue
+ diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)
+ if diagnostic is None or not diagnostic.unresolved_names:
+ continue
+ adopted_keys = [
+ name
+ for name in diagnostic.unresolved_names
+ if name in available_declared_keys and name not in parameter_keys
+ ]
+ if not adopted_keys:
+ continue
+ raw_keys = block.get("parameter_keys")
+ merged_keys = (
+ [str(key) for key in raw_keys if isinstance(key, str) and key] if isinstance(raw_keys, list) else []
+ )
+ for key in adopted_keys:
+ if key not in merged_keys:
+ merged_keys.append(key)
+ block["parameter_keys"] = merged_keys
+ adopted_by_label[label] = adopted_keys
+ if not adopted_by_label:
+ return workflow_yaml
+ LOG.info("copilot adopted exact declared parameter keys for unresolved names", adopted_by_label=adopted_by_label)
+ return yaml.safe_dump(parsed, sort_keys=False)
+
+
def _repair_context_log_values(values: list[str], *, max_items: int = 20) -> list[str]:
cleaned: list[str] = []
for raw_value in values[:max_items]:
@@ -3523,6 +3568,8 @@ async def _update_workflow(
user_facing_summary=_compiled_authoring_user_summary(),
data=_code_repair_progress_data(),
)
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ workflow_yaml = _adopt_exact_declared_parameter_keys_for_unresolved_names(workflow_yaml)
params["workflow_yaml"] = workflow_yaml
parameter_contract_error = _code_block_parameter_contract_error(workflow_yaml)
if parameter_contract_error is not None:
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 9a9e718f9..c9fa8344a 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -738,6 +738,64 @@ class TestCodeSafetySeam:
assert accepted["ok"] is True
assert ctx.last_code_authoring_repair_context is None
+ @pytest.mark.asyncio
+ async def test_code_only_exact_declared_string_parameters_are_adopted_for_unresolved_names(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: City}
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("#query").fill(str(provider_query))
+ await page.locator("#location").fill(str(search_location))
+ """
+ )
+
+ accepted = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert accepted["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_query", "search_location"]
+
+ @pytest.mark.asyncio
+ async def test_code_only_partial_declared_parameter_name_still_rejects(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample}
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("#query").fill(str(provider_name))
+ """
+ )
+
+ rejected = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert rejected["ok"] is False
+ result_context = rejected["data"]["authoring_repair_context"]
+ assert result_context["unresolved_names"] == ["provider_name"]
+ assert result_context["available_parameter_keys"] == ["provider_query"]
+ assert result_context["binding_candidates"] == ["provider_query", "provider_name"]
+
@pytest.mark.asyncio
async def test_unresolved_name_repair_context_includes_existing_workflow_binding_candidates(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index ab0f8e835..4a0d9a846 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -22,8 +22,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (
_structured_record_has_identifier,
evaluate_completion_criteria,
grade_definition_criteria,
+ grade_present_value_criteria,
grade_record_semantic_consistency,
grade_structured_record_criteria,
+ grade_terminal_goal_record_criteria,
+ structured_record_has_goal_content,
structured_record_has_identity,
summarize_unsatisfied_outcomes,
)
@@ -136,6 +139,21 @@ def _record_payload(**overrides: Any) -> dict[str, Any]:
return payload
+def _terminal_goal_payload(**overrides: Any) -> dict[str, Any]:
+ payload = {
+ "submitted": True,
+ "blocker": None,
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ "deposit_amount": "$41.00 plus initiation fee",
+ "next_owner": "Provider",
+ "evidence_text": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",
+ }
+ payload.update(overrides)
+ return payload
+
+
def _status_snapshot(
status: str,
*,
@@ -439,6 +457,187 @@ def test_structured_record_criteria_satisfy_structured_record_outputs(block_outp
assert _satisfied_criterion_ids(verdicts) == _STRUCTURED_RECORD_CRITERION_IDS
+def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})
+
+ verdicts = grade_terminal_goal_record_criteria(
+ [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ )
+
+ assert verdicts == [
+ CriterionVerdict(
+ criterion_id="c0",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_water_request",
+ )
+ ]
+
+
+def test_terminal_goal_record_accepts_family_artifact_without_self_asserted_boolean() -> None:
+ payload = _terminal_goal_payload(submitted=None)
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})
+
+ verdicts = grade_terminal_goal_record_criteria(
+ [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ )
+
+ assert _satisfied_criterion_ids(verdicts) == {"c0"}
+
+
+ "payload",
+ [
+ _terminal_goal_payload(confirmation_number=None),
+ _terminal_goal_payload(confirmation_number=True),
+ _terminal_goal_payload(confirmation_number="", account_number="100245"),
+ _terminal_goal_payload(confirmation_number=None, record_number="RN-100245"),
+ _terminal_goal_payload(confirmation_number=None, customer_id="cus_123456"),
+ ],
+)
+def test_terminal_goal_record_rejects_ordinary_identifiers(payload: dict[str, Any]) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})
+
+ assert (
+ grade_terminal_goal_record_criteria(
+ [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ )
+ == []
+ )
+
+
+ ("payload", "outcome"),
+ [
+ (
+ _terminal_goal_payload(confirmation_number="WTR-1842-DEMO"),
+ "a commercial water service request is submitted",
+ ),
+ (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "an order is placed"),
+ (
+ _terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),
+ "an application is submitted",
+ ),
+ (
+ _terminal_goal_payload(form_submitted=True, submitted=None, submission_id="SUB-1842"),
+ "the form is submitted",
+ ),
+ (
+ _terminal_goal_payload(request_submitted=True, submitted=None, request_id="REQ-1842"),
+ "a service request is submitted",
+ ),
+ ],
+)
+def test_terminal_goal_record_accepts_narrow_terminal_artifacts(payload: dict[str, Any], outcome: str) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ verdicts = grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot)
+
+ assert _satisfied_criterion_ids(verdicts) == {"c0"}
+
+
+ ("payload", "outcome"),
+ [
+ (_terminal_goal_payload(), "an order is placed"),
+ (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "a request is submitted"),
+ (
+ _terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),
+ "an order is placed",
+ ),
+ ],
+)
+def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []
+
+
+ "payload",
+ [
+ _terminal_goal_payload(submitted=None, completed=True),
+ _terminal_goal_payload(submitted=None, succeeded=True),
+ _terminal_goal_payload(submitted=None, success=True),
+ _terminal_goal_payload(submitted=None, status="completed"),
+ ],
+)
+def test_terminal_goal_record_rejects_generic_success_synonyms(payload: dict[str, Any]) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []
+
+
+ "outcome",
+ [
+ "the account status is retrieved",
+ "the record lookup result is returned",
+ "the water request status is shown",
+ ],
+)
+def test_terminal_goal_record_abstains_for_lookup_and_status_criteria(outcome: str) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": _terminal_goal_payload()})
+
+ assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []
+
+
+ "payload",
+ [
+ _terminal_goal_payload(blocker="provider requires a phone call"),
+ _terminal_goal_payload(error="submission failed"),
+ _terminal_goal_payload(failure_reason="network failure"),
+ _terminal_goal_payload(challenge_detected=True),
+ _terminal_goal_payload(submitted=False),
+ _terminal_goal_payload(status="failed"),
+ _terminal_goal_payload(status="denied"),
+ _terminal_goal_payload(status="cancelled"),
+ _terminal_goal_payload(status="canceled"),
+ _terminal_goal_payload(status="incomplete"),
+ _terminal_goal_payload(status="timeout"),
+ _terminal_goal_payload(status="captcha required"),
+ _terminal_goal_payload(status="not submitted"),
+ _terminal_goal_payload(status="unable to submit"),
+ ],
+)
+def test_terminal_goal_record_negative_guards_abstain(payload: dict[str, Any]) -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert (
+ grade_terminal_goal_record_criteria(
+ [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ )
+ == []
+ )
+
+
[email protected]("key", ["not_submitted", "previously_submitted"])
+def test_terminal_goal_record_rejects_negated_or_temporal_action_keys(key: str) -> None:
+ payload = _terminal_goal_payload(submitted=None)
+ payload[key] = True
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert (
+ grade_terminal_goal_record_criteria(
+ [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ )
+ == []
+ )
+
+
+def test_terminal_goal_record_does_not_take_literal_criteria_from_present_value() -> None:
+ criteria = [_criterion("c0", "the confirmation number WTR-1842-DEMO is reported")]
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": _terminal_goal_payload()})
+
+ assert grade_terminal_goal_record_criteria(criteria, snapshot) == []
+ assert _satisfied_criterion_ids(grade_present_value_criteria(criteria, snapshot)) == {"c0"}
+
+
+def test_structured_record_goal_content_remains_strict_for_flat_terminal_payload() -> None:
+ assert structured_record_has_goal_content(_terminal_goal_payload()) is False
+
+
def test_structured_record_partial_matches_do_not_combine_across_blocks() -> None:
snapshot = RunEvidenceSnapshot(
block_outputs={
@@ -842,6 +1041,26 @@ def _structured_record_top_level_output_result() -> dict:
}
+def _terminal_goal_output_result(**payload_overrides: Any) -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_terminal_goal",
+ "overall_status": "completed",
+ "executed_block_labels": ["submit_water_request"],
+ "current_url": "https://example.test/confirmation",
+ "blocks": [
+ {
+ "label": "submit_water_request",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": _terminal_goal_payload(**payload_overrides),
+ }
+ ],
+ },
+ }
+
+
def _run_ctx() -> CopilotContext:
ctx = CopilotContext(
organization_id="o",
@@ -1669,6 +1888,46 @@ async def test_page_observation_verification_recognizes_budgeted_outcome(
assert "SKU-12345 is present in the cart" in seen_prompt["prompt"]
+async def test_page_observation_verification_does_not_apply_terminal_goal_record_upgrade(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": False,
+ "reason_code": "no_evidence",
+ }
+ ]
+ }
+
+ _patch_completion_handler(monkeypatch, handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ )
+ ctx.last_test_ok = False
+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"
+ ctx.copilot_run_start_monotonic = time.monotonic()
+ _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url="https://example.com/confirmation",
+ observed_data=_terminal_goal_payload(),
+ )
+
+ result = await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url="https://example.com/confirmation",
+ observed_data=_terminal_goal_payload(),
+ )
+
+ assert result is not None
+ assert result.is_fully_satisfied() is False
+
+
@pytest.mark.asyncio
async def test_page_observation_verification_does_not_overwrite_satisfied_verdict(
monkeypatch: pytest.MonkeyPatch,
@@ -2049,6 +2308,97 @@ async def test_maybe_run_completion_verification_treats_fallback_record_as_crite
assert handler_lookup_calls == 0
+async def test_maybe_run_completion_verification_terminal_goal_bypasses_judge(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler_lookup_calls = 0
+
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("deterministically covered terminal goal must not call the judge")
+
+ async def handler_lookup(_ctx: object) -> object:
+ nonlocal handler_lookup_calls
+ handler_lookup_calls += 1
+ return fail_handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
+ handler_lookup,
+ )
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_water_request")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"
+ assert handler_lookup_calls == 0
+
+
+async def test_maybe_run_completion_verification_terminal_goal_without_boolean_bypasses_judge(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("artifact-backed terminal goal must not call the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_water_request")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx, _terminal_goal_output_result(submitted=None), time.monotonic()
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"
+
+
+async def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_judge_unknown(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c1",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "block_outputs:submit_water_request",
+ }
+ ]
+ }
+
+ _patch_completion_handler(monkeypatch, handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_water_request")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c0", "a commercial water service request is submitted"),
+ _criterion("c1", "the selected start date is reported"),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert {verdict.criterion_id: verdict.reason_code for verdict in verification.verdicts} == {
+ "c0": "evidence_confirms",
+ "c1": "evidence_confirms",
+ }
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_fails_closed_on_no_judge_for_judge_needed_criteria(
monkeypatch: pytest.MonkeyPatch,