Skip to content

Automate browser based workflows with AI

Stop rejecting valid requested outputs after copying errors (#7596)

Andrew Neilsonf1e679b9f039
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.test.tsx
index bbf65c6a5..535046c46 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.test.tsx
@@ -555,24 +555,6 @@ describe("RunView failure banner", () => {
expect(scope.queryByText("canceled by user")).toBeNull();
});
-
- test("hides the run failure banner outside the Timeline view", () => {
- seedCompletedRun({
- status: Status.Failed,
- failure_reason: "Login page rejected the credentials",
- });
- const { container } = renderRunView();
- const scope = within(container);
-
- // Visible on the default Timeline view.
- expect(
- scope.getByText("Login page rejected the credentials"),
- ).not.toBeNull();
-
- // Switching to another tab hides the run-level failure banner.
- fireEvent.click(scope.getByRole("button", { name: "Outputs" }));
- expect(scope.queryByText("Login page rejected the credentials")).toBeNull();
- });
});
describe("RunView live affordances", () => {
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
index 20c7ac2e0..65c849742 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
@@ -352,7 +352,7 @@ export function RunView({
</div>
) : null}
- {failed && !failureDismissed && view === "timeline" ? (
+ {failed && !failureDismissed ? (
<div className="shrink-0 rounded-lg border border-destructive/40 bg-slate-elevation1 p-4">
<div className="flex items-start gap-2 text-sm font-semibold text-foreground">
<ExclamationTriangleIcon className="mt-0.5 h-4 w-4 shrink-0 text-destructive" />
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
index 6932555b0..64b1ee89a 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
@@ -150,43 +150,16 @@ describe("WorkflowRunBlockDetail router", () => {
expect(screen.queryByText("Pull the price from the page")).toBeNull();
});
- it("shows the failure_reason once, scoped to the Summary sub-panel", () => {
+ it("renders the failure_reason in the detail body (not the header) when set", () => {
const block = buildBlock({
block_type: "extraction",
failure_reason: "Could not locate the price element",
});
render(<WorkflowRunBlockDetail activeItem={block} timeline={[]} />);
- // A failed block opens on Summary and shows the failure exactly once — no
- // longer duplicated between the Summary field and a persistent callout.
expect(screen.getByText("Failure")).toBeDefined();
expect(
- screen.getAllByText("Could not locate the price element"),
- ).toHaveLength(1);
- // Switching sub-panels no longer repeats the failure.
- fireEvent.mouseDown(screen.getByRole("tab", { name: "Inputs" }));
- expect(screen.queryByText("Could not locate the price element")).toBeNull();
- });
-
- it("shows a triggered-workflow failure once, not in a separate body section", () => {
- // The backend copies the child failure into both block.failure_reason and
- // block.output.failure_reason, so the Summary callout already covers it.
- const reason = "Triggered workflow execution failed: boom";
- const block = buildBlock({
- block_type: "workflow_trigger",
- failure_reason: reason,
- output: {
- workflow_run_id: "wr_child",
- status: "failed",
- failure_reason: reason,
- },
- });
- render(
- <MemoryRouter>
- <WorkflowRunBlockDetail activeItem={block} timeline={[]} />
- </MemoryRouter>,
- );
- expect(screen.getAllByText(reason)).toHaveLength(1);
- expect(screen.queryByText("Sub-workflow failure")).toBeNull();
+ screen.getAllByText("Could not locate the price element").length,
+ ).toBeGreaterThanOrEqual(1);
});
it("renders code block extracted information without the raw output wrapper", () => {
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailConditional.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailConditional.tsx
index c493b5026..1417c0ebc 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailConditional.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailConditional.tsx
@@ -3,7 +3,7 @@ import {
type WorkflowRunBlock,
} from "../../types/workflowRunTypes";
import { JsonExplorer } from "./BlockInspector";
-import { CodeBlock, Section } from "./shared";
+import { BlockDetailFailure, CodeBlock, Section } from "./shared";
import { cn } from "@/util/utils";
type Props = {
@@ -44,6 +44,7 @@ function BlockDetailConditional({ block }: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{hasExecutedBranch && evaluations && evaluations.length > 0 ? (
<Section title="Branches">
<div className="space-y-2">
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailGeneric.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailGeneric.tsx
index 2f3ad65c4..35cbc1d6c 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailGeneric.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailGeneric.tsx
@@ -1,5 +1,5 @@
import type { WorkflowRunBlock } from "../../types/workflowRunTypes";
-import { GoalText, Section } from "./shared";
+import { BlockDetailFailure, GoalText, Section } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -8,6 +8,7 @@ type Props = {
function BlockDetailGeneric({ block }: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{block.prompt && (
<Section title="Prompt">
<GoalText text={block.prompt} />
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHttpRequest.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHttpRequest.tsx
index 6189fb224..0c4faff38 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHttpRequest.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHttpRequest.tsx
@@ -1,6 +1,6 @@
import type { WorkflowRunBlock } from "../../types/workflowRunTypes";
import { JsonExplorer } from "./BlockInspector";
-import { Section } from "./shared";
+import { BlockDetailFailure, Section } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -9,6 +9,7 @@ type Props = {
function BlockDetailHttpRequest({ block }: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{block.url && (
<Section title="URL">
<span className="break-all text-xs text-tertiary-foreground">
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHumanInteraction.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHumanInteraction.tsx
index ee0a9cabd..d6fbc58ca 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHumanInteraction.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailHumanInteraction.tsx
@@ -1,6 +1,6 @@
import type { WorkflowRunBlock } from "../../types/workflowRunTypes";
import { WorkflowRunHumanInteraction } from "../WorkflowRunHumanInteraction";
-import { GoalText, Section } from "./shared";
+import { BlockDetailFailure, GoalText, Section } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -10,6 +10,7 @@ function BlockDetailHumanInteraction({ block }: Props) {
const recipients = block.recipients ?? [];
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{block.instructions && (
<Section title="Instructions">
<GoalText text={block.instructions} />
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailLoop.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailLoop.tsx
index 08a6a0241..614937fad 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailLoop.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailLoop.tsx
@@ -1,6 +1,6 @@
import type { WorkflowRunBlock } from "../../types/workflowRunTypes";
import { stringifyTimelineValue } from "./formatValue";
-import { CodeBlock, Section } from "./shared";
+import { BlockDetailFailure, CodeBlock, Section } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -29,6 +29,7 @@ function BlockDetailLoop({ block, iterationIndex = null }: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{showIterationOnly && (
<Section title={`Iteration ${(highlightIndex ?? 0) + 1} value`}>
<CodeBlock>{stringifyTimelineValue(highlightedValue)}</CodeBlock>
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailTask.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailTask.tsx
index cd659b2c1..e98fc6230 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailTask.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailTask.tsx
@@ -3,7 +3,7 @@ import type {
WorkflowRunBlock,
} from "../../types/workflowRunTypes";
import type { WorkflowRunOverviewActiveElement } from "../WorkflowRunOverview";
-import { BlockThoughtList } from "./shared";
+import { BlockDetailFailure, BlockThoughtList } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -13,12 +13,14 @@ type Props = {
};
function BlockDetailTask({
+ block,
activeItem,
thoughts = [],
onThoughtSelect,
}: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
<BlockThoughtList
thoughts={thoughts}
activeItem={activeItem}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailWorkflowTrigger.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailWorkflowTrigger.tsx
index f0caf9674..2b72947fb 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailWorkflowTrigger.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockDetailWorkflowTrigger.tsx
@@ -1,6 +1,6 @@
import { Link } from "react-router-dom";
import type { WorkflowRunBlock } from "../../types/workflowRunTypes";
-import { JsonView, Section } from "./shared";
+import { BlockDetailFailure, JsonView, Section } from "./shared";
type Props = {
block: WorkflowRunBlock;
@@ -10,6 +10,7 @@ type SubWorkflowRun = {
workflow_run_id?: string;
workflow_permanent_id?: string;
status?: string;
+ failure_reason?: string | null;
outputs?: unknown;
};
@@ -34,6 +35,7 @@ function BlockDetailWorkflowTrigger({ block }: Props) {
return (
<div className="space-y-4 px-3 py-3 empty:hidden">
+ <BlockDetailFailure block={block} />
{subRun?.workflow_run_id && (
<Section title="Triggered run">
<div className="flex flex-wrap items-center gap-2 text-xs">
@@ -54,6 +56,13 @@ function BlockDetailWorkflowTrigger({ block }: Props) {
</div>
</Section>
)}
+ {subRun?.failure_reason && (
+ <Section title="Sub-workflow failure">
+ <div className="rounded border border-destructive/40 bg-destructive/10 px-2.5 py-2 text-xs leading-relaxed text-destructive">
+ {subRun.failure_reason}
+ </div>
+ </Section>
+ )}
{hasSubRunOutput && (
<Section title="Sub-workflow output">
<JsonView value={subRun!.outputs} />
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.test.tsx
index 9c5914885..5ea7f9ca5 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.test.tsx
@@ -150,37 +150,3 @@ describe("BlockInspector Inputs tab", () => {
expect(screen.getByText("Fill out the form")).toBeDefined();
});
});
-
-describe("BlockInspector failure scoping", () => {
- it("shows the failure reason on Summary and not on the other sub-panels", () => {
- const block = buildBlock({
- failure_reason: "Element not found",
- });
- render(<BlockInspector block={block} />);
-
- expect(screen.getByText("Failure")).toBeDefined();
- expect(screen.getByText("Element not found")).toBeDefined();
-
- fireEvent.mouseDown(screen.getByRole("tab", { name: "Inputs" }));
- expect(screen.queryByText("Element not found")).toBeNull();
- });
-
- it("opens on Summary for a failed block even when output exists", () => {
- const block = buildBlock({
- failure_reason: "Element not found",
- output: { result: "partial" },
- });
- render(<BlockInspector block={block} />);
-
- // Default tab is Summary (not Outputs), so the failure is visible on load.
- expect(screen.getByText("Element not found")).toBeDefined();
- });
-
- it("surfaces the block failure on Summary when an action is selected", () => {
- const block = buildBlock({ failure_reason: "Login rejected" });
- const action = buildAction({ action_type: ActionTypes.Click });
- render(<BlockInspector block={block} action={action} />);
-
- expect(screen.getByText("Login rejected")).toBeDefined();
- });
-});
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
index 2f99cc38f..032a38209 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
@@ -19,7 +19,6 @@ import {
shouldShowExtractedInformation,
type WorkflowRunBlock,
} from "../../types/workflowRunTypes";
-import { BlockDetailFailure } from "./shared";
type InspectorField = {
label: string;
@@ -381,6 +380,7 @@ function getSummaryFields(block: WorkflowRunBlock): Array<InspectorField> {
const fields: Array<InspectorField> = [];
pushField(fields, "Task ID", block.task_id);
pushField(fields, "Engine", block.engine);
+ pushField(fields, "Failure reason", block.failure_reason);
pushField(fields, "Executed branch", block.executed_branch_expression);
pushField(fields, "Executed branch result", block.executed_branch_result);
pushField(fields, "Executed next block", block.executed_branch_next_block);
@@ -543,13 +543,7 @@ function BlockInspector({
const outputTabLabel = showsExtractedInformation
? "Extracted Information"
: "Outputs";
- // A failed block/action opens on Summary so its failure reason (the only
- // place it now renders) is visible without hunting through tabs.
- const defaultTab = block.failure_reason
- ? "summary"
- : hasOutput
- ? "outputs"
- : "summary";
+ const defaultTab = hasOutput ? "outputs" : "summary";
const [activeTab, setActiveTab] = useState(defaultTab);
const diagnosticsTaskId = action?.task_id ?? block.task_id;
const diagnosticsStepIndex = action
@@ -605,8 +599,7 @@ function BlockInspector({
</Link>
)}
</TabsList>
- <TabsContent value="summary" className="m-0 space-y-3">
- <BlockDetailFailure block={block} />
+ <TabsContent value="summary" className="m-0">
<FieldList
fields={summaryFields}
emptyText="No additional summary data."
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 3dca42cc1..734c83f48 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -62,8 +62,8 @@ completion_contract:
completion_criteria:
- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, deliverable_confirmation_criterion_id, implicit, method_mandated, level, output_path, expected_output_value, expected_output_shape, requested_output_evidence_source, kind, terminal_action_family, classification_output_key, expected_classification, judgment_predicate, judgment_polarity_when_holds}.
-- Each item may additionally carry request_slot_source_id and request_slot_source_quote under the fresh contract described below.
-- When a criterion represents a request-owned output member, set request_slot_source_id and request_slot_source_quote to the exact source ID and smallest unique exact quote from REQUEST SLOT SOURCES below. The typed request-slot producer owns its final identity, path, plane, and pinability. Do not infer or emit pinability here.
+- Each item may additionally carry request_slot_source_id and request_slot_source_quote under the fresh contract described below. Do not emit request_slot_datum_binding; a separate typed producer owns that binding contract.
+- When a criterion represents a request-owned output member, set request_slot_source_id and request_slot_source_quote to the exact source ID and smallest unique exact quote from REQUEST SLOT SOURCES below. The server verifies source identity and exact-span uniqueness, and the typed request-slot producer owns its final identity, path, plane, and pinability. Do not infer or emit pinability here.
- Set request_slot_source_id and request_slot_source_quote to null for criteria that are not request-owned output members. Never invent, normalize, rename, or paraphrase a source quote.
- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
index b63c779cd..5fac9048d 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
@@ -1,11 +1,13 @@
Classify the request-owned output slots for the latest Workflow Copilot turn. Return only JSON.
Schema-valid examples:
-{"version":"1","slots":[]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"required_columns","plane":"definition","pinability":"shapeless_valid","antecedent_family":"unconditional"}]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"processed record count","plane":"run","pinability":"shapeless_valid","antecedent_family":"unconditional"}]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"status must equal complete","plane":"run","pinability":"pinned","antecedent_family":"unconditional"}]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"report the blocker if online submission is unavailable","plane":"run","pinability":"shapeless_valid","antecedent_family":"blocker"}]}
+{"version":"1","slots":[],"datum_bindings":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"required_columns","plane":"definition","pinability":"shapeless_valid","antecedent_family":"unconditional"}],"datum_bindings":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"processed record count","plane":"run","pinability":"shapeless_valid","antecedent_family":"unconditional"}],"datum_bindings":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"status must equal complete","plane":"run","pinability":"pinned","antecedent_family":"unconditional"}],"datum_bindings":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"report the blocker if online submission is unavailable","plane":"run","pinability":"shapeless_valid","antecedent_family":"blocker"}],"datum_bindings":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"request id","plane":"run","pinability":"shapeless_valid","antecedent_family":"unconditional"}],"datum_bindings":[{"criterion_index":0,"datum_field":"output_path","declined":false,"source_id":"u0","source_quote":"request id"}]}
+{"version":"1","slots":[],"datum_bindings":[{"criterion_index":0,"datum_field":"output_path","declined":true}]}
Rules:
- Return version exactly "1". Return an empty slots list when the user requests no output member; empty membership is an explicit contract, not an error.
@@ -24,6 +26,7 @@ Rules:
- undecidable: the source does not establish whether the output is unconditional or blocker-contingent. Never guess unconditional.
- Include every request-owned output member exactly once. A conditional fallback is still an output member even when another classifier might omit its condition. Do not emit paths, path segments, IDs, aliases, labels, counts, expected values, explanations, or unknown fields.
- Do not infer slots from sampled page shapes or non-user context. Do not normalize, rename, or paraphrase source quotes.
+- Return datum_bindings as an array. When DATUM TARGETS is empty, return an empty array. Otherwise answer every target exactly once and no others, identifying each target only by its criterion_index and datum_field. When user text names the datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":false,"source_id":<source ID>,"source_quote":<smallest unique exact quote>} whose independently declared slot names that target datum. Every accepted binding must name one exact slot in your slots array, and two targets may not bind the same slot. When no source quote actually names a target datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":true}. The server owns datum_value and criterion_outcome_sha256; never return either field. Decline instead of guessing or borrowing an adjacent datum.
Ordered, deduplicated user request sources:
```{{ request_sources }}```
@@ -33,3 +36,5 @@ Latest assistant turn (non-authoritative):
```{{ latest_assistant_turn }}```
Bounded global context (non-authoritative):
```{{ global_context }}```
+Datum targets from the immutable request-policy payload:
+```{{ datum_targets }}```
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index d41627ec7..eb67c7e64 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import asyncio
+import copy
+import hashlib
import json
import re
import time
@@ -13,6 +15,7 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.request_logging import redact_sensitive_fields
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import StructuredContext, sanitize_global_llm_context_for_prompt
@@ -22,8 +25,10 @@ from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOA
from skyvern.forge.sdk.copilot.request_slots import (
CanonicalRequestSlotV1,
RequestSlotContractV1,
+ RequestSlotDatumTargetV1,
RequestSlotProducerInputV1,
produce_request_slots,
+ request_slot_request_digest,
request_slot_source_text,
request_slot_sources,
)
@@ -347,6 +352,90 @@ class TerminalActionReconciliationV1:
terminal_action_family: TerminalActionFamily | None
+@dataclass(frozen=True)
+class RequestSlotFailureSurfaceV1:
+ version: Literal["1"]
+ reason: Literal["request_slot_failure"]
+ failure_kind: Literal["request_slot_failure"]
+ retryable: Literal[False]
+ request_slot_failure_kind: str
+
+
+RequestSlotDatumField = Literal["output_path", "classification_output_key"]
+
+
+@dataclass(frozen=True)
+class RequestSlotDatumBindingV1:
+ version: Literal["1"]
+ criterion_index: int
+ datum_field: RequestSlotDatumField
+ datum_value: str
+ source_id: str
+ source_quote: str
+
+
+@dataclass(frozen=True)
+class RequestSlotDatumBindingTargetV1:
+ criterion_index: int
+ datum_field: RequestSlotDatumField
+ datum_value: str
+ criterion_outcome_sha256: str
+
+
+@dataclass(frozen=True)
+class TrustedRequestSlotDatumBindingV1:
+ version: Literal["1"]
+ criterion_index: int
+ datum_field: RequestSlotDatumField
+ datum_value: str
+ criterion_outcome_sha256: str
+ source_id: str
+ source_quote: str
+ slot_id: str
+
+
+DatumBindingApplicationPredicate = Literal[
+ "accepted",
+ "invalid_criteria_container",
+ "invalid_contract",
+ "anchor_incoherent",
+ "anchor_not_unique",
+]
+
+
+@dataclass(frozen=True)
+class RequestSlotDatumBindingApplicationDecisionV1:
+ version: Literal["1"]
+ predicate: DatumBindingApplicationPredicate
+ criterion_index: int | None = None
+ accepted_payload: dict[str, Any] | None = None
+ trusted_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...] = ()
+
+
+AnchorCorrectionPredicate = Literal[
+ "accepted",
+ "invalid_criteria_container",
+ "criterion_count_changed",
+ "payload_semantics_changed",
+ "criteria_semantics_changed",
+ "criterion_not_object",
+ "criterion_semantics_changed",
+ "missing_corrected_anchor",
+ "original_quote_not_admissible",
+ "missing_request_slot_fields",
+ "corrected_anchor_not_admissible",
+ "unexpected_anchor_change",
+]
+
+
+@dataclass(frozen=True)
+class RequestSlotAnchorCorrectionDecisionV1:
+ version: Literal["1"]
+ predicate: AnchorCorrectionPredicate
+ criterion_index: int | None = None
+ accepted_payload: dict[str, Any] | None = None
+
+
@dataclass
class RequestPolicy:
testing_intent: str = "unspecified"
@@ -929,6 +1018,108 @@ def _request_slot_anchor_is_valid(
return start >= 0 and source.text.find(source_quote, start + 1) < 0
+def _request_slot_datum_binding(
+ item: dict[str, Any],
+ *,
+ criterion_index: int,
+) -> RequestSlotDatumBindingV1 | None:
+ raw = item.get("request_slot_datum_binding")
+ if not isinstance(raw, dict) or set(raw) != {
+ "version",
+ "criterion_index",
+ "datum_field",
+ "datum_value",
+ "source_id",
+ "source_quote",
+ }:
+ return None
+ version = raw.get("version")
+ raw_index = raw.get("criterion_index")
+ datum_field = raw.get("datum_field")
+ datum_value = raw.get("datum_value")
+ source_id = raw.get("source_id")
+ source_quote = raw.get("source_quote")
+ if (
+ version != "1"
+ or type(raw_index) is not int
+ or raw_index != criterion_index
+ or datum_field not in {"output_path", "classification_output_key"}
+ or not isinstance(datum_value, str)
+ or not datum_value
+ or not isinstance(source_id, str)
+ or not isinstance(source_quote, str)
+ or not source_quote
+ ):
+ return None
+ return RequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=raw_index,
+ datum_field=cast(RequestSlotDatumField, datum_field),
+ datum_value=datum_value,
+ source_id=source_id,
+ source_quote=source_quote,
+ )
+
+
+def _request_slot_datum_binding_is_valid(
+ item: dict[str, Any],
+ *,
+ criterion_index: int,
+ request_slot_request: RequestSlotProducerInputV1,
+) -> bool:
+ binding = _request_slot_datum_binding(item, criterion_index=criterion_index)
+ if binding is None:
+ return False
+ anchor = _request_slot_anchor(item)
+ kind = _coerce_criterion_kind(item.get("kind"))
+ if kind == "validation_classification":
+ expected_datum_field: RequestSlotDatumField | None = "classification_output_key"
+ active_datum_value = _coerce_classification_output_key(item.get("classification_output_key"))
+ elif isinstance(item.get("output_path"), str) and item["output_path"].strip():
+ expected_datum_field = "output_path"
+ active_datum_value = item[expected_datum_field].strip()
+ else:
+ expected_datum_field = None
+ active_datum_value = None
+ return (
+ binding.datum_field == expected_datum_field
+ and active_datum_value is not None
+ and item.get(binding.datum_field) == active_datum_value == binding.datum_value
+ and anchor == (binding.source_id, binding.source_quote)
+ and _request_slot_anchor_is_valid(item, request_slot_request=request_slot_request)
+ )
+
+
+def _trusted_request_slot_datum_binding_is_valid(
+ item: dict[str, Any],
+ *,
+ binding: TrustedRequestSlotDatumBindingV1,
+ criterion_index: int,
+ request_slot_request: RequestSlotProducerInputV1,
+) -> bool:
+ raw_binding = item.get("request_slot_datum_binding")
+ outcome = item.get("outcome")
+ return _type_strict_json_equal(
+ raw_binding,
+ {
+ "version": binding.version,
+ "criterion_index": binding.criterion_index,
+ "datum_field": binding.datum_field,
+ "datum_value": binding.datum_value,
+ "criterion_outcome_sha256": binding.criterion_outcome_sha256,
+ "source_id": binding.source_id,
+ "source_quote": binding.source_quote,
+ },
+ ) and (
+ binding.criterion_index == criterion_index
+ and item.get(binding.datum_field) == binding.datum_value
+ and isinstance(outcome, str)
+ and hashlib.sha256(outcome.encode()).hexdigest() == binding.criterion_outcome_sha256
+ and _request_slot_anchor(item) == (binding.source_id, binding.source_quote)
+ and _request_slot_anchor_is_valid(item, request_slot_request=request_slot_request)
+ )
+
+
def _request_slot_anchor_matches_criterion_datum(item: dict[str, Any]) -> bool:
anchor = _request_slot_anchor(item)
if anchor is None:
@@ -954,11 +1145,216 @@ def _request_slot_anchor_is_admissible(
item: dict[str, Any],
*,
request_slot_request: RequestSlotProducerInputV1,
+ criterion_index: int | None = None,
+ trusted_binding: TrustedRequestSlotDatumBindingV1 | None = None,
+ allow_embedded_binding: bool = False,
) -> bool:
- return _request_slot_anchor_is_valid(
- item,
+ if (
+ criterion_index is not None
+ and trusted_binding is not None
+ and _trusted_request_slot_datum_binding_is_valid(
+ item,
+ binding=trusted_binding,
+ criterion_index=criterion_index,
+ request_slot_request=request_slot_request,
+ )
+ ):
+ return True
+ if (
+ criterion_index is not None
+ and allow_embedded_binding
+ and _request_slot_datum_binding_is_valid(
+ item,
+ criterion_index=criterion_index,
+ request_slot_request=request_slot_request,
+ )
+ ):
+ return True
+ return _request_slot_anchor_is_valid(item, request_slot_request=request_slot_request) and (
+ _request_slot_anchor_matches_criterion_datum(item)
+ )
+
+
+def _type_strict_json_equal(left: Any, right: Any) -> bool:
+ if type(left) is not type(right):
+ return False
+ if isinstance(left, dict):
+ return left.keys() == right.keys() and all(_type_strict_json_equal(left[key], right[key]) for key in left)
+ if isinstance(left, list):
+ return len(left) == len(right) and all(
+ _type_strict_json_equal(left_item, right_item) for left_item, right_item in zip(left, right, strict=True)
+ )
+ return bool(left == right)
+
+
+def _request_slot_datum_target(
+ item: dict[str, Any],
+ *,
+ criterion_index: int,
+) -> RequestSlotDatumBindingTargetV1 | None:
+ outcome = item.get("outcome")
+ if not isinstance(outcome, str) or not outcome:
+ return None
+ kind = _coerce_criterion_kind(item.get("kind"))
+ if kind == "validation_classification":
+ datum_value = _coerce_classification_output_key(item.get("classification_output_key"))
+ datum_field: RequestSlotDatumField = "classification_output_key"
+ elif kind == "terminal_action":
+ return None
+ else:
+ raw_output_path = item.get("output_path")
+ datum_value = raw_output_path.strip() if isinstance(raw_output_path, str) and raw_output_path.strip() else None
+ datum_field = "output_path"
+ if datum_value is None:
+ return None
+ return RequestSlotDatumBindingTargetV1(
+ criterion_index=criterion_index,
+ datum_field=datum_field,
+ datum_value=datum_value,
+ criterion_outcome_sha256=hashlib.sha256(outcome.encode()).hexdigest(),
+ )
+
+
+def _request_slot_datum_binding_targets(
+ raw_criteria: Any,
+ *,
+ request_slot_request: RequestSlotProducerInputV1,
+) -> tuple[RequestSlotDatumBindingTargetV1, ...]:
+ if not isinstance(raw_criteria, list):
+ return ()
+ targets: list[RequestSlotDatumBindingTargetV1] = []
+ for criterion_index, item in enumerate(raw_criteria):
+ if not isinstance(item, dict):
+ continue
+ target = _request_slot_datum_target(item, criterion_index=criterion_index)
+ if target is None:
+ continue
+ # First-pass model assertions are not provenance. Only the unchanged legacy
+ # lexical check or a server-joined producer contract may license the datum.
+ if _request_slot_anchor_is_valid(item, request_slot_request=request_slot_request) and (
+ _request_slot_anchor_matches_criterion_datum(item)
+ ):
+ continue
+ targets.append(target)
+ return tuple(targets)
+
+
+def _apply_request_slot_datum_bindings(
+ original: dict[str, Any],
+ *,
+ request_slot_request: RequestSlotProducerInputV1,
+ request_slot_contract: RequestSlotContractV1,
+) -> RequestSlotDatumBindingApplicationDecisionV1:
+ criteria = original.get("completion_criteria")
+ if not isinstance(criteria, list):
+ return RequestSlotDatumBindingApplicationDecisionV1(version="1", predicate="invalid_criteria_container")
+ if request_slot_contract.request_digest != request_slot_request_digest(request_slot_request):
+ return RequestSlotDatumBindingApplicationDecisionV1(version="1", predicate="invalid_contract")
+
+ targets = _request_slot_datum_binding_targets(
+ criteria,
request_slot_request=request_slot_request,
- ) and _request_slot_anchor_matches_criterion_datum(item)
+ )
+ target_by_key = {(target.criterion_index, target.datum_field): target for target in targets}
+ binding_by_key = {
+ (binding.criterion_index, binding.datum_field): binding for binding in request_slot_contract.datum_bindings
+ }
+ decline_by_key = {
+ (decline.criterion_index, decline.datum_field): decline for decline in request_slot_contract.datum_declines
+ }
+ if (
+ len(binding_by_key) != len(request_slot_contract.datum_bindings)
+ or len(decline_by_key) != len(request_slot_contract.datum_declines)
+ or set(binding_by_key) & set(decline_by_key)
+ or set(binding_by_key) | set(decline_by_key) != set(target_by_key)
+ ):
+ return RequestSlotDatumBindingApplicationDecisionV1(version="1", predicate="invalid_contract")
+
+ for key, decline in decline_by_key.items():
+ target = target_by_key[key]
+ if (
+ decline.datum_value != target.datum_value
+ or decline.criterion_outcome_sha256 != target.criterion_outcome_sha256
+ ):
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="invalid_contract", criterion_index=target.criterion_index
+ )
+
+ trusted_bindings: list[TrustedRequestSlotDatumBindingV1] = []
+ for key, binding in binding_by_key.items():
+ target = target_by_key[key]
+ criterion = criteria[target.criterion_index]
+ if not isinstance(criterion, dict):
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="invalid_contract", criterion_index=target.criterion_index
+ )
+ outcome = criterion.get("outcome")
+ if (
+ binding.datum_value != target.datum_value
+ or binding.criterion_outcome_sha256 != target.criterion_outcome_sha256
+ ):
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="invalid_contract", criterion_index=target.criterion_index
+ )
+ if (
+ not isinstance(outcome, str)
+ or hashlib.sha256(outcome.encode()).hexdigest() != target.criterion_outcome_sha256
+ ):
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="anchor_incoherent", criterion_index=target.criterion_index
+ )
+ candidate = {
+ **criterion,
+ "request_slot_source_id": binding.source_id,
+ "request_slot_source_quote": binding.source_quote,
+ }
+ if not _request_slot_anchor_is_valid(candidate, request_slot_request=request_slot_request):
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="anchor_not_unique", criterion_index=target.criterion_index
+ )
+ resolved_slot = _request_slot_for_anchor(
+ (binding.source_id, binding.source_quote),
+ request_slot_request=request_slot_request,
+ request_slot_contract=request_slot_contract,
+ )
+ if resolved_slot is None or binding.slot_id != resolved_slot.slot_id:
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1", predicate="invalid_contract", criterion_index=target.criterion_index
+ )
+ trusted_bindings.append(
+ TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ source_id=binding.source_id,
+ source_quote=binding.source_quote,
+ slot_id=binding.slot_id,
+ )
+ )
+
+ accepted_payload = copy.deepcopy(original)
+ accepted_criteria = accepted_payload["completion_criteria"]
+ for trusted_binding in trusted_bindings:
+ accepted_item = accepted_criteria[trusted_binding.criterion_index]
+ accepted_item["request_slot_source_id"] = trusted_binding.source_id
+ accepted_item["request_slot_source_quote"] = trusted_binding.source_quote
+ accepted_item["request_slot_datum_binding"] = {
+ "version": trusted_binding.version,
+ "criterion_index": trusted_binding.criterion_index,
+ "datum_field": trusted_binding.datum_field,
+ "datum_value": trusted_binding.datum_value,
+ "criterion_outcome_sha256": trusted_binding.criterion_outcome_sha256,
+ "source_id": trusted_binding.source_id,
+ "source_quote": trusted_binding.source_quote,
+ }
+ return RequestSlotDatumBindingApplicationDecisionV1(
+ version="1",
+ predicate="accepted",
+ accepted_payload=accepted_payload,
+ trusted_bindings=tuple(trusted_bindings),
+ )
def _request_slot_claims_need_anchor_correction(
@@ -969,8 +1365,12 @@ def _request_slot_claims_need_anchor_correction(
return isinstance(raw_criteria, list) and any(
isinstance(item, dict)
and _item_claims_request_slot(item)
- and not _request_slot_anchor_is_admissible(item, request_slot_request=request_slot_request)
- for item in raw_criteria
+ and not _request_slot_anchor_is_admissible(
+ item,
+ request_slot_request=request_slot_request,
+ criterion_index=criterion_index,
+ )
+ for criterion_index, item in enumerate(raw_criteria)
)
@@ -979,17 +1379,17 @@ def _accept_request_slot_anchor_correction(
corrected: dict[str, Any],
*,
request_slot_request: RequestSlotProducerInputV1,
-) -> dict[str, Any] | None:
+) -> RequestSlotAnchorCorrectionDecisionV1:
original_criteria = original.get("completion_criteria")
corrected_criteria = corrected.get("completion_criteria")
if not isinstance(original_criteria, list) or not isinstance(corrected_criteria, list):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(version="1", predicate="invalid_criteria_container")
if len(original_criteria) != len(corrected_criteria):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(version="1", predicate="criterion_count_changed")
anchor_fields = {"request_slot_source_id", "request_slot_source_quote"}
if _classifier_payload_semantics(original) != _classifier_payload_semantics(corrected):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(version="1", predicate="payload_semantics_changed")
original_semantic_criteria = [
{key: value for key, value in item.items() if key not in anchor_fields}
for item in original_criteria
@@ -1003,24 +1403,40 @@ def _accept_request_slot_anchor_correction(
if _parse_completion_criteria(original_semantic_criteria, emit_mint_events=False) != _parse_completion_criteria(
corrected_semantic_criteria, emit_mint_events=False
):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(version="1", predicate="criteria_semantics_changed")
accepted_criteria: list[dict[str, Any]] = []
- for original_item, corrected_item in zip(original_criteria, corrected_criteria, strict=True):
+ binding_absent = object()
+ for criterion_index, (original_item, corrected_item) in enumerate(
+ zip(original_criteria, corrected_criteria, strict=True)
+ ):
if not isinstance(original_item, dict) or not isinstance(corrected_item, dict):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="criterion_not_object", criterion_index=criterion_index
+ )
+ if not _type_strict_json_equal(
+ original_item.get("request_slot_datum_binding", binding_absent),
+ corrected_item.get("request_slot_datum_binding", binding_absent),
+ ):
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="criterion_semantics_changed", criterion_index=criterion_index
+ )
original_without_anchors = {key: value for key, value in original_item.items() if key not in anchor_fields}
corrected_without_anchors = {key: value for key, value in corrected_item.items() if key not in anchor_fields}
if _classifier_criterion_semantics(original_without_anchors) != _classifier_criterion_semantics(
corrected_without_anchors
):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="criterion_semantics_changed", criterion_index=criterion_index
+ )
accepted_item = dict(original_item)
if _item_claims_request_slot(original_item):
original_quote = original_item.get("request_slot_source_quote")
corrected_anchor = _request_slot_anchor(corrected_item)
if corrected_anchor is None:
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="missing_corrected_anchor", criterion_index=criterion_index
+ )
corrected_source_id, corrected_quote = corrected_anchor
if isinstance(original_quote, str) and original_quote:
corrected_source_for_original_quote = {
@@ -1030,12 +1446,18 @@ def _accept_request_slot_anchor_correction(
if not _request_slot_anchor_is_admissible(
corrected_source_for_original_quote,
request_slot_request=request_slot_request,
+ criterion_index=criterion_index,
+ allow_embedded_binding=True,
):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="original_quote_not_admissible", criterion_index=criterion_index
+ )
accepted_item["request_slot_source_id"] = corrected_source_id
else:
if not _item_claims_request_slot_fields(original_item):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="missing_request_slot_fields", criterion_index=criterion_index
+ )
corrected_anchor_for_original_datum = {
**original_item,
"request_slot_source_id": corrected_source_id,
@@ -1044,14 +1466,66 @@ def _accept_request_slot_anchor_correction(
if not _request_slot_anchor_is_admissible(
corrected_anchor_for_original_datum,
request_slot_request=request_slot_request,
+ criterion_index=criterion_index,
+ allow_embedded_binding=True,
):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="corrected_anchor_not_admissible", criterion_index=criterion_index
+ )
accepted_item["request_slot_source_id"] = corrected_source_id
accepted_item["request_slot_source_quote"] = corrected_quote
elif any(corrected_item.get(field) != original_item.get(field) for field in anchor_fields):
- return None
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1", predicate="unexpected_anchor_change", criterion_index=criterion_index
+ )
accepted_criteria.append(accepted_item)
- return {**original, "completion_criteria": accepted_criteria}
+ return RequestSlotAnchorCorrectionDecisionV1(
+ version="1",
+ predicate="accepted",
+ accepted_payload={**original, "completion_criteria": accepted_criteria},
+ )
+
+
+def _redact_anchor_correction_capture_value(value: Any) -> Any:
+ if isinstance(value, str):
+ return redact_raw_secrets_for_prompt(value)
+ if isinstance(value, dict):
+ redacted: dict[Any, Any] = {}
+ for key, item in value.items():
+ if key == "raw_secret_evidence":
+ redacted[key] = None if item is None else "[REDACTED_SECRET]"
+ else:
+ redacted[key] = _redact_anchor_correction_capture_value(item)
+ return redacted
+ if isinstance(value, list):
+ return [_redact_anchor_correction_capture_value(item) for item in value]
+ return value
+
+
+def _anchor_correction_rejection_capture(
+ original: dict[str, Any],
+ corrected: dict[str, Any],
+) -> dict[str, str] | None:
+ try:
+ original_redacted = _redact_anchor_correction_capture_value(redact_sensitive_fields(original))
+ corrected_redacted = _redact_anchor_correction_capture_value(redact_sensitive_fields(corrected))
+ original_json = json.dumps(original_redacted, ensure_ascii=True, separators=(",", ":"), sort_keys=True)
+ corrected_json = json.dumps(corrected_redacted, ensure_ascii=True, separators=(",", ":"), sort_keys=True)
+ pair_json = json.dumps(
+ {"corrected": corrected_redacted, "original": original_redacted},
+ ensure_ascii=True,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
+ except (TypeError, ValueError, RecursionError):
+ return None
+ return {
+ "original_payload_json": original_json,
+ "corrected_payload_json": corrected_json,
+ "original_sha256": hashlib.sha256(original_json.encode()).hexdigest(),
+ "corrected_sha256": hashlib.sha256(corrected_json.encode()).hexdigest(),
+ "pair_sha256": hashlib.sha256(pair_json.encode()).hexdigest(),
+ }
def _classifier_payload_semantics(payload: dict[str, Any]) -> RequestPolicy:
@@ -1200,12 +1674,16 @@ def _parse_fresh_request_slot_criteria(
*,
request_slot_request: RequestSlotProducerInputV1,
request_slot_contract: RequestSlotContractV1,
+ trusted_datum_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...] = (),
) -> list[CompletionCriterion]:
if request_slot_contract.version != "1" or not isinstance(raw, list):
return []
+ criterion_index_by_item_id = {id(item): index for index, item in enumerate(raw) if isinstance(item, dict)}
+ trusted_binding_by_index = {binding.criterion_index: binding for binding in trusted_datum_bindings}
bound_by_slot_id: dict[str, CompletionCriterion] = {}
non_slot_criteria: list[CompletionCriterion] = []
for item, criterion in _parse_completion_criterion_entries(raw):
+ criterion_index = criterion_index_by_item_id[id(item)]
anchor = _request_slot_anchor(item)
slot = (
_request_slot_for_anchor(
@@ -1213,7 +1691,13 @@ def _parse_fresh_request_slot_criteria(
request_slot_request=request_slot_request,
request_slot_contract=request_slot_contract,
)
- if _request_slot_anchor_is_admissible(item, request_slot_request=request_slot_request)
+ if _request_slot_anchor_is_admissible(
+ item,
+ request_slot_request=request_slot_request,
+ criterion_index=criterion_index,
+ trusted_binding=trusted_binding_by_index.get(criterion_index),
+ allow_embedded_binding=False,
+ )
else None
)
if anchor is None or slot is None:
@@ -1268,6 +1752,7 @@ def _classification_from_raw(
request_slot_request: RequestSlotProducerInputV1 | None = None,
request_slot_contract: RequestSlotContractV1 | None = None,
request_slot_failure_kind: str | None = None,
+ trusted_datum_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...] = (),
) -> RequestPolicy:
raw = _coerce_classifier_payload(raw)
if raw is None:
@@ -1288,6 +1773,7 @@ def _classification_from_raw(
raw_criteria,
request_slot_request=request_slot_request,
request_slot_contract=request_slot_contract,
+ trusted_datum_bindings=trusted_datum_bindings,
)
elif claims_request_slots:
completion_criteria = _fresh_request_slot_failure_criteria(raw_criteria)
@@ -2720,6 +3206,26 @@ def _log_terminal_action_reconciliation_omitted(
failure_kind: str = "none",
retry_count: int = 0,
) -> None:
+ request_slot_failure_kind = policy.request_slot_failure_kind
+ if request_slot_failure_kind:
+ surface = RequestSlotFailureSurfaceV1(
+ version="1",
+ reason="request_slot_failure",
+ failure_kind="request_slot_failure",
+ retryable=False,
+ request_slot_failure_kind=request_slot_failure_kind,
+ )
+ LOG.info(
+ "copilot_terminal_action_reconciliation_omitted",
+ reason=surface.reason,
+ failure_kind=surface.failure_kind,
+ retryable=surface.retryable,
+ request_slot_failure_kind=surface.request_slot_failure_kind,
+ retry_count=retry_count,
+ credential_input_kind=policy.credential_input_kind,
+ completion_criteria_count=len(policy.completion_criteria),
+ )
+ return
LOG.info(
"copilot_terminal_action_reconciliation_omitted",
reason=reason,
@@ -2929,10 +3435,79 @@ async def _classify_request(
requested_output_path_aliases=requested_output_path_aliases,
)
+ # The primary classifier may describe candidate criteria, but it cannot mint
+ # provenance for itself. Datum bindings become trusted only after the separate
+ # producer-owned envelope is mechanically joined below.
+ raw_payload = copy.deepcopy(raw_payload)
+ raw_criteria = raw_payload.get("completion_criteria")
+ if isinstance(raw_criteria, list):
+ for item in raw_criteria:
+ if isinstance(item, dict):
+ item.pop("request_slot_datum_binding", None)
+
request_slot_contract: RequestSlotContractV1 | None = None
request_slot_failure_kind: str | None = None
- raw_criteria = raw_payload.get("completion_criteria")
- if _request_slot_claims_need_anchor_correction(
+ trusted_datum_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...] = ()
+ datum_binding_targets = _request_slot_datum_binding_targets(
+ raw_criteria,
+ request_slot_request=request_slot_request,
+ )
+ if datum_binding_targets:
+ targeted_request_data = request_slot_request.model_dump()
+ try:
+ targeted_request_data["datum_targets"] = tuple(
+ RequestSlotDatumTargetV1(
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ )
+ for target in datum_binding_targets
+ )
+ targeted_request = RequestSlotProducerInputV1.model_validate(targeted_request_data)
+ except ValueError:
+ targeted_request = None
+ request_slot_failure_kind = "invalid_output"
+ LOG.warning(
+ "request-policy request-slot target construction failed",
+ failure_kind=request_slot_failure_kind,
+ target_count=len(datum_binding_targets),
+ )
+ if targeted_request is not None:
+ request_slot_result = await produce_request_slots(request=targeted_request, handler=handler)
+ if request_slot_result.status == "success":
+ request_slot_contract = request_slot_result.contract
+ else:
+ request_slot_failure_kind = (
+ request_slot_result.failure_kind.value
+ if request_slot_result.failure_kind is not None
+ else "unknown"
+ )
+ LOG.warning(
+ "request-policy request-slot producer failed",
+ failure_kind=request_slot_failure_kind,
+ attempts=request_slot_result.attempts,
+ )
+ if request_slot_contract is not None:
+ datum_binding_decision = _apply_request_slot_datum_bindings(
+ raw_payload,
+ request_slot_request=targeted_request,
+ request_slot_contract=request_slot_contract,
+ )
+ if datum_binding_decision.accepted_payload is not None:
+ raw_payload = datum_binding_decision.accepted_payload
+ raw_criteria = raw_payload.get("completion_criteria")
+ trusted_datum_bindings = datum_binding_decision.trusted_bindings
+ else:
+ request_slot_failure_kind = "invalid_anchor_correction"
+ request_slot_contract = None
+ LOG.warning(
+ "copilot_request_slot_datum_binding_contract_rejected",
+ event_version="1",
+ predicate=datum_binding_decision.predicate,
+ criterion_index=datum_binding_decision.criterion_index,
+ )
+ elif _request_slot_claims_need_anchor_correction(
raw_criteria,
request_slot_request=request_slot_request,
):
@@ -2946,7 +3521,7 @@ async def _classify_request(
)
retry_count += correction_retry_count
corrected_payload = _coerce_classifier_payload(corrected_raw)
- accepted_payload = (
+ anchor_correction_decision = (
_accept_request_slot_anchor_correction(
raw_payload,
corrected_payload,
@@ -2955,16 +3530,36 @@ async def _classify_request(
if corrected_payload is not None
else None
)
+ accepted_payload = (
+ anchor_correction_decision.accepted_payload if anchor_correction_decision is not None else None
+ )
if accepted_payload is not None:
raw_payload = accepted_payload
raw_criteria = raw_payload.get("completion_criteria")
else:
request_slot_failure_kind = "invalid_anchor_correction"
+ capture = (
+ _anchor_correction_rejection_capture(raw_payload, corrected_payload)
+ if corrected_payload is not None
+ else None
+ )
+ if anchor_correction_decision is not None and capture is not None:
+ LOG.warning(
+ "copilot_request_slot_anchor_correction_rejected",
+ event_version="1",
+ predicate=anchor_correction_decision.predicate,
+ criterion_index=anchor_correction_decision.criterion_index,
+ **capture,
+ )
LOG.warning(
"request-policy request-slot anchor correction failed",
failure_kind=correction_failure_kind,
+ predicate=(anchor_correction_decision.predicate if anchor_correction_decision is not None else None),
+ criterion_index=(
+ anchor_correction_decision.criterion_index if anchor_correction_decision is not None else None
+ ),
)
- if request_slot_failure_kind is None:
+ if request_slot_failure_kind is None and request_slot_contract is None:
# The independent producer owns slot membership as well as type. Run it even
# when the primary classifier declares no slots so an omitted conditional
# fallback cannot disappear from the completion contract.
@@ -2985,6 +3580,7 @@ async def _classify_request(
request_slot_request=request_slot_request,
request_slot_contract=request_slot_contract,
request_slot_failure_kind=request_slot_failure_kind,
+ trusted_datum_bindings=trusted_datum_bindings,
)
policy.classifier_retry_count = retry_count
policy.classifier_non_runtime_requested_output_evidence_sources = sorted(
diff --git a/skyvern/forge/sdk/copilot/request_slots.py b/skyvern/forge/sdk/copilot/request_slots.py
index 5a0055726..86fb9f027 100644
--- a/skyvern/forge/sdk/copilot/request_slots.py
+++ b/skyvern/forge/sdk/copilot/request_slots.py
@@ -62,6 +62,15 @@ class RequestSlotProducerFailureKind(StrEnum):
INCONSISTENT_OUTPUT = "inconsistent_output"
+class RequestSlotDatumTargetV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ criterion_index: int = Field(ge=0, le=63)
+ datum_field: Literal["output_path", "classification_output_key"]
+ datum_value: str = Field(min_length=1, max_length=256)
+ criterion_outcome_sha256: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
+
+
class RequestSlotProducerInputV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@@ -73,6 +82,7 @@ class RequestSlotProducerInputV1(BaseModel):
latest_assistant_turn: str = Field(max_length=4_096)
retained_history: tuple[str, ...] = Field(max_length=8)
global_context: str = Field(max_length=32_768)
+ datum_targets: tuple[RequestSlotDatumTargetV1, ...] = Field(default=(), max_length=64)
@field_validator("retained_history")
@classmethod
@@ -81,6 +91,16 @@ class RequestSlotProducerInputV1(BaseModel):
raise ValueError("retained history entries must be non-empty and at most 4096 characters")
return entries
+ @field_validator("datum_targets")
+ @classmethod
+ def _validate_datum_targets(
+ cls, targets: tuple[RequestSlotDatumTargetV1, ...]
+ ) -> tuple[RequestSlotDatumTargetV1, ...]:
+ identities = [(target.criterion_index, target.datum_field) for target in targets]
+ if len(set(identities)) != len(identities):
+ raise ValueError("datum targets must have unique criterion/field identities")
+ return targets
+
class RequestSlotSourceV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@@ -113,22 +133,66 @@ class RequestSlotDeclarationV1(BaseModel):
return quote
+class RequestSlotDatumBindingDeclarationV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ criterion_index: int = Field(ge=0, le=63)
+ datum_field: Literal["output_path", "classification_output_key"]
+ declined: Literal[False]
+ source_id: str = Field(min_length=2, max_length=3, pattern=_SOURCE_ID_PATTERN.pattern)
+ source_quote: str = Field(min_length=1, max_length=_MAX_SOURCE_QUOTE_CHARS)
+
+ @field_validator("source_quote")
+ @classmethod
+ def _validate_source_quote(cls, quote: str) -> str:
+ if not quote.strip():
+ raise ValueError("source_quote must contain non-whitespace request text")
+ return quote
+
+
+class RequestSlotDatumDeclineDeclarationV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ criterion_index: int = Field(ge=0, le=63)
+ datum_field: Literal["output_path", "classification_output_key"]
+ declined: Literal[True]
+
+
class RequestSlotEnvelopeV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
version: Literal["1"]
slots: tuple[RequestSlotDeclarationV1, ...] = Field(max_length=64)
+ datum_bindings: tuple[RequestSlotDatumBindingDeclarationV1 | RequestSlotDatumDeclineDeclarationV1, ...] = Field(
+ default=(), max_length=64
+ )
-def _request_digest(version: str, sources: tuple[RequestSlotSourceV1, ...]) -> str:
+def _request_digest(
+ version: str,
+ sources: tuple[RequestSlotSourceV1, ...],
+ datum_targets: tuple[RequestSlotDatumTargetV1, ...],
+) -> str:
+ digest_payload: list[object] = [
+ version,
+ [[source.source_id, source.text] for source in sources],
+ ]
+ # Preserve persisted slot identities for legacy/unbound contracts while making
+ # targeted producer contracts identity-complete for replay and caching.
+ if datum_targets:
+ digest_payload.append([target.model_dump(mode="json") for target in datum_targets])
encoded = json.dumps(
- [version, [[source.source_id, source.text] for source in sources]],
+ digest_payload,
ensure_ascii=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(encoded).hexdigest()
+def request_slot_request_digest(request: RequestSlotProducerInputV1) -> str:
+ return _request_digest(request.version, request_slot_sources(request), request.datum_targets)
+
+
def _canonical_path(request_digest: str, ordinal: int) -> tuple[str, tuple[str, str]]:
leaf = f"request_slot_{request_digest[:48]}_{ordinal:02d}"
return f"output.{leaf}", ("output", leaf)
@@ -175,6 +239,27 @@ class CanonicalRequestSlotV1(BaseModel):
return self
+class CanonicalRequestSlotDatumBindingV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ criterion_index: int = Field(ge=0, le=63)
+ datum_field: Literal["output_path", "classification_output_key"]
+ datum_value: str = Field(min_length=1, max_length=256)
+ criterion_outcome_sha256: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
+ source_id: str = Field(min_length=2, max_length=3, pattern=_SOURCE_ID_PATTERN.pattern)
+ source_quote: str = Field(min_length=1, max_length=_MAX_SOURCE_QUOTE_CHARS)
+ slot_id: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
+
+
+class CanonicalRequestSlotDatumDeclineV1(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+ criterion_index: int = Field(ge=0, le=63)
+ datum_field: Literal["output_path", "classification_output_key"]
+ datum_value: str = Field(min_length=1, max_length=256)
+ criterion_outcome_sha256: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
+
+
class RequestSlotContractV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@@ -182,6 +267,8 @@ class RequestSlotContractV1(BaseModel):
request_digest: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
slots: tuple[CanonicalRequestSlotV1, ...] = Field(max_length=64)
count: int = Field(ge=0, le=64)
+ datum_bindings: tuple[CanonicalRequestSlotDatumBindingV1, ...] = Field(default=(), max_length=64)
+ datum_declines: tuple[CanonicalRequestSlotDatumDeclineV1, ...] = Field(default=(), max_length=64)
@model_validator(mode="after")
def _validate_derived_membership(self) -> RequestSlotContractV1:
@@ -193,6 +280,19 @@ class RequestSlotContractV1(BaseModel):
raise ValueError("slot membership contains a canonical path collision")
if len({slot.slot_id for slot in self.slots}) != self.count:
raise ValueError("slot membership contains a canonical slot identity collision")
+ if len({binding.slot_id for binding in self.datum_bindings}) != len(self.datum_bindings):
+ raise ValueError("datum bindings must map injectively to canonical slots")
+ binding_identities = [(binding.criterion_index, binding.datum_field) for binding in self.datum_bindings]
+ if len(set(binding_identities)) != len(binding_identities):
+ raise ValueError("datum bindings must have unique criterion/field identities")
+ decline_identities = [(decline.criterion_index, decline.datum_field) for decline in self.datum_declines]
+ if len(set(decline_identities)) != len(decline_identities):
+ raise ValueError("datum declines must have unique criterion/field identities")
+ if set(binding_identities) & set(decline_identities):
+ raise ValueError("datum targets cannot be both bound and declined")
+ slot_ids = {slot.slot_id for slot in self.slots}
+ if any(binding.slot_id not in slot_ids for binding in self.datum_bindings):
+ raise ValueError("datum binding references an unknown canonical slot")
previous: CanonicalRequestSlotV1 | None = None
for slot in self.slots:
@@ -305,6 +405,18 @@ def canonicalize_request_slots(
raise ValueError("request and response contract versions must match")
sources = request_slot_sources(request)
source_by_id = {source.source_id: source for source in sources}
+ target_by_identity = {(target.criterion_index, target.datum_field): target for target in request.datum_targets}
+ if len(envelope.datum_bindings) != len(target_by_identity):
+ raise ValueError("datum bindings must cover every requested target exactly once")
+ resolution_by_identity: dict[
+ tuple[int, str], RequestSlotDatumBindingDeclarationV1 | RequestSlotDatumDeclineDeclarationV1
+ ] = {}
+ for resolution in envelope.datum_bindings:
+ identity = (resolution.criterion_index, resolution.datum_field)
+ target = target_by_identity.get(identity)
+ if target is None or identity in resolution_by_identity:
+ raise ValueError("datum resolution has an unknown or duplicate target")
+ resolution_by_identity[identity] = resolution
resolved: list[tuple[RequestSlotDeclarationV1, RequestSlotSourceV1, int, int]] = []
for declaration in envelope.slots:
source = source_by_id.get(declaration.source_id)
@@ -324,7 +436,7 @@ def canonicalize_request_slots(
raise ValueError(f"source anchors overlap in {item[1].source_id}")
previous = item
- digest = _request_digest(envelope.version, sources)
+ digest = request_slot_request_digest(request)
canonical_slots: list[CanonicalRequestSlotV1] = []
seen_paths: set[str] = set()
seen_slot_ids: set[str] = set()
@@ -351,11 +463,53 @@ def canonicalize_request_slots(
antecedent_family=declaration.antecedent_family,
)
)
+ slot_by_anchor = {(slot.source_id, slot.source_start, slot.source_end): slot for slot in canonical_slots}
+ canonical_bindings: list[CanonicalRequestSlotDatumBindingV1] = []
+ canonical_declines: list[CanonicalRequestSlotDatumDeclineV1] = []
+ seen_binding_slot_ids: set[str] = set()
+ for target in request.datum_targets:
+ resolution = resolution_by_identity[(target.criterion_index, target.datum_field)]
+ if isinstance(resolution, RequestSlotDatumDeclineDeclarationV1):
+ canonical_declines.append(
+ CanonicalRequestSlotDatumDeclineV1(
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ )
+ )
+ continue
+ binding = resolution
+ source = source_by_id.get(binding.source_id)
+ if source is None:
+ raise ValueError(f"unknown datum-binding source: {binding.source_id}")
+ start = source.text.find(binding.source_quote)
+ if start < 0 or source.text.find(binding.source_quote, start + 1) >= 0:
+ raise ValueError("datum-binding source quote must be present exactly once")
+ slot = slot_by_anchor.get((binding.source_id, start, start + len(binding.source_quote)))
+ if slot is None:
+ raise ValueError("datum binding must name one exact independently produced request slot")
+ if slot.slot_id in seen_binding_slot_ids:
+ raise ValueError("datum bindings must map injectively to canonical slots")
+ seen_binding_slot_ids.add(slot.slot_id)
+ canonical_bindings.append(
+ CanonicalRequestSlotDatumBindingV1(
+ criterion_index=binding.criterion_index,
+ datum_field=binding.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ source_id=binding.source_id,
+ source_quote=binding.source_quote,
+ slot_id=slot.slot_id,
+ )
+ )
return RequestSlotContractV1(
version=envelope.version,
request_digest=digest,
slots=tuple(canonical_slots),
count=len(canonical_slots),
+ datum_bindings=tuple(canonical_bindings),
+ datum_declines=tuple(canonical_declines),
)
@@ -372,6 +526,11 @@ def _render_prompt(request: RequestSlotProducerInputV1) -> str:
workflow_context=_safe_prompt_text(request.workflow_context, _MAX_WORKFLOW_PROMPT_CHARS),
latest_assistant_turn=_safe_prompt_text(request.latest_assistant_turn, _MAX_TRANSCRIPT_ANCHOR_PROMPT_CHARS),
global_context=_safe_prompt_text(safe_global_context, _MAX_GLOBAL_CONTEXT_PROMPT_CHARS),
+ datum_targets=json.dumps(
+ [target.model_dump(mode="json") for target in request.datum_targets],
+ ensure_ascii=True,
+ separators=(",", ":"),
+ ),
)
@@ -393,7 +552,7 @@ def request_slot_contracts_agree(
) -> bool:
if first.version != second.version or first.request_digest != second.request_digest or first.count != second.count:
return False
- return all(
+ slots_agree = all(
first_slot.ordinal == second_slot.ordinal
and first_slot.source_id == second_slot.source_id
and first_slot.plane == second_slot.plane
@@ -403,6 +562,15 @@ def request_slot_contracts_agree(
and second_slot.source_start < first_slot.source_end
for first_slot, second_slot in zip(first.slots, second.slots, strict=True)
)
+ bindings_agree = len(first.datum_bindings) == len(second.datum_bindings) and all(
+ first_binding.criterion_index == second_binding.criterion_index
+ and first_binding.datum_field == second_binding.datum_field
+ and first_binding.datum_value == second_binding.datum_value
+ and first_binding.criterion_outcome_sha256 == second_binding.criterion_outcome_sha256
+ and first_binding.slot_id == second_binding.slot_id
+ for first_binding, second_binding in zip(first.datum_bindings, second.datum_bindings, strict=True)
+ )
+ return slots_agree and bindings_agree and first.datum_declines == second.datum_declines
async def produce_request_slots(
diff --git a/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_clean_retry_custody_gap.json b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_clean_retry_custody_gap.json
new file mode 100644
index 000000000..51a597edd
--- /dev/null
+++ b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_clean_retry_custody_gap.json
@@ -0,0 +1,17 @@
+{
+ "version": "1",
+ "workflow_permanent_id": "wpid_553704674845352182",
+ "observed_at": "2026-07-21T02:57:20.557924Z",
+ "event": "copilot_request_slot_datum_binding_correction_rejected",
+ "predicate": "target_coverage_mismatch",
+ "criterion_index": 2,
+ "original_payload_sha256": "9f3a7ee247f6c15f677c68402930a7a5c3d23f334b5719442198127724c00cab",
+ "raw_original_payload_retained": false,
+ "raw_correction_payload_retained": false,
+ "retained_evidence": [
+ ".skyvern-agentic-qa/evidence/sky-12671-p8-clean-retry-4678cd58/visual-monitor/journey.md",
+ ".skyvern-agentic-qa/evidence/sky-12671-p8-clean-retry-4678cd58/visual-monitor/final-report.md"
+ ],
+ "custody_status": "exact_replay_unavailable",
+ "custody_note": "The clean retry retained the typed rejection event and visual evidence, but not the original or correction payload bytes. This record must not be represented as an exact replay fixture."
+}
diff --git a/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_noop_anchor_correction_pair.json b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_noop_anchor_correction_pair.json
new file mode 100644
index 000000000..e329b9e5c
--- /dev/null
+++ b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_noop_anchor_correction_pair.json
@@ -0,0 +1,359 @@
+{
+ "corrected": {
+ "authoring_intent": "author_now",
+ "clarification_reason": "none",
+ "completion_contract": null,
+ "completion_criteria": [
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": null,
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "definition",
+ "method_mandated": false,
+ "outcome": "the workflow accepts reusable inputs for the service address parts, desired start date, and business name",
+ "output_path": null,
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "The service address parts, desired start date, and business name should be reusable inputs",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": null,
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "terminal_action",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "a QuickConnect request exists in the My QuickConnects table",
+ "output_path": null,
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "creates or verifies a gas QuickConnect request",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": "request"
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "reference_code",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the request id is output",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "output the request id",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "address",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the provider-captured address is output",
+ "output_path": "output.provider_captured_address",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "provider-captured address",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "date",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the requested date is output",
+ "output_path": "output.requested_date",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "requested date",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "status_label",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the request status is output",
+ "output_path": "output.status",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "status",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "status_label",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "whether the request was newly submitted or already present is output",
+ "output_path": "output.request_submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "whether the request was newly submitted or already present",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ }
+ ],
+ "credential_input_kind": "credential_name",
+ "credential_refs": [
+ "mock-portal-login"
+ ],
+ "login_page_urls": [
+ "http://localhost:8900/utility_services/peach_gas/"
+ ],
+ "raw_secret_evidence": null,
+ "raw_secret_handling": "none",
+ "requires_user_clarification": false,
+ "testing_intent": "require_test"
+ },
+ "latest_request": "I need a reusable workflow that creates or verifies a gas QuickConnect request in the mock Peach State Gas GasConnect Hub at http://localhost:8900/utility_services/peach_gas/. Sign in with the saved credential named mock-portal-login. The service address parts, desired start date, and business name should be reusable inputs; for this run use 77 Gaslight Way, Decatur, GA 30030, 2026-06-24, and Example Realty Labs Inc. After sign-in, the workflow should open the QuickConnect create flow, use only the sourced address/date/business data, submit only when the exact address is visible, then read the My QuickConnects table and output the request id, provider-captured address, requested date, status, and whether the request was newly submitted or already present.",
+ "original": {
+ "authoring_intent": "author_now",
+ "clarification_reason": "none",
+ "completion_contract": null,
+ "completion_criteria": [
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": null,
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "definition",
+ "method_mandated": false,
+ "outcome": "the workflow accepts reusable inputs for the service address parts, desired start date, and business name",
+ "output_path": null,
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "The service address parts, desired start date, and business name should be reusable inputs",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": null,
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "terminal_action",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "a QuickConnect request exists in the My QuickConnects table",
+ "output_path": null,
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "creates or verifies a gas QuickConnect request",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": "request"
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "reference_code",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the request id is output",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "output the request id",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "address",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the provider-captured address is output",
+ "output_path": "output.provider_captured_address",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "provider-captured address",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "date",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the requested date is output",
+ "output_path": "output.requested_date",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "requested date",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "status_label",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "the request status is output",
+ "output_path": "output.status",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "status",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ },
+ {
+ "classification_output_key": null,
+ "contingent_antecedent_output_path": null,
+ "contingent_on": null,
+ "deliverable_confirmation_criterion_id": null,
+ "deliverable_kind": null,
+ "expected_classification": null,
+ "expected_output_shape": "status_label",
+ "expected_output_value": null,
+ "implicit": false,
+ "judgment_polarity_when_holds": null,
+ "judgment_predicate": null,
+ "kind": "outcome",
+ "level": "run",
+ "method_mandated": false,
+ "outcome": "whether the request was newly submitted or already present is output",
+ "output_path": "output.request_submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "whether the request was newly submitted or already present",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal_action_family": null
+ }
+ ],
+ "credential_input_kind": "credential_name",
+ "credential_refs": [
+ "mock-portal-login"
+ ],
+ "login_page_urls": [
+ "http://localhost:8900/utility_services/peach_gas/"
+ ],
+ "raw_secret_evidence": null,
+ "raw_secret_handling": "none",
+ "requires_user_clarification": false,
+ "testing_intent": "require_test"
+ },
+ "schema_version": 1,
+ "source": {
+ "backend_log_start_offset": 337104,
+ "captured_at": "2026-07-20T21:41:53.197485Z",
+ "corrected_sha256": "bec6a09bb7e11398d24bcca40206a84ef1954729b56e011a00e20d21daf481fb",
+ "criterion_index": 6,
+ "original_sha256": "bec6a09bb7e11398d24bcca40206a84ef1954729b56e011a00e20d21daf481fb",
+ "pair_sha256": "e5be31a1cc861dd40c26a9c4df5d0240e6fabe42ca37bc1c5e2bcedbe56fbac5",
+ "predicate": "original_quote_not_admissible",
+ "ticket": "SKY-12671",
+ "workflow_permanent_id": "wpid_553623387522932594"
+ }
+}
diff --git a/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_original_payload.json b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_original_payload.json
new file mode 100644
index 000000000..2a3f56e12
--- /dev/null
+++ b/tests/unit/fixtures/copilot/request_policy/sky_12671_p8_original_payload.json
@@ -0,0 +1 @@
+{"authoring_intent":"author_now","clarification_reason":"none","completion_contract":null,"completion_criteria":[{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":null,"expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"definition","method_mandated":false,"outcome":"the workflow accepts reusable inputs for the service address parts, desired start date, and business name","output_path":null,"request_slot_source_id":"u0","request_slot_source_quote":"The service address parts, desired start date, and business name should be reusable inputs","requested_output_evidence_source":"runtime_output","terminal_action_family":null},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":null,"expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"terminal_action","level":"run","method_mandated":false,"outcome":"a QuickConnect request exists in the My QuickConnects table","output_path":null,"request_slot_source_id":"u0","request_slot_source_quote":"creates or verifies a gas QuickConnect request","requested_output_evidence_source":"runtime_output","terminal_action_family":"request"},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":"reference_code","expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"run","method_mandated":false,"outcome":"the request id is output","output_path":"output.request_id","request_slot_source_id":"u0","request_slot_source_quote":"output the request id","requested_output_evidence_source":"runtime_output","terminal_action_family":null},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":"address","expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"run","method_mandated":false,"outcome":"the provider-captured address is output","output_path":"output.provider_captured_address","request_slot_source_id":"u0","request_slot_source_quote":"provider-captured address","requested_output_evidence_source":"runtime_output","terminal_action_family":null},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":"date","expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"run","method_mandated":false,"outcome":"the requested date is output","output_path":"output.requested_date","request_slot_source_id":"u0","request_slot_source_quote":"requested date","requested_output_evidence_source":"runtime_output","terminal_action_family":null},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":"status_label","expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"run","method_mandated":false,"outcome":"the request status is output","output_path":"output.status","request_slot_source_id":"u0","request_slot_source_quote":"status","requested_output_evidence_source":"runtime_output","terminal_action_family":null},{"classification_output_key":null,"contingent_antecedent_output_path":null,"contingent_on":null,"deliverable_confirmation_criterion_id":null,"deliverable_kind":null,"expected_classification":null,"expected_output_shape":"status_label","expected_output_value":null,"implicit":false,"judgment_polarity_when_holds":null,"judgment_predicate":null,"kind":"outcome","level":"run","method_mandated":false,"outcome":"whether the request was newly submitted or already present is output","output_path":"output.request_submission_state","request_slot_source_id":"u0","request_slot_source_quote":"whether the request was newly submitted or already present","requested_output_evidence_source":"runtime_output","terminal_action_family":null}],"credential_input_kind":"credential_name","credential_refs":["mock-portal-login"],"login_page_urls":["http://localhost:8900/utility_services/peach_gas/"],"raw_secret_evidence":null,"raw_secret_handling":"none","requires_user_clarification":false,"testing_intent":"require_test"}
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index 94e1154aa..7c8781576 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
is_fallback_floor_base_criterion,
request_policy_has_present_completion_contract,
)
+from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_PROMPT_NAME
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
from tests.unit.conftest import make_copilot_context
@@ -80,6 +81,8 @@ async def _classify_with_terminal_action_reconciliation(
prompts: list[str] = []
async def handler(prompt: str, prompt_name: str) -> dict[str, object]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return {"version": "1", "slots": []}
assert prompt_name == request_policy_module.PROMPT_NAME
prompts.append(prompt)
if "TERMINAL ACTION RECONCILIATION MODE" in prompt:
@@ -238,6 +241,87 @@ async def test_ambiguous_terminal_action_reconciliation_abstains_without_rekeyin
] == ["malformed_response"]
+async def test_request_slot_failure_precedes_later_transient_reconciliation_label(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ original = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "credential_name",
+ "credential_refs": ["Portal Login"],
+ "requires_user_clarification": False,
+ "completion_criteria": [
+ {
+ "outcome": "The requested service setup is created.",
+ "request_slot_source_id": "u9",
+ "request_slot_source_quote": "create the requested service setup",
+ }
+ ],
+ }
+ drifted = {
+ **original,
+ "completion_criteria": [
+ {
+ **original["completion_criteria"][0],
+ "outcome": "A guessed service setup is created.",
+ }
+ ],
+ }
+ classifier_calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal classifier_calls
+ assert prompt_name == request_policy_module.PROMPT_NAME
+ classifier_calls += 1
+ return original if classifier_calls == 1 else drifted
+
+ run_classifier = request_policy_module._run_request_policy_classifier
+
+ async def run_with_transient_reconciliation(
+ handler: Any,
+ prompt: str,
+ *,
+ deadline: float | None = None,
+ ) -> tuple[Any | None, str, int]:
+ if "TERMINAL ACTION RECONCILIATION MODE" in prompt:
+ return None, "transient_error", 0
+ return await run_classifier(handler, prompt, deadline=deadline)
+
+ monkeypatch.setattr(request_policy_module, "_run_request_policy_classifier", run_with_transient_reconciliation)
+
+ with capture_logs() as logs:
+ policy = await request_policy_module._classify_request(
+ "Sign in and create the requested service setup.", "", [], "", handler
+ )
+
+ assert policy.request_slot_failure_kind == "invalid_anchor_correction"
+ (event,) = (entry for entry in logs if entry["event"] == "copilot_terminal_action_reconciliation_omitted")
+ assert event["reason"] == "request_slot_failure"
+ assert event["failure_kind"] == "request_slot_failure"
+ assert event["retryable"] is False
+ assert event["request_slot_failure_kind"] == "invalid_anchor_correction"
+
+
+def test_unknown_request_slot_failure_kind_fails_closed_at_omission_boundary() -> None:
+ policy = RequestPolicy(
+ credential_input_kind="credential_name",
+ request_slot_failure_kind="future_request_slot_failure",
+ )
+
+ with capture_logs() as logs:
+ request_policy_module._log_terminal_action_reconciliation_omitted(
+ policy,
+ reason="classifier_failed",
+ failure_kind="transient_error",
+ )
+
+ (event,) = (entry for entry in logs if entry["event"] == "copilot_terminal_action_reconciliation_omitted")
+ assert event["reason"] == "request_slot_failure"
+ assert event["failure_kind"] == "request_slot_failure"
+ assert event["retryable"] is False
+ assert event["request_slot_failure_kind"] == "future_request_slot_failure"
+
+
@pytest.mark.asyncio
async def test_terminal_action_reconciliation_gets_its_own_classifier_deadline(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.py
index 250c3bf58..7dd0ffd71 100644
--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py
+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py
@@ -1,5 +1,9 @@
from __future__ import annotations
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -16,6 +20,8 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
from skyvern.forge.sdk.copilot.completion_verification import RunEvidenceSnapshot, evaluate_completion_criteria
from skyvern.forge.sdk.copilot.request_policy import (
_accept_request_slot_anchor_correction,
+ _anchor_correction_rejection_capture,
+ _apply_request_slot_datum_bindings,
_classification_from_raw,
_classify_request,
schema_output_path_aliases_from_criteria,
@@ -23,6 +29,9 @@ from skyvern.forge.sdk.copilot.request_policy import (
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_PROMPT_NAME
from skyvern.forge.sdk.copilot.request_slots import (
RequestSlotAntecedentFamily,
+ RequestSlotDatumBindingDeclarationV1,
+ RequestSlotDatumDeclineDeclarationV1,
+ RequestSlotDatumTargetV1,
RequestSlotDeclarationV1,
RequestSlotEnvelopeV1,
RequestSlotPinability,
@@ -30,6 +39,10 @@ from skyvern.forge.sdk.copilot.request_slots import (
RequestSlotProducerInputV1,
canonicalize_request_slots,
)
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatHistoryMessage,
+ WorkflowCopilotChatSender,
+)
P9_REQUEST = "Return completion status, the visible path label, and whether the path is login-only."
CONFIRMATION_NUMBER_REQUEST = "Turn on international roaming and return the confirmation number."
@@ -42,6 +55,167 @@ EXAMPLE_WATER_SERVICE_REQUEST = (
"account number, selected start date, deposit amount, and next owner. If the site only exposes an email/manual-"
"service path instead of an online form, report that as the blocker rather than inventing values."
)
+P8_QUICKCONNECT_REQUEST = (
+ "I need a reusable workflow that creates or verifies a gas QuickConnect request in the mock Peach State Gas "
+ "GasConnect Hub at http://localhost:8906/utility_services/peach_gas/. Sign in with the saved credential named "
+ "mock-portal-login. The service address parts, desired start date, and business name should be reusable inputs; "
+ "for this run use 77 Gaslight Way, Decatur, GA 30030, 2026-06-24, and Example Realty Labs Inc. After sign-in, "
+ "the workflow should open the QuickConnect create flow, use only the sourced address/date/business data, submit "
+ "only when the exact address is visible, then read the My QuickConnects table and output the request id, "
+ "provider-captured address, requested date, status, and whether the request was newly submitted or already present."
+)
+P8_QUICKCONNECT_REQUEST_SHA256 = "87440c6781124905ac58113b2fdc4629415ef883e69fdc479fc23ba380d3e353"
+P8_QUICKCONNECT_PRIOR_TURN = (
+ "The prior QuickConnect result included the request id, provider-captured address, requested date, and status."
+)
+P8_QUICKCONNECT_WORKFLOW_CONTEXT = (
+ "Existing workflow context also mentions request id, provider-captured address, requested date, and status."
+)
+P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE = (
+ Path(__file__).parent / "fixtures" / "copilot" / "request_policy" / "sky_12671_p8_noop_anchor_correction_pair.json"
+)
+P8_LIVE_ORIGINAL_PAYLOAD_FIXTURE = (
+ Path(__file__).parent / "fixtures" / "copilot" / "request_policy" / "sky_12671_p8_original_payload.json"
+)
+P8_CLEAN_RETRY_CUSTODY_GAP_FIXTURE = (
+ Path(__file__).parent / "fixtures" / "copilot" / "request_policy" / "sky_12671_p8_clean_retry_custody_gap.json"
+)
+
+
+def _recorded_p8_original_payload_bytes() -> bytes:
+ # The fixture stores backend.log:1316's original_payload_json verbatim; the
+ # repository text-file newline is not part of the recorded JSON field.
+ return P8_LIVE_ORIGINAL_PAYLOAD_FIXTURE.read_bytes().removesuffix(b"\n")
+
+
+def test_clean_retry_custody_gap_is_explicit_and_never_substituted_with_a_synthetic_replay() -> None:
+ custody = json.loads(P8_CLEAN_RETRY_CUSTODY_GAP_FIXTURE.read_text())
+
+ assert custody["workflow_permanent_id"] == "wpid_553704674845352182"
+ assert custody["original_payload_sha256"] == ("9f3a7ee247f6c15f677c68402930a7a5c3d23f334b5719442198127724c00cab")
+ assert custody["predicate"] == "target_coverage_mismatch"
+ assert custody["raw_original_payload_retained"] is False
+ assert custody["raw_correction_payload_retained"] is False
+ assert custody["custody_status"] == "exact_replay_unavailable"
+
+
+def _request_slot_envelope_with_target_bindings(
+ prompt: str,
+ envelope: RequestSlotEnvelopeV1,
+ *,
+ anchors_by_index: dict[int, tuple[str, str]],
+) -> dict[str, Any]:
+ marker = "Datum targets from the immutable request-policy payload:\n```"
+ assert marker in prompt
+ targets = json.loads(prompt.split(marker, 1)[1].split("```", 1)[0])
+ payload = envelope.model_dump(mode="json")
+ payload["datum_bindings"] = [
+ {
+ "criterion_index": target["criterion_index"],
+ "datum_field": target["datum_field"],
+ "declined": False,
+ "source_id": anchors_by_index[target["criterion_index"]][0],
+ "source_quote": anchors_by_index[target["criterion_index"]][1],
+ }
+ for target in targets
+ ]
+ return payload
+
+
+def _p8_quickconnect_classifier_fixture(*, source_id: str) -> dict[str, Any]:
+ # Only P8_QUICKCONNECT_REQUEST is copied from the admitted backend logs. The
+ # classifier packet was not retained, so this response is deliberately fixture-labeled.
+ return {
+ "testing_intent": "require_test",
+ "credential_input_kind": "credential_name",
+ "credential_refs": ["mock-portal-login"],
+ "requires_user_clarification": False,
+ "completion_criteria": [
+ {
+ "outcome": "A matching QuickConnect request is created or verified.",
+ "kind": "terminal_action",
+ "terminal_action_family": "request",
+ "request_slot_source_id": source_id,
+ "request_slot_source_quote": "creates or verifies a gas QuickConnect request",
+ },
+ {
+ "outcome": "Only the sourced address, date, and business data are used.",
+ "request_slot_source_id": source_id,
+ "request_slot_source_quote": "use only the sourced address/date/business data",
+ },
+ {
+ "outcome": "Submission occurs only when the exact address is visible.",
+ "request_slot_source_id": source_id,
+ "request_slot_source_quote": "submit only when the exact address is visible",
+ },
+ *[
+ {
+ "outcome": outcome,
+ "output_path": output_path,
+ "expected_output_shape": "status_label",
+ "request_slot_source_id": source_id,
+ "request_slot_source_quote": source_quote,
+ }
+ for outcome, output_path, source_quote in (
+ ("The request id is returned.", "output.request_id", "request id"),
+ (
+ "The provider-captured address is returned.",
+ "output.provider_captured_address",
+ "provider-captured address",
+ ),
+ ("The requested date is returned.", "output.requested_date", "requested date"),
+ ("The request status is returned.", "output.status", "status"),
+ )
+ ],
+ ],
+ }
+
+
+def _p8_quickconnect_envelope(*, source_id: str = "u0") -> RequestSlotEnvelopeV1:
+ return RequestSlotEnvelopeV1(
+ version="1",
+ slots=tuple(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+ for source_quote in ("request id", "provider-captured address", "requested date", "status")
+ ),
+ )
+
+
+def _p8_payload_with_typed_state_binding(
+ payload: dict[str, Any], *, binding_overrides: dict[str, Any] | None = None
+) -> dict[str, Any]:
+ enriched = json.loads(json.dumps(payload))
+ criterion_index = 6
+ criterion = enriched["completion_criteria"][criterion_index]
+ criterion["request_slot_datum_binding"] = {
+ "version": "1",
+ "criterion_index": criterion_index,
+ "datum_field": "output_path",
+ "datum_value": "output.request_submission_state",
+ "source_id": "u0",
+ "source_quote": "whether the request was newly submitted or already present",
+ **(binding_overrides or {}),
+ }
+ return enriched
+
+
+def _request_slot_input(latest_request: str) -> RequestSlotProducerInputV1:
+ return RequestSlotProducerInputV1(
+ version="1",
+ latest_request=latest_request,
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
def _synthetic_anchor_only_payload() -> dict[str, Any]:
@@ -817,7 +991,7 @@ def test_anchor_correction_semantic_validation_does_not_emit_mint_events(
]
}
- accepted = _accept_request_slot_anchor_correction(
+ decision = _accept_request_slot_anchor_correction(
original,
corrected,
request_slot_request=RequestSlotProducerInputV1(
@@ -832,7 +1006,7 @@ def test_anchor_correction_semantic_validation_does_not_emit_mint_events(
),
)
- assert accepted is not None
+ assert decision.predicate == "accepted"
assert events == []
@@ -840,7 +1014,7 @@ def test_anchor_correction_semantic_validation_does_not_emit_mint_events(
async def test_eight_row_anchor_correction_accepts_representation_drift_only() -> None:
original = _synthetic_eight_row_anchor_payload(valid_anchors=False)
corrected = _semantically_identical_anchor_correction()
- accepted = _accept_request_slot_anchor_correction(
+ decision = _accept_request_slot_anchor_correction(
original,
corrected,
request_slot_request=RequestSlotProducerInputV1(
@@ -855,7 +1029,9 @@ async def test_eight_row_anchor_correction_accepts_representation_drift_only() -
),
)
- assert accepted is not None
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ accepted = decision.accepted_payload
assert accepted["opaque_provider_metadata"] == {"source": "original"}
accepted_criteria = accepted["completion_criteria"]
assert len(accepted_criteria) == 8
@@ -886,227 +1062,1428 @@ async def test_eight_row_anchor_correction_accepts_representation_drift_only() -
assert policy.to_trace_data()["mint_degraded_criterion_count"] == 0
[email protected]("correction_kind", ["semantic_drift", "reordered_rows", "claim_expansion"])
-async def test_eight_row_anchor_correction_drift_fails_closed(correction_kind: str) -> None:
- original = _synthetic_eight_row_anchor_payload(valid_anchors=False)
- corrected = _semantically_identical_anchor_correction()
- corrected_criteria = corrected["completion_criteria"]
- if correction_kind == "semantic_drift":
- corrected_criteria[2]["outcome"] = "The workflow returns a guessed confirmation number."
- elif correction_kind == "reordered_rows":
- corrected_criteria[2], corrected_criteria[3] = corrected_criteria[3], corrected_criteria[2]
- else:
- corrected_criteria[0]["request_slot_source_id"] = "u0"
- corrected_criteria[0]["request_slot_source_quote"] = "reusable workflow"
- calls = 0
-
- async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
- nonlocal calls
- calls += 1
- return original if calls == 1 else corrected
-
- policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
-
- assert calls == 2
- assert policy.request_slot_failure_kind == "invalid_anchor_correction"
- assert sum(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria) == 6
- assert all(
- criterion.mint_degrade == "undecidable_judgment"
- for criterion in policy.completion_criteria
- if criterion.mint_disposition == "degraded"
- )
-
[email protected]("recorded_run", ["run1", "run2", "run3"])
+def test_p8_custody_triplet_source_bytes_accepts_fixture_labeled_anchor_correction(recorded_run: str) -> None:
+ assert recorded_run in {"run1", "run2", "run3"}
+ assert hashlib.sha256(P8_QUICKCONNECT_REQUEST.encode()).hexdigest() == P8_QUICKCONNECT_REQUEST_SHA256
+ original = _p8_quickconnect_classifier_fixture(source_id="u9")
+ corrected = _p8_quickconnect_classifier_fixture(source_id="u0")
-async def test_recorded_confirmation_runs_reach_same_run_plane_terminal() -> None:
- recorded_packets = (
- {
- "workflow_run_id": "wr_synthetic_recorded_1",
- "confirmation_number": "WTR-1842-DEMO",
- "account_number": "100245",
- "selected_start_date": "2026-06-22",
- },
- {
- "workflow_run_id": "wr_synthetic_recorded_3",
- "confirmation_number": "WTR-1842-DEMO",
- "account_number": "100245",
- "selected_start_date": "2026-06-22",
- },
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=RequestSlotProducerInputV1(
+ version="1",
+ latest_request=P8_QUICKCONNECT_REQUEST,
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ ),
)
- terminal_states: list[tuple[bool, bool, frozenset[str | None], int]] = []
-
- for packet in recorded_packets:
- original = _synthetic_eight_row_anchor_payload(valid_anchors=False)
- original["completion_criteria"] = original["completion_criteria"][2:5]
- corrected = _semantically_identical_anchor_correction()
- corrected["completion_criteria"] = corrected["completion_criteria"][2:5]
- envelope = _example_water_service_envelope()
- recorded_envelope = RequestSlotEnvelopeV1(version="1", slots=envelope.slots[:3])
- request_policy_calls = 0
-
- async def classify_handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
- nonlocal request_policy_calls
- if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return recorded_envelope.model_dump(mode="json")
- request_policy_calls += 1
- return original if request_policy_calls == 1 else corrected
-
- policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", classify_handler)
- reloaded_criteria = list(criteria_from_json(criteria_to_json(policy.completion_criteria)))
- evidence_label = f"recorded_{packet['workflow_run_id']}"
-
- async def verification_handler(**_: object) -> dict[str, Any]:
- return {
- "verdicts": [
- {
- "criterion_id": criterion.id,
- "satisfied": True,
- "reason_code": "evidence_confirms",
- "evidence_ref": f"block_outputs:{evidence_label}",
- }
- for criterion in reloaded_criteria
- ]
- }
-
- verification = await evaluate_completion_criteria(
- reloaded_criteria,
- RunEvidenceSnapshot(
- workflow_run_id=packet["workflow_run_id"],
- block_outputs={evidence_label: packet},
- block_output_sources={evidence_label: "runtime_output"},
- run_terminal_status="completed",
- ),
- verification_handler,
- )
-
- assert policy.to_trace_data()["mint_degraded_criterion_count"] == 0
- terminal_states.append(
- (
- verification.is_fully_satisfied(),
- verification.no_gradeable_run_plane,
- frozenset(verdict.evidence_source for verdict in verification.verdicts),
- len(verification.criterion_ids),
- )
- )
- assert terminal_states == [(True, False, frozenset({"runtime_output"}), 3)] * 2
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ accepted = decision.accepted_payload
+ assert [item["outcome"] for item in accepted["completion_criteria"]] == [
+ item["outcome"] for item in original["completion_criteria"]
+ ]
+ assert [item.get("request_slot_source_quote") for item in accepted["completion_criteria"]] == [
+ item.get("request_slot_source_quote") for item in original["completion_criteria"]
+ ]
+ assert [item.get("request_slot_source_id") for item in accepted["completion_criteria"]] == ["u0"] * 7
@pytest.mark.asyncio
-async def test_anchor_only_slot_producer_failure_degrades() -> None:
+async def test_p8_admitted_source_fixture_mints_ordered_decidable_canonical_slots() -> None:
+ assert hashlib.sha256(P8_QUICKCONNECT_REQUEST.encode()).hexdigest() == P8_QUICKCONNECT_REQUEST_SHA256
+ original = _p8_quickconnect_classifier_fixture(source_id="u9")
+ envelope = _p8_quickconnect_envelope(source_id="u1")
calls: list[str] = []
+ request_slot_envelopes: list[RequestSlotEnvelopeV1] = []
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return {"invalid": "request-slot payload"}
- return _synthetic_anchor_only_payload()
+ response = _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={
+ 3: ("u1", "request id"),
+ 4: ("u1", "provider-captured address"),
+ 5: ("u1", "requested date"),
+ 6: ("u1", "status"),
+ },
+ )
+ request_slot_envelopes.append(RequestSlotEnvelopeV1.model_validate_json(json.dumps(response)))
+ return response
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("two-pass producer consensus must not be echoed through a third model call")
+ if "TERMINAL ACTION RECONCILIATION MODE" in prompt:
+ return {"version": "1", "criterion_id": "c0", "terminal_action_family": "request"}
+ return original
+
+ policy = await _classify_request(
+ P8_QUICKCONNECT_REQUEST,
+ P8_QUICKCONNECT_WORKFLOW_CONTEXT,
+ [
+ WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.USER,
+ content=P8_QUICKCONNECT_PRIOR_TURN,
+ created_at=datetime(2026, 7, 20, tzinfo=timezone.utc),
+ )
+ ],
+ "",
+ handler,
+ )
- policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
+ assert calls == [
+ "workflow-copilot-request-policy",
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ "workflow-copilot-request-policy",
+ ]
+ assert policy.request_slot_failure_kind is None
+ assert [criterion.outcome for criterion in policy.completion_criteria] == [
+ "The request id is returned.",
+ "The provider-captured address is returned.",
+ "The requested date is returned.",
+ "The request status is returned.",
+ "A matching QuickConnect request is created or verified.",
+ "Only the sourced address, date, and business data are used.",
+ "Submission occurs only when the exact address is visible.",
+ ]
+ expected_request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request=P8_QUICKCONNECT_REQUEST,
+ workflow_context=P8_QUICKCONNECT_WORKFLOW_CONTEXT,
+ earliest_user_turn=P8_QUICKCONNECT_PRIOR_TURN,
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ datum_targets=tuple(
+ RequestSlotDatumTargetV1(
+ criterion_index=index,
+ datum_field="output_path",
+ datum_value=original["completion_criteria"][index]["output_path"],
+ criterion_outcome_sha256=hashlib.sha256(
+ original["completion_criteria"][index]["outcome"].encode()
+ ).hexdigest(),
+ )
+ for index in range(3, 7)
+ ),
+ )
+ assert [criterion.request_slot_id for criterion in policy.completion_criteria[:4]] == [
+ slot.slot_id
+ for slot in canonicalize_request_slots(
+ request=expected_request,
+ envelope=request_slot_envelopes[0],
+ ).slots
+ ]
+ assert all(criterion.mint_disposition == "decidable" for criterion in policy.completion_criteria[:4])
+ assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria[4:])
+ assert sum(criterion.kind == "terminal_action" for criterion in policy.completion_criteria) == 1
+ trace_data = policy.to_trace_data()
+ assert "request_slot_failure_kind" not in trace_data
+ assert trace_data["mint_degraded_criterion_count"] == 3
+
+
+def test_live_p8_noop_pair_rejects_unbound_and_accepts_typed_datum_binding() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ original_payload_bytes = _recorded_p8_original_payload_bytes()
+ assert hashlib.sha256(original_payload_bytes).hexdigest() == packet["source"]["original_sha256"]
+ recorded_original = json.loads(original_payload_bytes)
+ assert recorded_original == packet["original"]
+ capture = _anchor_correction_rejection_capture(recorded_original, packet["corrected"])
+
+ assert capture is not None
+ assert capture["original_sha256"] == packet["source"]["original_sha256"]
+ assert capture["corrected_sha256"] == packet["source"]["corrected_sha256"]
+ assert capture["pair_sha256"] == packet["source"]["pair_sha256"]
+ assert capture["pair_sha256"] == "e5be31a1cc861dd40c26a9c4df5d0240e6fabe42ca37bc1c5e2bcedbe56fbac5"
+
+ request_slot_request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request=packet["latest_request"],
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ legacy = _accept_request_slot_anchor_correction(
+ recorded_original,
+ packet["corrected"],
+ request_slot_request=request_slot_request,
+ )
+ assert legacy.predicate == packet["source"]["predicate"] == "original_quote_not_admissible"
+ assert legacy.criterion_index == packet["source"]["criterion_index"] == 6
- assert calls == ["workflow-copilot-request-policy", *([REQUEST_SLOT_PROMPT_NAME] * 4)]
- assert policy.request_slot_failure_kind == "invalid_output"
- assert all(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria)
- assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria)
+ enriched_original = _p8_payload_with_typed_state_binding(packet["original"])
+ enriched_corrected = _p8_payload_with_typed_state_binding(packet["corrected"])
+ accepted = _accept_request_slot_anchor_correction(
+ enriched_original,
+ enriched_corrected,
+ request_slot_request=request_slot_request,
+ )
+ assert accepted.predicate == "accepted"
+ assert accepted.accepted_payload == enriched_original
+
+
+def test_p8_producer_consensus_joins_only_the_enumerated_datum() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ request = _request_slot_input(packet["latest_request"])
+ outcome_sha256 = hashlib.sha256(packet["original"]["completion_criteria"][6]["outcome"].encode()).hexdigest()
+ request = request.model_copy(
+ update={
+ "datum_targets": (
+ RequestSlotDatumTargetV1(
+ criterion_index=6,
+ datum_field="output_path",
+ datum_value="output.request_submission_state",
+ criterion_outcome_sha256=outcome_sha256,
+ ),
+ )
+ }
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether the request was newly submitted or already present",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=6,
+ datum_field="output_path",
+ declined=False,
+ source_id="u0",
+ source_quote="whether the request was newly submitted or already present",
+ ),
+ ),
+ ),
+ )
+ decision = _apply_request_slot_datum_bindings(
+ packet["original"],
+ request_slot_request=request,
+ request_slot_contract=contract,
+ )
+
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ assert decision.accepted_payload["completion_criteria"][:6] == packet["original"]["completion_criteria"][:6]
+ assert decision.accepted_payload["completion_criteria"][6]["request_slot_datum_binding"] == {
+ "version": "1",
+ "criterion_index": 6,
+ "datum_field": "output_path",
+ "datum_value": "output.request_submission_state",
+ "criterion_outcome_sha256": outcome_sha256,
+ "source_id": "u0",
+ "source_quote": "whether the request was newly submitted or already present",
+ }
+ assert decision.trusted_bindings == (
+ request_policy_module.TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=6,
+ datum_field="output_path",
+ datum_value="output.request_submission_state",
+ criterion_outcome_sha256=outcome_sha256,
+ source_id="u0",
+ source_quote="whether the request was newly submitted or already present",
+ slot_id=contract.datum_bindings[0].slot_id,
+ ),
+ )
@pytest.mark.asyncio
-async def test_anchor_only_unpinnable_slots_degrade() -> None:
+async def test_producer_decline_degrades_junk_without_discarding_legitimate_sibling() -> None:
+ sibling_quote = "whether submission was new or existing"
+ junk_quote = "account number"
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "Submission state is returned.",
+ "output_path": "output.submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": sibling_quote,
+ },
+ {
+ "outcome": "Confirmation ID is returned.",
+ "output_path": "output.confirmation_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": junk_quote,
+ },
+ ]
+ }
+ request_text = "Return whether submission was new or existing and the adjacent account number."
+ slots = tuple(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+ for quote in dict.fromkeys((sibling_quote, "account number" if junk_quote != sibling_quote else sibling_quote))
+ )
calls: list[str] = []
- envelope = _anchor_only_envelope(pinability=RequestSlotPinability.UNPINNABLE)
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return envelope.model_dump(mode="json")
- return _synthetic_anchor_only_payload()
+ marker = "Datum targets from the immutable request-policy payload:\n```"
+ targets = json.loads(prompt.split(marker, 1)[1].split("```", 1)[0])
+ return {
+ "version": "1",
+ "slots": [slot.model_dump(mode="json") for slot in slots],
+ "datum_bindings": [
+ {
+ "criterion_index": targets[0]["criterion_index"],
+ "datum_field": targets[0]["datum_field"],
+ "declined": False,
+ "source_id": "u0",
+ "source_quote": sibling_quote,
+ },
+ {"criterion_index": 1, "datum_field": "output_path", "declined": True},
+ ],
+ }
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("producer consensus must not be echoed through a correction call")
+ return original
+
+ policy = await _classify_request(request_text, "", [], "", handler)
+
+ assert calls.count(REQUEST_SLOT_PROMPT_NAME) == 2
+ assert policy.request_slot_failure_kind is None
+ by_path = {criterion.floor_rekeyed_from_path: criterion for criterion in policy.completion_criteria}
+ assert by_path["output.submission_state"].mint_disposition == "decidable"
+ assert by_path["output.submission_state"].request_slot_id is not None
+ assert by_path["output.confirmation_id"].mint_disposition == "degraded"
+ assert by_path["output.confirmation_id"].mint_degrade == "undecidable_judgment"
+
+
[email protected]("producer_declines", [False, True])
+def test_server_applies_consensus_binding_or_local_decline(producer_declines: bool) -> None:
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "Submission state is returned.",
+ "output_path": "output.submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "whether submission was new or existing",
+ }
+ ]
+ }
+ target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256=hashlib.sha256(b"Submission state is returned.").hexdigest(),
+ )
+ request = _request_slot_input("Return whether submission was new or existing.").model_copy(
+ update={"datum_targets": (target,)}
+ )
+ producer_resolution: RequestSlotDatumBindingDeclarationV1 | RequestSlotDatumDeclineDeclarationV1
+ if producer_declines:
+ producer_resolution = RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=True,
+ )
+ else:
+ producer_resolution = RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(producer_resolution,),
+ ),
+ )
+ decision = _apply_request_slot_datum_bindings(
+ original,
+ request_slot_request=request,
+ request_slot_contract=contract,
+ )
+
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ if producer_declines:
+ assert decision.accepted_payload == original
+ assert decision.trusted_bindings == ()
+ else:
+ assert decision.accepted_payload["completion_criteria"][0]["request_slot_source_quote"] == (
+ "whether submission was new or existing"
+ )
+ assert decision.trusted_bindings[0].slot_id == contract.slots[0].slot_id
+
+
+ ("contract_mutation", "producer_declines"),
+ [
+ ("missing_resolution", False),
+ ("duplicate_binding", False),
+ ("stale_binding_value", False),
+ ("stale_decline_hash", True),
+ ],
+)
+def test_server_rejects_malformed_or_stale_consensus_contracts_without_mutating_payload(
+ contract_mutation: str,
+ producer_declines: bool,
+) -> None:
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "Submission state is returned.",
+ "output_path": "output.submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "whether submission was new or existing",
+ }
+ ]
+ }
+ target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256=hashlib.sha256(b"Submission state is returned.").hexdigest(),
+ )
+ request = _request_slot_input("Return whether submission was new or existing.").model_copy(
+ update={"datum_targets": (target,)}
+ )
+ resolution: RequestSlotDatumBindingDeclarationV1 | RequestSlotDatumDeclineDeclarationV1
+ if producer_declines:
+ resolution = RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=True,
+ )
+ else:
+ resolution = RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(resolution,),
+ ),
+ )
+
+ if contract_mutation == "missing_resolution":
+ malformed_contract = contract.model_copy(update={"datum_bindings": (), "datum_declines": ()})
+ elif contract_mutation == "duplicate_binding":
+ malformed_contract = contract.model_copy(update={"datum_bindings": contract.datum_bindings * 2})
+ elif contract_mutation == "stale_binding_value":
+ stale_binding = contract.datum_bindings[0].model_copy(update={"datum_value": "output.other_state"})
+ malformed_contract = contract.model_copy(update={"datum_bindings": (stale_binding,)})
+ else:
+ stale_decline = contract.datum_declines[0].model_copy(update={"criterion_outcome_sha256": "0" * 64})
+ malformed_contract = contract.model_copy(update={"datum_declines": (stale_decline,)})
+
+ decision = _apply_request_slot_datum_bindings(
+ original,
+ request_slot_request=request,
+ request_slot_contract=malformed_contract,
+ )
+
+ assert decision.predicate == "invalid_contract"
+ assert decision.accepted_payload is None
+ assert decision.trusted_bindings == ()
+ assert original["completion_criteria"][0]["request_slot_source_quote"] == ("whether submission was new or existing")
+ assert "request_slot_datum_binding" not in original["completion_criteria"][0]
+
+
+def test_server_rejects_binding_contract_minted_for_different_request() -> None:
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "Submission state is returned.",
+ "output_path": "output.submission_state",
+ }
+ ]
+ }
+ target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256=hashlib.sha256(b"Submission state is returned.").hexdigest(),
+ )
+ current_request = _request_slot_input("Return whether submission was new or existing.").model_copy(
+ update={"datum_targets": (target,)}
+ )
+ foreign_request = _request_slot_input(
+ "Return whether submission was new or existing. Do not submit a replacement."
+ ).model_copy(update={"datum_targets": (target,)})
+ foreign_contract = canonicalize_request_slots(
+ request=foreign_request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id="u0",
+ source_quote="whether submission was new or existing",
+ ),
+ ),
+ ),
+ )
+
+ decision = _apply_request_slot_datum_bindings(
+ original,
+ request_slot_request=current_request,
+ request_slot_contract=foreign_contract,
+ )
+
+ assert decision.predicate == "invalid_contract"
+ assert decision.accepted_payload is None
+ assert "request_slot_datum_binding" not in original["completion_criteria"][0]
+
+
+def test_trusted_binding_rejects_non_string_outcome_without_coercion() -> None:
+ item = {
+ "outcome": 123,
+ "output_path": "output.submission_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "submission state",
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "datum_value": "output.submission_state",
+ "criterion_outcome_sha256": hashlib.sha256(b"123").hexdigest(),
+ "source_id": "u0",
+ "source_quote": "submission state",
+ },
+ }
+ request = _request_slot_input("Return the submission state.")
+ binding = request_policy_module.TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256=hashlib.sha256(b"123").hexdigest(),
+ source_id="u0",
+ source_quote="submission state",
+ slot_id="slot-submission-state",
+ )
+
+ assert not request_policy_module._trusted_request_slot_datum_binding_is_valid(
+ item,
+ binding=binding,
+ criterion_index=0,
+ request_slot_request=request,
+ )
+
+
+async def test_datum_target_index_overflow_degrades_through_typed_failure_path() -> None:
+ raw = {
+ "completion_criteria": [
+ {
+ "outcome": f"Datum {index} is returned.",
+ "output_path": f"output.datum_{index}",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "all requested data",
+ }
+ for index in range(65)
+ ]
+ }
+ calls: list[str] = []
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ del prompt
+ calls.append(prompt_name)
+ return raw
+
+ policy = await _classify_request("Return all requested data.", "", [], "", handler)
+
+ assert calls == ["workflow-copilot-request-policy"]
+ assert policy.request_slot_failure_kind == "invalid_output"
+ assert policy.completion_criteria
+ assert all(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria)
+
+
+ "binding_overrides",
+ [
+ {"version": "2"},
+ {"criterion_index": 5},
+ {"datum_field": "outcome"},
+ {"datum_value": "output.other"},
+ {"source_id": "u1"},
+ {"source_quote": "status"},
+ {"unknown": "value"},
+ ],
+)
+def test_live_p8_invalid_typed_binding_preserves_existing_rejection(binding_overrides: dict[str, Any]) -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ original = _p8_payload_with_typed_state_binding(packet["original"], binding_overrides=binding_overrides)
+ corrected = _p8_payload_with_typed_state_binding(packet["corrected"], binding_overrides=binding_overrides)
+
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=RequestSlotProducerInputV1(
+ version="1",
+ latest_request=packet["latest_request"],
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ ),
+ )
+
+ assert decision.predicate == "original_quote_not_admissible"
+ assert decision.criterion_index == 6
+
+
+async def test_live_p8_typed_binding_mints_state_criterion_through_production_path() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ original_payload_bytes = _recorded_p8_original_payload_bytes()
+ assert hashlib.sha256(original_payload_bytes).hexdigest() == packet["source"]["original_sha256"]
+ raw = json.loads(original_payload_bytes)
+ assert raw == packet["original"]
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=tuple(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+ for source_quote in (
+ "output the request id",
+ "provider-captured address",
+ "requested date",
+ "status",
+ "whether the request was newly submitted or already present",
+ )
+ ),
+ )
+ calls: list[str] = []
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ calls.append(prompt_name)
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={
+ 6: ("u0", "whether the request was newly submitted or already present"),
+ },
+ )
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("two-pass producer consensus must not be echoed through a third model call")
+ if "TERMINAL ACTION RECONCILIATION MODE" in prompt:
+ return {"version": "1", "criterion_id": None, "terminal_action_family": None}
+ return raw
+
+ policy = await _classify_request(packet["latest_request"], "", [], "", handler)
+
+ assert calls == [
+ "workflow-copilot-request-policy",
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ "workflow-copilot-request-policy",
+ ]
+ assert policy.request_slot_failure_kind is None
+ state = next(
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.floor_rekeyed_from_path == "output.request_submission_state"
+ )
+ assert state.request_slot_id is not None
+ assert state.mint_disposition == "decidable"
+ assert state.mint_degrade is None
+
+
+async def test_primary_self_asserted_binding_is_replaced_by_two_pass_producer_consensus() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ raw = _p8_payload_with_typed_state_binding(packet["original"])
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether the request was newly submitted or already present",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ )
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={
+ 6: ("u0", "whether the request was newly submitted or already present"),
+ },
+ )
+ return raw
+
+ policy = await _classify_request(packet["latest_request"], "", [], "", handler)
+
+ state = next(
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.floor_rekeyed_from_path == "output.request_submission_state"
+ )
+ assert policy.request_slot_failure_kind is None
+ assert state.request_slot_id is not None
+ assert state.mint_disposition == "decidable"
+
+
[email protected]("binding", [None, {"version": "2"}])
+def test_invalid_typed_binding_uses_unchanged_legacy_admissibility(binding: Any) -> None:
+ criterion = {
+ "outcome": "The request id is returned.",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "request id",
+ "request_slot_datum_binding": binding,
+ }
+ payload = {"completion_criteria": [criterion]}
+
+ decision = _accept_request_slot_anchor_correction(
+ payload,
+ payload,
+ request_slot_request=RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return the request id.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ ),
+ )
+
+ assert decision.predicate == "accepted"
+
+
+def test_typed_classification_binding_accepts_nonlexical_datum_name() -> None:
+ criterion = {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "kind": "validation_classification",
+ "classification_output_key": "decision_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "whether the request was approved or denied",
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "classification_output_key",
+ "datum_value": "decision_state",
+ "source_id": "u0",
+ "source_quote": "whether the request was approved or denied",
+ },
+ }
+ payload = {"completion_criteria": [criterion]}
+
+ decision = _accept_request_slot_anchor_correction(
+ payload,
+ payload,
+ request_slot_request=RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return whether the request was approved or denied.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ ),
+ )
+
+ assert decision.predicate == "accepted"
+
+
+def test_anchor_correction_rejects_typed_binding_drift() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ original = _p8_payload_with_typed_state_binding(packet["original"])
+ corrected = _p8_payload_with_typed_state_binding(
+ packet["corrected"],
+ binding_overrides={"datum_value": "output.other"},
+ )
+
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=RequestSlotProducerInputV1(
+ version="1",
+ latest_request=packet["latest_request"],
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ ),
+ )
+
+ assert decision.predicate == "criterion_semantics_changed"
+ assert decision.criterion_index == 6
+
+
+ ("latest_request", "source_id"),
+ [
+ ("Return whether the request was approved or denied.", "u9"),
+ (
+ "Return whether the request was approved or denied and repeat whether the request was approved or denied.",
+ "u0",
+ ),
+ ],
+)
+def test_typed_binding_requires_unique_span_in_nominated_source(latest_request: str, source_id: str) -> None:
+ source_quote = "whether the request was approved or denied"
+ criterion = {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "output_path": "output.decision_state",
+ "request_slot_source_id": source_id,
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "datum_value": "output.decision_state",
+ "source_id": source_id,
+ "source_quote": source_quote,
+ },
+ }
+ payload = {"completion_criteria": [criterion]}
+
+ decision = _accept_request_slot_anchor_correction(
+ payload,
+ payload,
+ request_slot_request=_request_slot_input(latest_request),
+ )
+
+ assert decision.predicate == "original_quote_not_admissible"
+ assert decision.criterion_index == 0
+
+
[email protected]("binding", [[], {"version": "1"}])
+async def test_malformed_binding_uses_legacy_path_through_classifier(binding: Any) -> None:
+ source_quote = "request id"
+ raw = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": [
+ {
+ "outcome": "The request id is returned.",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": binding,
+ }
+ ],
+ }
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ )
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return envelope.model_dump(mode="json")
+ if "REQUEST SLOT ANCHOR CORRECTION" in prompt:
+ pytest.fail("malformed binding must not disable legacy admissibility")
+ return raw
+
+ policy = await _classify_request("Return the request id.", "", [], "", handler)
+
+ criterion = policy.completion_criteria[0]
+ assert criterion.request_slot_id is not None
+ assert criterion.mint_disposition == "decidable"
+ assert criterion.mint_degrade is None
+
+
+async def test_typed_classification_binding_mints_through_independent_producer() -> None:
+ source_quote = "whether the request was approved or denied"
+ raw = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": [
+ {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "kind": "validation_classification",
+ "classification_output_key": "decision_state",
+ "expected_classification": "approved",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "classification_output_key",
+ "datum_value": "decision_state",
+ "source_id": "u0",
+ "source_quote": source_quote,
+ },
+ }
+ ],
+ }
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.PINNED,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ )
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={0: ("u0", source_quote)},
+ )
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("producer consensus must not be echoed through a correction call")
+ return raw
+
+ policy = await _classify_request(
+ "Return whether the request was approved or denied.",
+ "",
+ [],
+ "",
+ handler,
+ )
+
+ criterion = policy.completion_criteria[0]
+ assert criterion.request_slot_id is not None
+ assert criterion.classification_output_key == "decision_state"
+ assert criterion.expected_classification == "approved"
+ assert criterion.mint_disposition == "decidable"
+
+
[email protected]("corrected_binding", [None, {}])
+def test_anchor_correction_rejects_removed_or_malformed_binding_on_one_side(corrected_binding: Any) -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ original = _p8_payload_with_typed_state_binding(packet["original"])
+ corrected = _p8_payload_with_typed_state_binding(packet["corrected"])
+ corrected_criterion = corrected["completion_criteria"][6]
+ if corrected_binding is None:
+ corrected_criterion.pop("request_slot_datum_binding")
+ else:
+ corrected_criterion["request_slot_datum_binding"] = corrected_binding
+
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=_request_slot_input(packet["latest_request"]),
+ )
+
+ assert decision.predicate == "criterion_semantics_changed"
+ assert decision.criterion_index == 6
+
+
[email protected]("mutated_index", [False, 0.0])
+def test_anchor_correction_compares_binding_values_type_strictly(mutated_index: Any) -> None:
+ source_quote = "whether the request was approved or denied"
+ criterion = {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "output_path": "output.decision_state",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "datum_value": "output.decision_state",
+ "source_id": "u0",
+ "source_quote": source_quote,
+ },
+ }
+ original = {"completion_criteria": [criterion]}
+ corrected = json.loads(json.dumps(original))
+ corrected["completion_criteria"][0]["request_slot_datum_binding"]["criterion_index"] = mutated_index
+
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=_request_slot_input(f"Return {source_quote}."),
+ )
+
+ assert decision.predicate == "criterion_semantics_changed"
+ assert decision.criterion_index == 0
+
+
+def test_validation_classification_binding_cannot_certify_inactive_output_path() -> None:
+ source_quote = "whether the request was approved or denied"
+ criterion = {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "kind": "validation_classification",
+ "output_path": "output.zzz",
+ "classification_output_key": "unrelated_key",
+ "expected_classification": "approved",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "datum_value": "output.zzz",
+ "source_id": "u0",
+ "source_quote": source_quote,
+ },
+ }
+ payload = {"completion_criteria": [criterion]}
+
+ decision = _accept_request_slot_anchor_correction(
+ payload,
+ payload,
+ request_slot_request=_request_slot_input(f"Return {source_quote}."),
+ )
+
+ assert decision.predicate == "original_quote_not_admissible"
+ assert decision.criterion_index == 0
+
+
+def test_typed_binding_cannot_certify_classification_key_rejected_by_parser() -> None:
+ source_quote = "whether the request was approved or denied"
+ criterion = {
+ "outcome": "Whether the request was approved or denied is returned.",
+ "kind": "validation_classification",
+ "classification_output_key": "bad.key",
+ "expected_classification": "approved",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ "request_slot_datum_binding": {
+ "version": "1",
+ "criterion_index": 0,
+ "datum_field": "classification_output_key",
+ "datum_value": "bad.key",
+ "source_id": "u0",
+ "source_quote": source_quote,
+ },
+ }
+ payload = {"completion_criteria": [criterion]}
+
+ request = _request_slot_input(f"Return {source_quote}.")
+ decision = _accept_request_slot_anchor_correction(
+ payload,
+ payload,
+ request_slot_request=request,
+ )
+
+ assert decision.predicate == "original_quote_not_admissible"
+ assert decision.criterion_index == 0
+
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.PINNED,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ ),
+ )
+ policy = _classification_from_raw(
+ payload,
+ request_slot_request=request,
+ request_slot_contract=contract,
+ )
+ parsed = next(criterion for criterion in policy.completion_criteria if criterion.request_slot_id is None)
+ assert parsed.request_slot_id is None
+ assert parsed.classification_output_key is None
+ assert parsed.mint_disposition == "degraded"
+ assert parsed.mint_degrade == "undecidable_judgment"
+
+
+async def test_valid_typed_binding_degrades_when_independent_producer_disagrees() -> None:
+ packet = json.loads(P8_LIVE_NOOP_ANCHOR_CORRECTION_FIXTURE.read_text())
+ raw = _p8_payload_with_typed_state_binding(packet["original"])
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=tuple(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+ for source_quote in ("output the request id", "provider-captured address", "requested date", "status")
+ ),
+ )
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return envelope.model_dump(mode="json")
+ if "REQUEST SLOT ANCHOR CORRECTION" in prompt:
+ pytest.fail("valid typed binding must not enter correction")
+ if "TERMINAL ACTION RECONCILIATION MODE" in prompt:
+ return {"version": "1", "criterion_id": None, "terminal_action_family": None}
+ return raw
+
+ policy = await _classify_request(packet["latest_request"], "", [], "", handler)
+
+ state = next(
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.floor_rekeyed_from_path == "output.request_submission_state"
+ )
+ assert state.request_slot_id is None
+ assert state.mint_disposition == "degraded"
+ assert state.mint_degrade == "undecidable_judgment"
+
+
+def test_anchor_correction_accepts_quote_unique_in_nominated_source_with_cross_source_overlap() -> None:
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return the request id for the current QuickConnect.",
+ workflow_context="",
+ earliest_user_turn="Return the request id from the prior QuickConnect.",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "The request id is returned.",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u9",
+ "request_slot_source_quote": "request id",
+ }
+ ]
+ }
+ corrected = {
+ "completion_criteria": [
+ {
+ **original["completion_criteria"][0],
+ "request_slot_source_id": "u1",
+ }
+ ]
+ }
+
+ decision = _accept_request_slot_anchor_correction(
+ original,
+ corrected,
+ request_slot_request=request,
+ )
+
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ accepted = decision.accepted_payload
+ assert accepted["completion_criteria"][0]["request_slot_source_id"] == "u1"
+
+
+def test_anchor_correction_rejection_names_exact_failed_predicate() -> None:
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return the request id.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "The request id is returned.",
+ "output_path": "output.request_id",
+ "request_slot_source_id": "u9",
+ "request_slot_source_quote": "request id",
+ }
+ ]
+ }
+ corrected = {
+ "completion_criteria": [
+ {
+ **original["completion_criteria"][0],
+ "request_slot_source_id": "u1",
+ "request_slot_source_quote": "request",
+ }
+ ]
+ }
+
+ decision = _accept_request_slot_anchor_correction(original, corrected, request_slot_request=request)
+
+ assert decision.predicate == "original_quote_not_admissible"
+ assert decision.criterion_index == 0
+ assert decision.accepted_payload is None
+
+
+def test_anchor_correction_rejection_capture_is_redacted_and_content_addressed() -> None:
+ original = {
+ "password": "unredacted-password",
+ "raw_secret_evidence": "bare-credential-value-1234567890",
+ "completion_criteria": [
+ {
+ "outcome": "Return the request id after password=another-secret",
+ "request_slot_source_id": "u9",
+ }
+ ],
+ }
+ corrected = {
+ "api_key": "sk-abcdefghijklmnop",
+ "completion_criteria": [
+ {
+ "outcome": "Return the request id after password=another-secret",
+ "request_slot_source_id": "u0",
+ }
+ ],
+ }
+
+ capture = _anchor_correction_rejection_capture(original, corrected)
+
+ assert capture is not None
+ serialized_capture = json.dumps(capture, sort_keys=True)
+ assert "unredacted-password" not in serialized_capture
+ assert "bare-credential-value-1234567890" not in serialized_capture
+ assert "another-secret" not in serialized_capture
+ assert "sk-abcdefghijklmnop" not in serialized_capture
+ assert "****" in capture["original_payload_json"]
+ assert json.loads(capture["original_payload_json"])["raw_secret_evidence"] == "[REDACTED_SECRET]"
+ assert "[REDACTED_SECRET]" in capture["corrected_payload_json"]
+ assert hashlib.sha256(capture["original_payload_json"].encode()).hexdigest() == capture["original_sha256"]
+ assert hashlib.sha256(capture["corrected_payload_json"].encode()).hexdigest() == capture["corrected_sha256"]
+ pair_json = json.dumps(
+ {
+ "corrected": json.loads(capture["corrected_payload_json"]),
+ "original": json.loads(capture["original_payload_json"]),
+ },
+ ensure_ascii=True,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
+ assert hashlib.sha256(pair_json.encode()).hexdigest() == capture["pair_sha256"]
+
+
[email protected]("correction_kind", ["semantic_drift", "reordered_rows", "claim_expansion"])
+async def test_eight_row_anchor_correction_drift_fails_closed(correction_kind: str) -> None:
+ original = _synthetic_eight_row_anchor_payload(valid_anchors=False)
+ corrected = _semantically_identical_anchor_correction()
+ corrected_criteria = corrected["completion_criteria"]
+ if correction_kind == "semantic_drift":
+ corrected_criteria[2]["outcome"] = "The workflow returns a guessed confirmation number."
+ elif correction_kind == "reordered_rows":
+ corrected_criteria[2], corrected_criteria[3] = corrected_criteria[3], corrected_criteria[2]
+ else:
+ corrected_criteria[0]["request_slot_source_id"] = "u0"
+ corrected_criteria[0]["request_slot_source_quote"] = "reusable workflow"
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ nonlocal calls
+ calls += 1
+ return original if calls == 1 else corrected
policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
- assert calls == ["workflow-copilot-request-policy", REQUEST_SLOT_PROMPT_NAME, REQUEST_SLOT_PROMPT_NAME]
- assert policy.request_slot_failure_kind is None
- assert all(criterion.pinability == "unpinnable" for criterion in policy.completion_criteria)
- assert all(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria)
- assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria)
+ assert calls == 2
+ assert policy.request_slot_failure_kind == "invalid_anchor_correction"
+ assert sum(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria) == 6
+ assert all(
+ criterion.mint_degrade == "undecidable_judgment"
+ for criterion in policy.completion_criteria
+ if criterion.mint_disposition == "degraded"
+ )
@pytest.mark.asyncio
-async def test_unanchored_requested_output_with_invalid_correction_fails_closed() -> None:
- calls: list[str] = []
- unanchored = {
- "testing_intent": "require_test",
- "credential_input_kind": "none",
- "requires_user_clarification": False,
- "completion_criteria": [
- {
- "outcome": "The workflow returns the confirmation number.",
- "output_path": "output.confirmation_number",
- "expected_output_shape": "status_label",
+async def test_recorded_confirmation_runs_reach_same_run_plane_terminal() -> None:
+ recorded_packets = (
+ {
+ "workflow_run_id": "wr_synthetic_recorded_1",
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ },
+ {
+ "workflow_run_id": "wr_synthetic_recorded_3",
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ },
+ )
+ terminal_states: list[tuple[bool, bool, frozenset[str | None], int]] = []
+
+ for packet in recorded_packets:
+ original = _synthetic_eight_row_anchor_payload(valid_anchors=False)
+ original["completion_criteria"] = original["completion_criteria"][2:5]
+ corrected = _semantically_identical_anchor_correction()
+ corrected["completion_criteria"] = corrected["completion_criteria"][2:5]
+ envelope = _example_water_service_envelope()
+ recorded_envelope = RequestSlotEnvelopeV1(version="1", slots=envelope.slots[:3])
+ request_policy_calls = 0
+
+ async def classify_handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ nonlocal request_policy_calls
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return recorded_envelope.model_dump(mode="json")
+ request_policy_calls += 1
+ return original if request_policy_calls == 1 else corrected
+
+ policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", classify_handler)
+ reloaded_criteria = list(criteria_from_json(criteria_to_json(policy.completion_criteria)))
+ evidence_label = f"recorded_{packet['workflow_run_id']}"
+
+ async def verification_handler(**_: object) -> dict[str, Any]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": f"block_outputs:{evidence_label}",
+ }
+ for criterion in reloaded_criteria
+ ]
}
- ],
- }
+
+ verification = await evaluate_completion_criteria(
+ reloaded_criteria,
+ RunEvidenceSnapshot(
+ workflow_run_id=packet["workflow_run_id"],
+ block_outputs={evidence_label: packet},
+ block_output_sources={evidence_label: "runtime_output"},
+ run_terminal_status="completed",
+ ),
+ verification_handler,
+ )
+
+ assert policy.to_trace_data()["mint_degraded_criterion_count"] == 0
+ terminal_states.append(
+ (
+ verification.is_fully_satisfied(),
+ verification.no_gradeable_run_plane,
+ frozenset(verdict.evidence_source for verdict in verification.verdicts),
+ len(verification.criterion_ids),
+ )
+ )
+
+ assert terminal_states == [(True, False, frozenset({"runtime_output"}), 3)] * 2
+
+
+async def test_anchor_only_slot_producer_failure_degrades() -> None:
+ calls: list[str] = []
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
- return unanchored
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return {"invalid": "request-slot payload"}
+ return _synthetic_anchor_only_payload()
- policy = await _classify_request(CONFIRMATION_NUMBER_REQUEST, "", [], "", handler)
+ policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
- assert calls == ["workflow-copilot-request-policy", "workflow-copilot-request-policy"]
- criterion = policy.completion_criteria[0]
- assert criterion.level == "run"
- assert criterion.kind == "outcome"
- assert criterion.pinability is None
- assert criterion.output_path is None
- assert criterion.expected_output_shape is None
- assert criterion.floor_rekeyed_from_path == "output.confirmation_number"
- assert criterion.mint_disposition == "degraded"
- assert criterion.mint_degrade == "undecidable_judgment"
- assert policy.request_slot_failure_kind == "invalid_anchor_correction"
+ assert calls == ["workflow-copilot-request-policy", *([REQUEST_SLOT_PROMPT_NAME] * 4)]
+ assert policy.request_slot_failure_kind == "invalid_output"
+ assert all(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria)
+ assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria)
@pytest.mark.asyncio
[email protected]("corrected_quote", ["account number", "number"])
-async def test_unanchored_correction_must_match_its_structured_datum(corrected_quote: str) -> None:
+async def test_anchor_only_unpinnable_slots_degrade() -> None:
calls: list[str] = []
- request = "Return the confirmation code beside the adjacent account number."
- unanchored = {
- "completion_criteria": [
- {
- "outcome": "The workflow returns the confirmation code beside the account number.",
- "output_path": "output.confirmation_code",
- "expected_output_shape": "reference_code",
- }
- ]
- }
- corrected = {
- "completion_criteria": [
- {
- **unanchored["completion_criteria"][0],
- "request_slot_source_id": "u0",
- "request_slot_source_quote": corrected_quote,
- }
- ]
- }
+ envelope = _anchor_only_envelope(pinability=RequestSlotPinability.UNPINNABLE)
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
- return unanchored if len(calls) == 1 else corrected
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return envelope.model_dump(mode="json")
+ return _synthetic_anchor_only_payload()
- policy = await _classify_request(request, "", [], "", handler)
+ policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
- assert calls == ["workflow-copilot-request-policy", "workflow-copilot-request-policy"]
- criterion = policy.completion_criteria[0]
- assert policy.request_slot_failure_kind == "invalid_anchor_correction"
- assert criterion.output_path is None
- assert criterion.expected_output_shape is None
- assert criterion.mint_disposition == "degraded"
- assert criterion.mint_degrade == "undecidable_judgment"
+ assert calls == ["workflow-copilot-request-policy", REQUEST_SLOT_PROMPT_NAME, REQUEST_SLOT_PROMPT_NAME]
+ assert policy.request_slot_failure_kind is None
+ assert all(criterion.pinability == "unpinnable" for criterion in policy.completion_criteria)
+ assert all(criterion.mint_disposition == "degraded" for criterion in policy.completion_criteria)
+ assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria)
@pytest.mark.asyncio
-async def test_unanchored_correction_tied_to_its_structured_datum_mints_decidable() -> None:
+async def test_unanchored_requested_output_uses_producer_consensus_and_mints_decidable() -> None:
calls: list[str] = []
request = "Return the confirmation code."
unanchored = {
@@ -1118,15 +2495,6 @@ async def test_unanchored_correction_tied_to_its_structured_datum_mints_decidabl
}
]
}
- corrected = {
- "completion_criteria": [
- {
- **unanchored["completion_criteria"][0],
- "request_slot_source_id": "u0",
- "request_slot_source_quote": "confirmation code",
- }
- ]
- }
envelope = RequestSlotEnvelopeV1(
version="1",
slots=(
@@ -1143,13 +2511,18 @@ async def test_unanchored_correction_tied_to_its_structured_datum_mints_decidabl
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return envelope.model_dump(mode="json")
- return unanchored if calls.count("workflow-copilot-request-policy") == 1 else corrected
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={0: ("u0", "confirmation code")},
+ )
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("producer consensus must not be echoed through a correction call")
+ return unanchored
policy = await _classify_request(request, "", [], "", handler)
assert calls == [
- "workflow-copilot-request-policy",
"workflow-copilot-request-policy",
REQUEST_SLOT_PROMPT_NAME,
REQUEST_SLOT_PROMPT_NAME,
@@ -1162,7 +2535,7 @@ async def test_unanchored_correction_tied_to_its_structured_datum_mints_decidabl
@pytest.mark.asyncio
-async def test_source_valid_wrong_datum_anchor_gets_one_correction_and_fails_closed() -> None:
+async def test_source_valid_wrong_primary_anchor_is_replaced_by_producer_consensus() -> None:
calls: list[str] = []
request = "Return the confirmation ID and the adjacent account number."
wrong_datum = {
@@ -1179,6 +2552,13 @@ async def test_source_valid_wrong_datum_anchor_gets_one_correction_and_fails_clo
envelope = RequestSlotEnvelopeV1(
version="1",
slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="confirmation ID",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
RequestSlotDeclarationV1(
source_id="u0",
source_quote="account number",
@@ -1192,21 +2572,104 @@ async def test_source_valid_wrong_datum_anchor_gets_one_correction_and_fails_clo
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return envelope.model_dump(mode="json")
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={0: ("u0", "confirmation ID")},
+ )
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("producer consensus must not be echoed through a correction call")
return wrong_datum
policy = await _classify_request(request, "", [], "", handler)
- assert calls == ["workflow-copilot-request-policy", "workflow-copilot-request-policy"]
+ assert calls == [
+ "workflow-copilot-request-policy",
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ ]
criterion = policy.completion_criteria[0]
- assert policy.request_slot_failure_kind == "invalid_anchor_correction"
- assert criterion.request_slot_id is None
- assert criterion.mint_disposition == "degraded"
- assert criterion.mint_degrade == "undecidable_judgment"
+ assert policy.request_slot_failure_kind is None
+ assert criterion.request_slot_id is not None
+ assert criterion.mint_disposition == "decidable"
+ assert criterion.mint_degrade is None
+
+
+def test_producer_consensus_replaces_primary_anchor_with_agreed_datum_slot() -> None:
+ latest_request = "Return the confirmation ID and the adjacent account number."
+ original = {
+ "completion_criteria": [
+ {
+ "outcome": "The workflow returns the confirmation ID beside the account number.",
+ "output_path": "output.confirmation_id",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": "account number",
+ }
+ ]
+ }
+ outcome_sha256 = hashlib.sha256(original["completion_criteria"][0]["outcome"].encode()).hexdigest()
+ request = _request_slot_input(latest_request).model_copy(
+ update={
+ "datum_targets": (
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.confirmation_id",
+ criterion_outcome_sha256=outcome_sha256,
+ ),
+ )
+ }
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=tuple(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+ for quote in ("confirmation ID", "account number")
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id="u0",
+ source_quote="confirmation ID",
+ ),
+ ),
+ ),
+ )
+ decision = _apply_request_slot_datum_bindings(
+ original,
+ request_slot_request=request,
+ request_slot_contract=contract,
+ )
+
+ assert decision.predicate == "accepted"
+ assert decision.accepted_payload is not None
+ assert decision.accepted_payload["completion_criteria"][0]["request_slot_source_quote"] == "confirmation ID"
+ assert decision.trusted_bindings == (
+ request_policy_module.TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.confirmation_id",
+ criterion_outcome_sha256=outcome_sha256,
+ source_id="u0",
+ source_quote="confirmation ID",
+ slot_id=contract.datum_bindings[0].slot_id,
+ ),
+ )
@pytest.mark.asyncio
-async def test_anchor_correction_preserves_original_criterion_datum_quote() -> None:
+async def test_producer_consensus_replaces_invalid_primary_source_identity() -> None:
calls: list[str] = []
request = "Return the confirmation code."
original = {
@@ -1220,15 +2683,6 @@ async def test_anchor_correction_preserves_original_criterion_datum_quote() -> N
}
]
}
- corrected = {
- "completion_criteria": [
- {
- **original["completion_criteria"][0],
- "request_slot_source_id": "u0",
- "request_slot_source_quote": "Return the confirmation code",
- }
- ]
- }
envelope = RequestSlotEnvelopeV1(
version="1",
slots=(
@@ -1245,13 +2699,18 @@ async def test_anchor_correction_preserves_original_criterion_datum_quote() -> N
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
if prompt_name == REQUEST_SLOT_PROMPT_NAME:
- return envelope.model_dump(mode="json")
- return original if calls.count("workflow-copilot-request-policy") == 1 else corrected
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={0: ("u0", "confirmation code")},
+ )
+ if "REQUEST SLOT DATUM BINDING" in prompt:
+ pytest.fail("producer consensus must not be echoed through a correction call")
+ return original
policy = await _classify_request(request, "", [], "", handler)
assert calls == [
- "workflow-copilot-request-policy",
"workflow-copilot-request-policy",
REQUEST_SLOT_PROMPT_NAME,
REQUEST_SLOT_PROMPT_NAME,
@@ -1259,79 +2718,10 @@ async def test_anchor_correction_preserves_original_criterion_datum_quote() -> N
criterion = policy.completion_criteria[0]
assert policy.request_slot_failure_kind is None
assert criterion.floor_rekeyed_from_path == "output.confirmation_code"
+ assert criterion.request_slot_id is not None
assert criterion.mint_disposition == "decidable"
- "correction_kind",
- ["semantic_drift", "duplicate_quote", "unknown_source", "adjacent_datum", "bare_shared_token"],
-)
-async def test_invalid_anchor_correction_fails_closed(correction_kind: str) -> None:
- calls: list[str] = []
- if correction_kind == "duplicate_quote":
- request = "Return status and status."
- elif correction_kind in {"adjacent_datum", "bare_shared_token"}:
- request = "Return the confirmation ID and the adjacent account number."
- else:
- request = CONFIRMATION_NUMBER_REQUEST
- unanchored = {
- "testing_intent": "require_test",
- "credential_input_kind": "none",
- "requires_user_clarification": False,
- "completion_criteria": [
- {
- "outcome": "The workflow returns the confirmation number.",
- "output_path": "output.confirmation_number",
- "expected_output_shape": "status_label",
- }
- ],
- }
- if correction_kind in {"adjacent_datum", "bare_shared_token"}:
- unanchored["completion_criteria"][0]["outcome"] = (
- "The workflow returns the confirmation number beside the account data."
- )
- original_quote = (
- "status"
- if correction_kind == "duplicate_quote"
- else "confirmation ID"
- if correction_kind in {"adjacent_datum", "bare_shared_token"}
- else "return the confirmation number"
- )
- unanchored["completion_criteria"][0]["request_slot_source_id"] = "u8"
- unanchored["completion_criteria"][0]["request_slot_source_quote"] = original_quote
- corrected_item = {
- **unanchored["completion_criteria"][0],
- "request_slot_source_id": "u9" if correction_kind == "unknown_source" else "u0",
- "request_slot_source_quote": (
- "status"
- if correction_kind == "duplicate_quote"
- else "account number"
- if correction_kind == "adjacent_datum"
- else "number"
- if correction_kind == "bare_shared_token"
- else original_quote
- ),
- }
- if correction_kind == "semantic_drift":
- corrected_item["outcome"] = "The workflow returns a guessed confirmation."
- corrected = {**unanchored, "completion_criteria": [corrected_item]}
-
- async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
- calls.append(prompt_name)
- return unanchored if len(calls) == 1 else corrected
-
- policy = await _classify_request(request, "", [], "", handler)
-
- assert calls == ["workflow-copilot-request-policy", "workflow-copilot-request-policy"]
- criterion = policy.completion_criteria[0]
- assert criterion.output_path is None
- assert criterion.expected_output_shape is None
- assert criterion.mint_disposition == "degraded"
- assert criterion.mint_degrade == "undecidable_judgment"
- assert policy.request_slot_failure_kind == "invalid_anchor_correction"
-
-
@pytest.mark.asyncio
async def test_anchor_correction_still_requires_slot_producer_agreement() -> None:
calls: list[str] = []
@@ -1365,7 +2755,6 @@ async def test_anchor_correction_still_requires_slot_producer_agreement() -> Non
policy = await _classify_request(CONFIRMATION_NUMBER_REQUEST, "", [], "", handler)
assert calls == [
- "workflow-copilot-request-policy",
"workflow-copilot-request-policy",
*([REQUEST_SLOT_PROMPT_NAME] * 4),
]
diff --git a/tests/unit/test_copilot_request_slots.py b/tests/unit/test_copilot_request_slots.py
index 531ff5621..97aa7b24c 100644
--- a/tests/unit/test_copilot_request_slots.py
+++ b/tests/unit/test_copilot_request_slots.py
@@ -12,6 +12,9 @@ from skyvern.forge.sdk.copilot.request_slots import (
CanonicalRequestSlotV1,
RequestSlotAntecedentFamily,
RequestSlotContractV1,
+ RequestSlotDatumBindingDeclarationV1,
+ RequestSlotDatumDeclineDeclarationV1,
+ RequestSlotDatumTargetV1,
RequestSlotDeclarationV1,
RequestSlotEnvelopeV1,
RequestSlotPinability,
@@ -377,6 +380,53 @@ def test_contract_agreement_requires_antecedent_family_consensus() -> None:
assert not request_slot_contracts_agree(unconditional, blocker)
+def test_contract_agreement_uses_canonical_slot_not_redundant_binding_quote_bytes() -> None:
+ target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256="1" * 64,
+ )
+ request = _input(
+ latest_request="Return whether the request was newly submitted or already present.",
+ datum_targets=(target,),
+ )
+ source_id = _source_id(request, "newly submitted")
+
+ def contract_for_quote(quote: str) -> RequestSlotContractV1:
+ return canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote=quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=source_id,
+ source_quote=quote,
+ ),
+ ),
+ ),
+ )
+
+ broad = contract_for_quote("whether the request was newly submitted or already present")
+ narrow = contract_for_quote("newly submitted or already present")
+
+ assert broad.datum_bindings[0].source_quote != narrow.datum_bindings[0].source_quote
+ assert broad.datum_bindings[0].slot_id == narrow.datum_bindings[0].slot_id
+ assert request_slot_contracts_agree(broad, narrow)
+
+
@pytest.mark.asyncio
async def test_producer_accepts_two_agreeing_empty_contracts() -> None:
calls = 0
@@ -504,6 +554,55 @@ async def test_producer_retries_disagreement_until_consecutive_contracts_agree()
assert responses == []
+async def test_producer_binding_and_decline_disagreement_never_mints_a_contract() -> None:
+ target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256="1" * 64,
+ )
+ request = _input(
+ latest_request="Return whether submission was new or existing.",
+ datum_targets=(target,),
+ )
+ source_id = _source_id(request, "whether submission was new or existing")
+ slot = {
+ "source_id": source_id,
+ "source_quote": "whether submission was new or existing",
+ "plane": "run",
+ "pinability": "shapeless_valid",
+ "antecedent_family": "unconditional",
+ }
+ binding = {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": False,
+ "source_id": source_id,
+ "source_quote": "whether submission was new or existing",
+ }
+ decline = {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": True,
+ }
+ responses = [
+ {"version": "1", "slots": [slot], "datum_bindings": [resolution]}
+ for resolution in (binding, decline, binding, decline)
+ ]
+
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ return responses.pop(0)
+
+ result = await produce_request_slots(request=request, handler=handler, timeout_seconds=1.0)
+
+ assert result.status == "failure"
+ assert result.attempts == 4
+ assert result.failure_kind == RequestSlotProducerFailureKind.INCONSISTENT_OUTPUT
+ assert result.contract is None
+ assert responses == []
+
+
@pytest.mark.asyncio
async def test_producer_requires_plane_and_pinability_agreement() -> None:
request = _input(latest_request="Return status.")
@@ -727,3 +826,266 @@ def test_contract_namespace_never_collides_with_legacy_semantic_paths() -> None:
assert contract.slots[0].canonical_path != "output.path_is_login_only"
assert contract.slots[0].canonical_path.startswith("output.request_slot_")
assert request_slots_module.PROMPT_NAME == "workflow-copilot-request-slots"
+
+
+def test_datum_targets_must_map_injectively_to_independent_slots() -> None:
+ request = _input(
+ latest_request="Return the request id.",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.request_id",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ RequestSlotDatumTargetV1(
+ criterion_index=1,
+ datum_field="output_path",
+ datum_value="output.confirmation_id",
+ criterion_outcome_sha256="2" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, "request id")
+ slot = RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote="request id",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ )
+
+ with pytest.raises(ValueError, match="injectively"):
+ canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(slot,),
+ datum_bindings=tuple(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ declined=False,
+ source_id=source_id,
+ source_quote="request id",
+ )
+ for target in request.datum_targets
+ ),
+ ),
+ )
+
+
+def test_datum_binding_model_accepts_only_identity_and_new_source_evidence() -> None:
+ binding = RequestSlotDatumBindingDeclarationV1.model_validate(
+ {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": False,
+ "source_id": "u0",
+ "source_quote": "request id",
+ }
+ )
+
+ assert binding.model_dump() == {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": False,
+ "source_id": "u0",
+ "source_quote": "request id",
+ }
+ with pytest.raises(ValidationError):
+ RequestSlotDatumBindingDeclarationV1.model_validate(
+ {key: value for key, value in binding.model_dump().items() if key != "declined"}
+ )
+ with pytest.raises(ValidationError):
+ RequestSlotDatumBindingDeclarationV1.model_validate(
+ {
+ **binding.model_dump(),
+ "datum_value": "output.request_id",
+ "criterion_outcome_sha256": "1" * 64,
+ }
+ )
+
+
+def test_datum_decline_model_accepts_only_identity_and_explicit_decline() -> None:
+ decline = RequestSlotDatumDeclineDeclarationV1.model_validate(
+ {
+ "criterion_index": 0,
+ "datum_field": "classification_output_key",
+ "declined": True,
+ }
+ )
+
+ assert decline.model_dump() == {
+ "criterion_index": 0,
+ "datum_field": "classification_output_key",
+ "declined": True,
+ }
+ for extra in (
+ {"datum_value": "submission_state"},
+ {"criterion_outcome_sha256": "1" * 64},
+ {"source_id": "u0", "source_quote": "submission state"},
+ ):
+ with pytest.raises(ValidationError):
+ RequestSlotDatumDeclineDeclarationV1.model_validate({**decline.model_dump(), **extra})
+
+
[email protected]("resolution_kind", ["missing", "unknown", "duplicate"])
+def test_datum_resolution_preserves_exact_identity_coverage(resolution_kind: str) -> None:
+ request = _input(
+ latest_request="Return status.",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.status",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ resolution = RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=1 if resolution_kind == "unknown" else 0,
+ datum_field="output_path",
+ declined=True,
+ )
+ resolutions = () if resolution_kind == "missing" else (resolution,)
+ if resolution_kind == "duplicate":
+ resolutions = (resolution, resolution)
+
+ with pytest.raises(ValueError, match="cover every|unknown or duplicate"):
+ canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(version="1", slots=(), datum_bindings=resolutions),
+ )
+
+
+def test_typed_decline_preserves_bound_sibling_and_exact_target_coverage() -> None:
+ request = _input(
+ latest_request="Return whether submission was new or existing and ignore the adjacent account number.",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.submission_state",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ RequestSlotDatumTargetV1(
+ criterion_index=1,
+ datum_field="output_path",
+ datum_value="output.confirmation_id",
+ criterion_outcome_sha256="2" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, "whether submission was new or existing")
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote="whether submission was new or existing",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=source_id,
+ source_quote="whether submission was new or existing",
+ ),
+ RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=1,
+ datum_field="output_path",
+ declined=True,
+ ),
+ ),
+ ),
+ )
+
+ assert [(binding.criterion_index, binding.datum_field) for binding in contract.datum_bindings] == [
+ (0, "output_path")
+ ]
+ assert [binding.model_dump() for binding in contract.datum_bindings] == [
+ {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "datum_value": "output.submission_state",
+ "criterion_outcome_sha256": "1" * 64,
+ "source_id": source_id,
+ "source_quote": "whether submission was new or existing",
+ "slot_id": contract.slots[0].slot_id,
+ }
+ ]
+ assert [decline.model_dump() for decline in contract.datum_declines] == [
+ {
+ "criterion_index": 1,
+ "datum_field": "output_path",
+ "datum_value": "output.confirmation_id",
+ "criterion_outcome_sha256": "2" * 64,
+ }
+ ]
+
+
+def test_untargeted_request_digest_preserves_legacy_slot_identity() -> None:
+ request = _input(latest_request="Return status.")
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=_envelope(request, ("status", "run", "shapeless_valid")),
+ )
+
+ assert contract.request_digest == "07a98d0716f45eb84e11d7435ad3431331253f87cbac549a8e6514e4f97d658a"
+ assert contract.slots[0].slot_id == "70f1f255440256d7e42a583556a62d20c2b2b051ec990bd8fef83c2f3db97638"
+
+
+def test_request_digest_covers_every_datum_target_identity_member_and_order() -> None:
+ request = _input(latest_request="Return status.")
+ base_target = RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.status",
+ criterion_outcome_sha256="1" * 64,
+ )
+
+ def contract_for(targets: tuple[RequestSlotDatumTargetV1, ...]) -> RequestSlotContractV1:
+ targeted = request.model_copy(update={"datum_targets": targets})
+ return canonicalize_request_slots(
+ request=targeted,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(),
+ datum_bindings=tuple(
+ RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ declined=True,
+ )
+ for target in targets
+ ),
+ ),
+ )
+
+ base_digest = contract_for((base_target,)).request_digest
+ for mutated_target in (
+ base_target.model_copy(update={"criterion_index": 1}),
+ base_target.model_copy(update={"datum_field": "classification_output_key"}),
+ base_target.model_copy(update={"datum_value": "output.state"}),
+ base_target.model_copy(update={"criterion_outcome_sha256": "2" * 64}),
+ ):
+ assert contract_for((mutated_target,)).request_digest != base_digest
+
+ second_target = base_target.model_copy(
+ update={
+ "criterion_index": 1,
+ "datum_value": "output.state",
+ "criterion_outcome_sha256": "2" * 64,
+ }
+ )
+ pair_digest = contract_for((base_target, second_target)).request_digest
+ assert pair_digest != base_digest
+ assert contract_for((second_target, base_target)).request_digest != pair_digest