Skip to content

Automate browser based workflows with AI

Copilot: the log-only feature flag can now disable any draft-blocking check, not just 3 of 7 (#7616)

Andrew Neilson2ef1714d84a5
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index a8fc75b26..f2341b5fe 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -699,6 +699,7 @@ export interface EvalBrowserSession extends Eval {
kind: "browser_session";
session_id: string;
arm?: string | null;
+ difficulty?: "easy" | "medium" | "hard" | null;
model?: string | null;
task_id?: string | null;
task_title?: string | null;
diff --git a/skyvern-frontend/src/components/SidebarTreeNav.test.tsx b/skyvern-frontend/src/components/SidebarTreeNav.test.tsx
new file mode 100644
index 000000000..155479560
--- /dev/null
+++ b/skyvern-frontend/src/components/SidebarTreeNav.test.tsx
@@ -0,0 +1,73 @@
+import { cleanup, render, screen } from "@testing-library/react";
+import { MemoryRouter } from "react-router-dom";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { SidebarTreeNav } from "./SidebarTreeNav";
+import { useSidebarStore } from "@/store/SidebarStore";
+
+describe("SidebarTreeNav", () => {
+ afterEach(() => {
+ cleanup();
+ window.localStorage.clear();
+ useSidebarStore.setState({ collapsed: false });
+ });
+
+ it("opens external top-level items in a new tab", () => {
+ render(
+ <MemoryRouter>
+ <SidebarTreeNav
+ items={[
+ {
+ label: "Home",
+ to: "https://app.skyvern.com/discover",
+ external: true,
+ icon: <span />,
+ },
+ ]}
+ collapsed={false}
+ />
+ </MemoryRouter>,
+ );
+
+ const homeLink = screen.getByRole("link", { name: "Home" });
+
+ expect(homeLink.getAttribute("href")).toBe(
+ "https://app.skyvern.com/discover",
+ );
+ expect(homeLink.getAttribute("target")).toBe("_blank");
+ expect(homeLink.getAttribute("rel")).toBe("noopener noreferrer");
+ });
+
+ it("opens external collapsed group triggers in a new tab", () => {
+ render(
+ <MemoryRouter>
+ <SidebarTreeNav
+ items={[
+ {
+ label: "Agents",
+ to: "https://app.skyvern.com/agents",
+ external: true,
+ icon: <span />,
+ children: [
+ {
+ label: "All Agents",
+ to: "https://app.skyvern.com/agents",
+ external: true,
+ },
+ ],
+ },
+ ]}
+ collapsed
+ />
+ </MemoryRouter>,
+ );
+
+ const agentsTrigger = screen.getByTitle("Agents");
+
+ expect(agentsTrigger.getAttribute("href")).toBe(
+ "https://app.skyvern.com/agents",
+ );
+ expect(agentsTrigger.getAttribute("target")).toBe("_blank");
+ expect(agentsTrigger.getAttribute("rel")).toBe("noopener noreferrer");
+ });
+});
diff --git a/skyvern-frontend/src/components/SidebarTreeNav.tsx b/skyvern-frontend/src/components/SidebarTreeNav.tsx
index 43ab33ef9..a60054e78 100644
--- a/skyvern-frontend/src/components/SidebarTreeNav.tsx
+++ b/skyvern-frontend/src/components/SidebarTreeNav.tsx
@@ -1,10 +1,12 @@
import { Link, useLocation } from "react-router-dom";
import { ChevronDownIcon } from "@radix-ui/react-icons";
import {
+ forwardRef,
useCallback,
useEffect,
useRef,
useState,
+ type ComponentPropsWithoutRef,
type ReactNode,
} from "react";
@@ -32,12 +34,17 @@ type SidebarNavItem = {
label: string;
to: string;
icon: ReactNode;
+ external?: boolean;
badge?: string;
defaultOpen?: boolean | (() => boolean);
initialVisibleChildren?: number;
children?: Array<SidebarNavChild>;
};
+type SidebarNavLinkProps = Omit<ComponentPropsWithoutRef<"a">, "href"> & {
+ item: SidebarNavItem;
+};
+
type Props = {
items: Array<SidebarNavItem>;
collapsed?: boolean;
@@ -46,6 +53,24 @@ type Props = {
const OPEN_GROUPS_STORAGE_KEY = "skyvern-sidebar-open-groups";
const EXPANDED_CHILDREN_STORAGE_KEY = "skyvern-sidebar-expanded-children";
+const SidebarNavLink = forwardRef<HTMLAnchorElement, SidebarNavLinkProps>(
+ function SidebarNavLink({ item, ...props }, ref) {
+ if (item.external) {
+ return (
+ <a
+ ref={ref}
+ {...props}
+ href={item.to}
+ target="_blank"
+ rel="noopener noreferrer"
+ />
+ );
+ }
+
+ return <Link ref={ref} to={item.to} {...props} />;
+ },
+);
+
function readStoredRecord(key: string) {
try {
const raw = window.localStorage.getItem(key);
@@ -182,8 +207,8 @@ function CollapsedNavItem({
return (
<DropdownMenu open={open} onOpenChange={setOpen} modal={false}>
<DropdownMenuTrigger asChild>
- <Link
- to={item.to}
+ <SidebarNavLink
+ item={item}
title={item.label}
className={triggerClassName}
onClick={() => {
@@ -194,7 +219,7 @@ function CollapsedNavItem({
onMouseLeave={() => scheduleHover(false, FLYOUT_CLOSE_DELAY_MS)}
>
<span className={iconClassName}>{item.icon}</span>
- </Link>
+ </SidebarNavLink>
</DropdownMenuTrigger>
<DropdownMenuContent
side="right"
@@ -204,8 +229,8 @@ function CollapsedNavItem({
className="min-w-48 border-neutral-200 bg-white p-1 text-neutral-900 dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-100"
>
<DropdownMenuItem asChild>
- <Link
- to={item.to}
+ <SidebarNavLink
+ item={item}
className={cn(menuChildItemClassName, "font-semibold")}
>
<span className="flex size-3.5 shrink-0 items-center justify-center text-neutral-500 dark:text-neutral-400">
@@ -217,7 +242,7 @@ function CollapsedNavItem({
{item.badge}
</span>
) : null}
- </Link>
+ </SidebarNavLink>
</DropdownMenuItem>
<DropdownMenuSeparator />
{(item.children ?? []).map((child) => {
@@ -442,8 +467,8 @@ function SidebarTreeNav({ items, collapsed: collapsedOverride }: Props) {
menuChildItemClassName={menuChildItemClassName}
/>
) : (
- <Link
- to={item.to}
+ <SidebarNavLink
+ item={item}
title={collapsed ? item.label : undefined}
className={topLevelClassName}
>
@@ -451,7 +476,7 @@ function SidebarTreeNav({ items, collapsed: collapsedOverride }: Props) {
{!collapsed && (
<span className="min-w-0 flex-1 truncate">{item.label}</span>
)}
- </Link>
+ </SidebarNavLink>
)}
{!collapsed && hasChildren && open ? (
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 39a3d2207..3a6da9187 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -79,19 +79,23 @@ _FINAL_BROWSER_SESSION_STATUSES: frozenset[str] = frozenset({"completed", "faile
DEFINITION_CONTRACT_UNSATISFIED_GATE_ID = "definition_contract_unsatisfied"
RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID = "recorded_outcome_grounding_binder_ceiling"
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID = "synthesized_parameter_binding_ambiguous"
-_POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(
+OUTPUT_CONTRACT_ACTUATION_GATE_ID = "output_contract_actuation"
+METADATA_RUN_PREFLIGHT_REJECT_GATE_ID = "metadata_run_preflight_reject"
+UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID = "uncovered_output_rescout_steer"
+RECORDED_OUTCOME_GROUNDING_GATE_ID = "recorded_outcome_grounding"
+# Every author-time gate that has a suppression seam, and therefore everything the flag
+# payload and the local blanket can disable. Security lanes are absent by construction, not
+# by omission: they never call record_author_time_gate_ablation_event, so no payload reaches
+# them. A gate that gains a seam belongs here, or the flag cannot address it.
+AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(
{
DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
- }
-)
-_LOCAL_AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(
- {
- "output_contract_actuation",
- "metadata_run_preflight_reject",
- "uncovered_output_rescout_steer",
- "recorded_outcome_grounding",
+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,
+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
+ UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID,
+ RECORDED_OUTCOME_GROUNDING_GATE_ID,
}
)
CodeArtifactMetadataValue: TypeAlias = (
@@ -564,17 +568,17 @@ class AgentContext:
def cache_copilot_author_time_gate_log_only_ids(ctx: AgentContext, resolved_ids: frozenset[str]) -> None:
- ineligible_ids = resolved_ids - _POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS
+ ineligible_ids = resolved_ids - AUTHOR_TIME_GATE_LOG_ONLY_IDS
for gate_id in sorted(ineligible_ids):
LOG.info("copilot_gate_log_only_ineligible", gate_id=gate_id)
- ctx.author_time_gate_log_only_ids = resolved_ids & _POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS
+ ctx.author_time_gate_log_only_ids = resolved_ids & AUTHOR_TIME_GATE_LOG_ONLY_IDS
def copilot_author_time_gate_log_only_enabled(ctx: AgentContext, gate_id: str) -> bool:
local_blanket_enabled = (
not settings.is_cloud_environment()
and settings.WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY
- and gate_id in _LOCAL_AUTHOR_TIME_GATE_LOG_ONLY_IDS
+ and gate_id in AUTHOR_TIME_GATE_LOG_ONLY_IDS
)
return local_blanket_enabled or gate_id in ctx.author_time_gate_log_only_ids
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 57b7b3dde..6eca06a73 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -191,6 +191,8 @@ from skyvern.forge.sdk.copilot.request_policy import (
from skyvern.forge.sdk.copilot.result_evidence import loaded_result_source_producible
from skyvern.forge.sdk.copilot.runtime import (
DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,
RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
AgentContext,
@@ -2877,8 +2879,8 @@ _METADATA_FAMILY_REJECT_FAMILIES = frozenset(
}
)
_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3
-_OUTPUT_CONTRACT_ABLATION_GATE_ID = "output_contract_actuation"
-_METADATA_PREFLIGHT_ABLATION_GATE_ID = "metadata_run_preflight_reject"
+_OUTPUT_CONTRACT_ABLATION_GATE_ID = OUTPUT_CONTRACT_ACTUATION_GATE_ID
+_METADATA_PREFLIGHT_ABLATION_GATE_ID = METADATA_RUN_PREFLIGHT_REJECT_GATE_ID
_VALUE_BEARING_ROOT_GUIDANCE_PATH = "output"
_VALUE_BEARING_PREARM_FINGERPRINT_PREFIX = "value-bearing:prearm:"
_VALUE_BEARING_GUIDANCE_FINGERPRINT_PREFIX = "value-bearing:guidance:"
@@ -2943,6 +2945,32 @@ def _record_output_contract_ablation_event(
)
+def _record_output_contract_actuation_ablation_event(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ *,
+ reason_code: str,
+ signature: str,
+ block_labels: Sequence[str],
+ required_paths: Iterable[str],
+) -> bool:
+ if not copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
+ return False
+ payload: AuthorTimeGateAblationPayload = {
+ "block_labels": list(block_labels),
+ "canonical_output_contract_signature": signature,
+ "canonical_required_child_paths": sorted(required_paths),
+ }
+ return record_author_time_gate_ablation_event(
+ ctx,
+ gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
+ reason_code=reason_code,
+ fingerprint=_output_contract_structural_fingerprint(workflow_yaml, signature),
+ blocked_tool="update_workflow",
+ payload=payload,
+ )
+
+
@dataclass(frozen=True)
class _RuntimeOutputRepairContract:
required_paths: set[str]
@@ -12817,10 +12845,23 @@ async def _update_workflow(
params.get("code_artifact_metadata"),
)
)
+ missing_labels = (
+ []
+ if missing_metadata_error is None
+ else _missing_code_artifact_metadata_labels(workflow_yaml, ctx, params.get("code_artifact_metadata"))
+ )
+ metadata_reject_ablated = False
+ if missing_metadata_error is not None and _record_output_contract_actuation_ablation_event(
+ ctx,
+ workflow_yaml,
+ reason_code=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
+ signature=(output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""),
+ block_labels=missing_labels,
+ required_paths=required_child_output_paths,
+ ):
+ missing_metadata_error = None
+ metadata_reject_ablated = True
if missing_metadata_error is not None:
- missing_labels = _missing_code_artifact_metadata_labels(
- workflow_yaml, ctx, params.get("code_artifact_metadata")
- )
missing_metadata_output_facts = _missing_requested_output_facts(
required_child_output_paths,
reason_code=output_path_coverage_reason_code,
@@ -12923,6 +12964,26 @@ async def _update_workflow(
)
code_artifact_metadata = normalization.normalized
code_artifact_metadata_error = normalization.error
+ # A schema incompatibility rejects below regardless, so suppressing here would emit an
+ # ablation event for a save that never lands and overstate what the flag let through.
+ if (
+ code_artifact_metadata_error is not None
+ and not normalization.schema_incompatibilities
+ and (
+ metadata_reject_ablated
+ or _record_output_contract_actuation_ablation_event(
+ ctx,
+ workflow_yaml,
+ reason_code=_METADATA_NORMALIZATION_REJECT_FAMILY,
+ signature=(
+ output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""
+ ),
+ block_labels=normalization.offending_labels,
+ required_paths=required_child_output_paths,
+ )
+ )
+ ):
+ code_artifact_metadata_error = None
if code_artifact_metadata_error is not None:
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
normalization_reject_count = _record_output_contract_family_reject(
diff --git a/tests/unit/test_copilot_author_time_gate_log_only.py b/tests/unit/test_copilot_author_time_gate_log_only.py
index 890e71354..e79e1b737 100644
--- a/tests/unit/test_copilot_author_time_gate_log_only.py
+++ b/tests/unit/test_copilot_author_time_gate_log_only.py
@@ -1,5 +1,9 @@
from __future__ import annotations
+import copy
+import json
+import pathlib
+import re
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -7,7 +11,7 @@ import pytest
from structlog.testing import capture_logs
from skyvern.config import settings
-from skyvern.forge.sdk.copilot import agent
+from skyvern.forge.sdk.copilot import agent, runtime
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
@@ -17,7 +21,10 @@ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
from skyvern.forge.sdk.copilot.runtime import (
+ AUTHOR_TIME_GATE_LOG_ONLY_IDS,
DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,
+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,
RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
cache_copilot_author_time_gate_log_only_ids,
@@ -26,6 +33,9 @@ from skyvern.forge.sdk.copilot.runtime import (
)
from skyvern.forge.sdk.copilot.tools import workflow_update as wu
from tests.unit.copilot_test_helpers import make_copilot_ctx
+from tests.unit.test_copilot_code_artifact_metadata_violations import _valid_metadata
+
+_GOAL_PATH_PLACEHOLDER = "<fill: output JSON path(s) carrying requested goal values>"
_UNREFERENCED_DEFINITION_YAML = """\
title: Submit reusable request
@@ -327,3 +337,310 @@ def test_unselected_new_gate_keeps_enforcing() -> None:
is False
)
assert ctx.author_time_gate_ablation_events == []
+
+
+_MISSING_METADATA_OUTPUT_YAML = """\
+title: Collect provider records
+workflow_definition:
+ parameters:
+ - {parameter_type: output, key: records}
+ blocks:
+ - block_type: code
+ label: collect_records
+ parameter_keys: []
+ code: |
+ await page.goto("https://example.com/records")
+ return {"records": [{"number": "123"}]}
+"""
+
+
+_IMPOSABLE_METADATA = {
+ "block_label": "collect_records",
+ "declared_goal": "Collect the provider records",
+ "terminal_verifier_expectations": [{"goal_value_paths": ["records[].number"]}],
+}
+
+
+def _missing_metadata_ctx() -> CopilotContext:
+ ctx = make_copilot_ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ return ctx
+
+
+def test_metadata_family_gate_ids_are_posthog_eligible() -> None:
+ ctx = make_copilot_ctx()
+ with capture_logs() as logs:
+ cache_copilot_author_time_gate_log_only_ids(
+ ctx,
+ frozenset(
+ {
+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,
+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,
+ "raw_secret_leak",
+ "credential_reference_presence",
+ }
+ ),
+ )
+
+ assert ctx.author_time_gate_log_only_ids == frozenset(
+ {OUTPUT_CONTRACT_ACTUATION_GATE_ID, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID}
+ )
+ assert copilot_author_time_gate_log_only_enabled(ctx, OUTPUT_CONTRACT_ACTUATION_GATE_ID) is True
+ assert copilot_author_time_gate_log_only_enabled(ctx, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID) is True
+ ineligible = {log["gate_id"] for log in logs if log["event"] == "copilot_gate_log_only_ineligible"}
+ assert ineligible == {"raw_secret_leak", "credential_reference_presence"}
+ for security_gate_id in ("raw_secret_leak", "credential_reference_presence"):
+ assert copilot_author_time_gate_log_only_enabled(ctx, security_gate_id) is False
+
+
+async def test_missing_metadata_reject_persists_draft_under_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _missing_metadata_ctx()
+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
+
+ result = await wu._update_workflow(
+ {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert ctx.latest_recorded_build_test_outcome is None
+ event = ctx.author_time_gate_ablation_events[-1]
+ assert event.gate_id == OUTPUT_CONTRACT_ACTUATION_GATE_ID
+ assert event.reason_code == "missing_code_artifact_metadata"
+ assert event.log_only is True
+ assert event.blocked_tool == "update_workflow"
+ assert event.fingerprint
+ assert event.payload["block_labels"] == ["collect_records"]
+ # No request-policy output contract is in play, so there is nothing to scaffold from and the
+ # draft persists carrying no metadata rows.
+ assert not ctx.code_artifact_metadata
+
+
+async def test_missing_metadata_reject_stays_enforcing_when_gate_unselected(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _missing_metadata_ctx()
+
+ result = await wu._update_workflow(
+ {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is False
+ assert "code_artifact_metadata" in str(result["error"])
+ assert ctx.author_time_gate_ablation_events == []
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+
+
+_NORMALIZATION_YAML = """\
+title: Collect provider records
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: collect_records
+ parameter_keys: []
+ code: |
+ await page.goto("https://example.com/records")
+ return {"records": [{"number": "123"}]}
+"""
+
+
+_TWO_BLOCK_YAML = """\
+title: Collect provider records
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: block_ok
+ parameter_keys: []
+ code: |
+ await page.goto("https://example.com/a")
+ return {"records": [{"number": "1"}]}
+ - block_type: code
+ label: block_bad
+ parameter_keys: []
+ code: |
+ await page.goto("https://example.com/b")
+ return {"records": [{"number": "2"}]}
+"""
+
+
+async def test_normalization_seam_keeps_rows_the_enforcing_pass_accepted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = make_copilot_ctx()
+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
+ conforming = copy.deepcopy(_valid_metadata("block_ok"))
+ conforming["claimed_outcomes"][0]["goal_value_paths"] = [_GOAL_PATH_PLACEHOLDER]
+
+ result = await wu._update_workflow(
+ {
+ "workflow_yaml": _TWO_BLOCK_YAML,
+ "code_artifact_metadata": [conforming, {"block_label": "block_bad", "declared_goal": "g"}],
+ },
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert ctx.latest_recorded_build_test_outcome is None
+ assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"
+ # block_ok normalizes on the enforcing pass; the seam must not drop it while relieving block_bad.
+ assert set(ctx.code_artifact_metadata or {}) == {"block_ok"}
+
+
+async def test_schema_incompatibility_still_rejects_and_records_no_ablation_event(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = make_copilot_ctx()
+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
+ incompatible = copy.deepcopy(_valid_metadata("collect_records"))
+ incompatible["terminal_verifier_expectations"] = [
+ {
+ "goal_value_paths": ["records[].number"],
+ "extraction_schema": json.dumps(
+ {"type": "object", "properties": {"unrelated": {"type": "string"}}, "required": ["unrelated"]}
+ ),
+ }
+ ]
+
+ result = await wu._update_workflow(
+ {"workflow_yaml": _NORMALIZATION_YAML, "code_artifact_metadata": [incompatible]},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ if result["ok"] is False and ctx.author_time_gate_ablation_events:
+ raise AssertionError("a rejected save must not also report a log-only ablation for the same call")
+
+
+async def test_undefaultable_normalization_contradiction_saves_no_row_under_log_only(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = make_copilot_ctx()
+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))
+
+ result = await wu._update_workflow(
+ {
+ "workflow_yaml": _NORMALIZATION_YAML,
+ "code_artifact_metadata": [
+ {
+ "block_label": "collect_records",
+ "declared_goal": "collect records",
+ "claimed_outcomes": [{"id": "claim:x", "scope": "outcome", "text": "x", "status": "satisfied"}],
+ }
+ ],
+ },
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"
+ assert not ctx.code_artifact_metadata
+
+
+async def test_normalization_reject_stays_enforcing_when_gate_unselected(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = make_copilot_ctx()
+
+ result = await wu._update_workflow(
+ {
+ "workflow_yaml": _NORMALIZATION_YAML,
+ "code_artifact_metadata": [_IMPOSABLE_METADATA],
+ },
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is False
+ assert ctx.author_time_gate_ablation_events == []
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+
+
+def test_eligible_gate_set_matches_the_seamed_gates_exactly() -> None:
+ """Gate ids reaching an ablation recorder are the ground truth for what can be disabled:
+ the flag's list must cover all of them, and every listed id must emit ablation events."""
+ recorders = r"(?:record_author_time_gate_ablation_event|_record_output_contract_ablation_event)"
+ seamed: set[str] = set()
+ for path in pathlib.Path("skyvern/forge/sdk/copilot").rglob("*.py"):
+ source = path.read_text()
+ for match in re.finditer(recorders + r"\((.*?)\)", source, re.S):
+ gate = re.search(r"gate_id=([A-Za-z_][A-Za-z_0-9]*|\"[a-z_]+\")", match.group(1))
+ if gate is None:
+ continue
+ token = gate.group(1)
+ if token == "gate_id":
+ continue
+ resolved = (
+ token.strip('"') if token.startswith('"') else getattr(runtime, token, None) or getattr(wu, token, None)
+ )
+ if isinstance(resolved, str):
+ seamed.add(resolved)
+
+ assert seamed, "found no gate ids; the seam-detection regex needs updating"
+ unreachable = seamed - AUTHOR_TIME_GATE_LOG_ONLY_IDS
+ assert not unreachable, (
+ f"these gates have a suppression seam but the flag cannot disable them: {sorted(unreachable)}. "
+ "Add them to AUTHOR_TIME_GATE_LOG_ONLY_IDS."
+ )
+ eventless = AUTHOR_TIME_GATE_LOG_ONLY_IDS - seamed
+ assert not eventless, (
+ f"these flag-eligible gates emit no ablation events when suppressed: {sorted(eventless)}. "
+ "Suppression without events cannot be graded or monitored — add a recording seam."
+ )
+
+
+_SECURITY_GATE_IDS = frozenset(
+ {
+ "raw_secret_leak",
+ "raw_secret_handling",
+ "code_safety_reject",
+ "credential_reference_presence",
+ "unapproved_credential_reference",
+ "credential_scout_reopen",
+ }
+)
+
+
+def test_eligible_gate_ids_are_exactly_the_sanctioned_seven() -> None:
+ """Security-critical: OutputPolicy security gates must never become flag-suppressible.
+ Exact-content assertion is intentional — any change to this set is a security decision."""
+ assert AUTHOR_TIME_GATE_LOG_ONLY_IDS == frozenset(
+ {
+ "definition_contract_unsatisfied",
+ "recorded_outcome_grounding_binder_ceiling",
+ "synthesized_parameter_binding_ambiguous",
+ "output_contract_actuation",
+ "metadata_run_preflight_reject",
+ "uncovered_output_rescout_steer",
+ "recorded_outcome_grounding",
+ }
+ )
+ assert not (AUTHOR_TIME_GATE_LOG_ONLY_IDS & _SECURITY_GATE_IDS)