Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: the log-only feature flag can now disable any draft-blocking check, not just 3 of 7 (#7616)
Andrew Neilson2ef1714d84a5
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex a8fc75b26..f2341b5fe 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -699,6 +699,7 @@ export interface EvalBrowserSession extends Eval {kind: "browser_session";session_id: string;arm?: string | null;+ difficulty?: "easy" | "medium" | "hard" | null;model?: string | null;task_id?: string | null;task_title?: string | null;diff --git a/skyvern-frontend/src/components/SidebarTreeNav.test.tsx b/skyvern-frontend/src/components/SidebarTreeNav.test.tsxnew file mode 100644index 000000000..155479560--- /dev/null+++ b/skyvern-frontend/src/components/SidebarTreeNav.test.tsx@@ -0,0 +1,73 @@+import { cleanup, render, screen } from "@testing-library/react";+import { MemoryRouter } from "react-router-dom";+import { afterEach, describe, expect, it } from "vitest";++import { SidebarTreeNav } from "./SidebarTreeNav";+import { useSidebarStore } from "@/store/SidebarStore";++describe("SidebarTreeNav", () => {+ afterEach(() => {+ cleanup();+ window.localStorage.clear();+ useSidebarStore.setState({ collapsed: false });+ });++ it("opens external top-level items in a new tab", () => {+ render(+ <MemoryRouter>+ <SidebarTreeNav+ items={[+ {+ label: "Home",+ to: "https://app.skyvern.com/discover",+ external: true,+ icon: <span />,+ },+ ]}+ collapsed={false}+ />+ </MemoryRouter>,+ );++ const homeLink = screen.getByRole("link", { name: "Home" });++ expect(homeLink.getAttribute("href")).toBe(+ "https://app.skyvern.com/discover",+ );+ expect(homeLink.getAttribute("target")).toBe("_blank");+ expect(homeLink.getAttribute("rel")).toBe("noopener noreferrer");+ });++ it("opens external collapsed group triggers in a new tab", () => {+ render(+ <MemoryRouter>+ <SidebarTreeNav+ items={[+ {+ label: "Agents",+ to: "https://app.skyvern.com/agents",+ external: true,+ icon: <span />,+ children: [+ {+ label: "All Agents",+ to: "https://app.skyvern.com/agents",+ external: true,+ },+ ],+ },+ ]}+ collapsed+ />+ </MemoryRouter>,+ );++ const agentsTrigger = screen.getByTitle("Agents");++ expect(agentsTrigger.getAttribute("href")).toBe(+ "https://app.skyvern.com/agents",+ );+ expect(agentsTrigger.getAttribute("target")).toBe("_blank");+ expect(agentsTrigger.getAttribute("rel")).toBe("noopener noreferrer");+ });+});diff --git a/skyvern-frontend/src/components/SidebarTreeNav.tsx b/skyvern-frontend/src/components/SidebarTreeNav.tsxindex 43ab33ef9..a60054e78 100644--- a/skyvern-frontend/src/components/SidebarTreeNav.tsx+++ b/skyvern-frontend/src/components/SidebarTreeNav.tsx@@ -1,10 +1,12 @@import { Link, useLocation } from "react-router-dom";import { ChevronDownIcon } from "@radix-ui/react-icons";import {+ forwardRef,useCallback,useEffect,useRef,useState,+ type ComponentPropsWithoutRef,type ReactNode,} from "react";@@ -32,12 +34,17 @@ type SidebarNavItem = {label: string;to: string;icon: ReactNode;+ external?: boolean;badge?: string;defaultOpen?: boolean | (() => boolean);initialVisibleChildren?: number;children?: Array<SidebarNavChild>;};+type SidebarNavLinkProps = Omit<ComponentPropsWithoutRef<"a">, "href"> & {+ item: SidebarNavItem;+};+type Props = {items: Array<SidebarNavItem>;collapsed?: boolean;@@ -46,6 +53,24 @@ type Props = {const OPEN_GROUPS_STORAGE_KEY = "skyvern-sidebar-open-groups";const EXPANDED_CHILDREN_STORAGE_KEY = "skyvern-sidebar-expanded-children";+const SidebarNavLink = forwardRef<HTMLAnchorElement, SidebarNavLinkProps>(+ function SidebarNavLink({ item, ...props }, ref) {+ if (item.external) {+ return (+ <a+ ref={ref}+ {...props}+ href={item.to}+ target="_blank"+ rel="noopener noreferrer"+ />+ );+ }++ return <Link ref={ref} to={item.to} {...props} />;+ },+);+function readStoredRecord(key: string) {try {const raw = window.localStorage.getItem(key);@@ -182,8 +207,8 @@ function CollapsedNavItem({return (<DropdownMenu open={open} onOpenChange={setOpen} modal={false}><DropdownMenuTrigger asChild>- <Link- to={item.to}+ <SidebarNavLink+ item={item}title={item.label}className={triggerClassName}onClick={() => {@@ -194,7 +219,7 @@ function CollapsedNavItem({onMouseLeave={() => scheduleHover(false, FLYOUT_CLOSE_DELAY_MS)}><span className={iconClassName}>{item.icon}</span>- </Link>+ </SidebarNavLink></DropdownMenuTrigger><DropdownMenuContentside="right"@@ -204,8 +229,8 @@ function CollapsedNavItem({className="min-w-48 border-neutral-200 bg-white p-1 text-neutral-900 dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-100"><DropdownMenuItem asChild>- <Link- to={item.to}+ <SidebarNavLink+ item={item}className={cn(menuChildItemClassName, "font-semibold")}><span className="flex size-3.5 shrink-0 items-center justify-center text-neutral-500 dark:text-neutral-400">@@ -217,7 +242,7 @@ function CollapsedNavItem({{item.badge}</span>) : null}- </Link>+ </SidebarNavLink></DropdownMenuItem><DropdownMenuSeparator />{(item.children ?? []).map((child) => {@@ -442,8 +467,8 @@ function SidebarTreeNav({ items, collapsed: collapsedOverride }: Props) {menuChildItemClassName={menuChildItemClassName}/>) : (- <Link- to={item.to}+ <SidebarNavLink+ item={item}title={collapsed ? item.label : undefined}className={topLevelClassName}>@@ -451,7 +476,7 @@ function SidebarTreeNav({ items, collapsed: collapsedOverride }: Props) {{!collapsed && (<span className="min-w-0 flex-1 truncate">{item.label}</span>)}- </Link>+ </SidebarNavLink>)}{!collapsed && hasChildren && open ? (diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 39a3d2207..3a6da9187 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -79,19 +79,23 @@ _FINAL_BROWSER_SESSION_STATUSES: frozenset[str] = frozenset({"completed", "faileDEFINITION_CONTRACT_UNSATISFIED_GATE_ID = "definition_contract_unsatisfied"RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID = "recorded_outcome_grounding_binder_ceiling"SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID = "synthesized_parameter_binding_ambiguous"-_POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(+OUTPUT_CONTRACT_ACTUATION_GATE_ID = "output_contract_actuation"+METADATA_RUN_PREFLIGHT_REJECT_GATE_ID = "metadata_run_preflight_reject"+UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID = "uncovered_output_rescout_steer"+RECORDED_OUTCOME_GROUNDING_GATE_ID = "recorded_outcome_grounding"+# Every author-time gate that has a suppression seam, and therefore everything the flag+# payload and the local blanket can disable. Security lanes are absent by construction, not+# by omission: they never call record_author_time_gate_ablation_event, so no payload reaches+# them. A gate that gains a seam belongs here, or the flag cannot address it.+AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset({DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,- }-)-_LOCAL_AUTHOR_TIME_GATE_LOG_ONLY_IDS = frozenset(- {- "output_contract_actuation",- "metadata_run_preflight_reject",- "uncovered_output_rescout_steer",- "recorded_outcome_grounding",+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,+ UNCOVERED_OUTPUT_RESCOUT_STEER_GATE_ID,+ RECORDED_OUTCOME_GROUNDING_GATE_ID,})CodeArtifactMetadataValue: TypeAlias = (@@ -564,17 +568,17 @@ class AgentContext:def cache_copilot_author_time_gate_log_only_ids(ctx: AgentContext, resolved_ids: frozenset[str]) -> None:- ineligible_ids = resolved_ids - _POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS+ ineligible_ids = resolved_ids - AUTHOR_TIME_GATE_LOG_ONLY_IDSfor gate_id in sorted(ineligible_ids):LOG.info("copilot_gate_log_only_ineligible", gate_id=gate_id)- ctx.author_time_gate_log_only_ids = resolved_ids & _POSTHOG_AUTHOR_TIME_GATE_LOG_ONLY_IDS+ ctx.author_time_gate_log_only_ids = resolved_ids & AUTHOR_TIME_GATE_LOG_ONLY_IDSdef copilot_author_time_gate_log_only_enabled(ctx: AgentContext, gate_id: str) -> bool:local_blanket_enabled = (not settings.is_cloud_environment()and settings.WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY- and gate_id in _LOCAL_AUTHOR_TIME_GATE_LOG_ONLY_IDS+ and gate_id in AUTHOR_TIME_GATE_LOG_ONLY_IDS)return local_blanket_enabled or gate_id in ctx.author_time_gate_log_only_idsdiff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 57b7b3dde..6eca06a73 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -191,6 +191,8 @@ from skyvern.forge.sdk.copilot.request_policy import (from skyvern.forge.sdk.copilot.result_evidence import loaded_result_source_produciblefrom skyvern.forge.sdk.copilot.runtime import (DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,AgentContext,@@ -2877,8 +2879,8 @@ _METADATA_FAMILY_REJECT_FAMILIES = frozenset(})_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3-_OUTPUT_CONTRACT_ABLATION_GATE_ID = "output_contract_actuation"-_METADATA_PREFLIGHT_ABLATION_GATE_ID = "metadata_run_preflight_reject"+_OUTPUT_CONTRACT_ABLATION_GATE_ID = OUTPUT_CONTRACT_ACTUATION_GATE_ID+_METADATA_PREFLIGHT_ABLATION_GATE_ID = METADATA_RUN_PREFLIGHT_REJECT_GATE_ID_VALUE_BEARING_ROOT_GUIDANCE_PATH = "output"_VALUE_BEARING_PREARM_FINGERPRINT_PREFIX = "value-bearing:prearm:"_VALUE_BEARING_GUIDANCE_FINGERPRINT_PREFIX = "value-bearing:guidance:"@@ -2943,6 +2945,32 @@ def _record_output_contract_ablation_event()+def _record_output_contract_actuation_ablation_event(+ ctx: AgentContext,+ workflow_yaml: str,+ *,+ reason_code: str,+ signature: str,+ block_labels: Sequence[str],+ required_paths: Iterable[str],+) -> bool:+ if not copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):+ return False+ payload: AuthorTimeGateAblationPayload = {+ "block_labels": list(block_labels),+ "canonical_output_contract_signature": signature,+ "canonical_required_child_paths": sorted(required_paths),+ }+ return record_author_time_gate_ablation_event(+ ctx,+ gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,+ reason_code=reason_code,+ fingerprint=_output_contract_structural_fingerprint(workflow_yaml, signature),+ blocked_tool="update_workflow",+ payload=payload,+ )++@dataclass(frozen=True)class _RuntimeOutputRepairContract:required_paths: set[str]@@ -12817,10 +12845,23 @@ async def _update_workflow(params.get("code_artifact_metadata"),))+ missing_labels = (+ []+ if missing_metadata_error is None+ else _missing_code_artifact_metadata_labels(workflow_yaml, ctx, params.get("code_artifact_metadata"))+ )+ metadata_reject_ablated = False+ if missing_metadata_error is not None and _record_output_contract_actuation_ablation_event(+ ctx,+ workflow_yaml,+ reason_code=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,+ signature=(output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""),+ block_labels=missing_labels,+ required_paths=required_child_output_paths,+ ):+ missing_metadata_error = None+ metadata_reject_ablated = Trueif missing_metadata_error is not None:- missing_labels = _missing_code_artifact_metadata_labels(- workflow_yaml, ctx, params.get("code_artifact_metadata")- )missing_metadata_output_facts = _missing_requested_output_facts(required_child_output_paths,reason_code=output_path_coverage_reason_code,@@ -12923,6 +12964,26 @@ async def _update_workflow()code_artifact_metadata = normalization.normalizedcode_artifact_metadata_error = normalization.error+ # A schema incompatibility rejects below regardless, so suppressing here would emit an+ # ablation event for a save that never lands and overstate what the flag let through.+ if (+ code_artifact_metadata_error is not None+ and not normalization.schema_incompatibilities+ and (+ metadata_reject_ablated+ or _record_output_contract_actuation_ablation_event(+ ctx,+ workflow_yaml,+ reason_code=_METADATA_NORMALIZATION_REJECT_FAMILY,+ signature=(+ output_contract_evaluation.canonical_signature if output_contract_evaluation is not None else ""+ ),+ block_labels=normalization.offending_labels,+ required_paths=required_child_output_paths,+ )+ )+ ):+ code_artifact_metadata_error = Noneif code_artifact_metadata_error is not None:record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)normalization_reject_count = _record_output_contract_family_reject(diff --git a/tests/unit/test_copilot_author_time_gate_log_only.py b/tests/unit/test_copilot_author_time_gate_log_only.pyindex 890e71354..e79e1b737 100644--- a/tests/unit/test_copilot_author_time_gate_log_only.py+++ b/tests/unit/test_copilot_author_time_gate_log_only.py@@ -1,5 +1,9 @@from __future__ import annotations+import copy+import json+import pathlib+import refrom types import SimpleNamespacefrom unittest.mock import AsyncMock@@ -7,7 +11,7 @@ import pytestfrom structlog.testing import capture_logsfrom skyvern.config import settings-from skyvern.forge.sdk.copilot import agent+from skyvern.forge.sdk.copilot import agent, runtimefrom skyvern.forge.sdk.copilot.build_test_outcome import (RecordedBuildTestOutcome,RecordedOutcomeBindingConstraint,@@ -17,7 +21,10 @@ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicyfrom skyvern.forge.sdk.copilot.runtime import (+ AUTHOR_TIME_GATE_LOG_ONLY_IDS,DEFINITION_CONTRACT_UNSATISFIED_GATE_ID,+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,RECORDED_OUTCOME_GROUNDING_BINDER_CEILING_GATE_ID,SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,cache_copilot_author_time_gate_log_only_ids,@@ -26,6 +33,9 @@ from skyvern.forge.sdk.copilot.runtime import ()from skyvern.forge.sdk.copilot.tools import workflow_update as wufrom tests.unit.copilot_test_helpers import make_copilot_ctx+from tests.unit.test_copilot_code_artifact_metadata_violations import _valid_metadata++_GOAL_PATH_PLACEHOLDER = "<fill: output JSON path(s) carrying requested goal values>"_UNREFERENCED_DEFINITION_YAML = """\title: Submit reusable request@@ -327,3 +337,310 @@ def test_unselected_new_gate_keeps_enforcing() -> None:is False)assert ctx.author_time_gate_ablation_events == []+++_MISSING_METADATA_OUTPUT_YAML = """\+title: Collect provider records+workflow_definition:+ parameters:+ - {parameter_type: output, key: records}+ blocks:+ - block_type: code+ label: collect_records+ parameter_keys: []+ code: |+ await page.goto("https://example.com/records")+ return {"records": [{"number": "123"}]}+"""+++_IMPOSABLE_METADATA = {+ "block_label": "collect_records",+ "declared_goal": "Collect the provider records",+ "terminal_verifier_expectations": [{"goal_value_paths": ["records[].number"]}],+}+++def _missing_metadata_ctx() -> CopilotContext:+ ctx = make_copilot_ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ return ctx+++def test_metadata_family_gate_ids_are_posthog_eligible() -> None:+ ctx = make_copilot_ctx()+ with capture_logs() as logs:+ cache_copilot_author_time_gate_log_only_ids(+ ctx,+ frozenset(+ {+ OUTPUT_CONTRACT_ACTUATION_GATE_ID,+ METADATA_RUN_PREFLIGHT_REJECT_GATE_ID,+ "raw_secret_leak",+ "credential_reference_presence",+ }+ ),+ )++ assert ctx.author_time_gate_log_only_ids == frozenset(+ {OUTPUT_CONTRACT_ACTUATION_GATE_ID, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID}+ )+ assert copilot_author_time_gate_log_only_enabled(ctx, OUTPUT_CONTRACT_ACTUATION_GATE_ID) is True+ assert copilot_author_time_gate_log_only_enabled(ctx, METADATA_RUN_PREFLIGHT_REJECT_GATE_ID) is True+ ineligible = {log["gate_id"] for log in logs if log["event"] == "copilot_gate_log_only_ineligible"}+ assert ineligible == {"raw_secret_leak", "credential_reference_presence"}+ for security_gate_id in ("raw_secret_leak", "credential_reference_presence"):+ assert copilot_author_time_gate_log_only_enabled(ctx, security_gate_id) is False+++async def test_missing_metadata_reject_persists_draft_under_log_only(monkeypatch: pytest.MonkeyPatch) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _missing_metadata_ctx()+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))++ result = await wu._update_workflow(+ {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ assert result["ok"] is True+ assert ctx.latest_recorded_build_test_outcome is None+ event = ctx.author_time_gate_ablation_events[-1]+ assert event.gate_id == OUTPUT_CONTRACT_ACTUATION_GATE_ID+ assert event.reason_code == "missing_code_artifact_metadata"+ assert event.log_only is True+ assert event.blocked_tool == "update_workflow"+ assert event.fingerprint+ assert event.payload["block_labels"] == ["collect_records"]+ # No request-policy output contract is in play, so there is nothing to scaffold from and the+ # draft persists carrying no metadata rows.+ assert not ctx.code_artifact_metadata+++async def test_missing_metadata_reject_stays_enforcing_when_gate_unselected(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _missing_metadata_ctx()++ result = await wu._update_workflow(+ {"workflow_yaml": _MISSING_METADATA_OUTPUT_YAML},+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ assert result["ok"] is False+ assert "code_artifact_metadata" in str(result["error"])+ assert ctx.author_time_gate_ablation_events == []+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+++_NORMALIZATION_YAML = """\+title: Collect provider records+workflow_definition:+ parameters: []+ blocks:+ - block_type: code+ label: collect_records+ parameter_keys: []+ code: |+ await page.goto("https://example.com/records")+ return {"records": [{"number": "123"}]}+"""+++_TWO_BLOCK_YAML = """\+title: Collect provider records+workflow_definition:+ parameters: []+ blocks:+ - block_type: code+ label: block_ok+ parameter_keys: []+ code: |+ await page.goto("https://example.com/a")+ return {"records": [{"number": "1"}]}+ - block_type: code+ label: block_bad+ parameter_keys: []+ code: |+ await page.goto("https://example.com/b")+ return {"records": [{"number": "2"}]}+"""+++async def test_normalization_seam_keeps_rows_the_enforcing_pass_accepted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_successful_update(monkeypatch)+ ctx = make_copilot_ctx()+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))+ conforming = copy.deepcopy(_valid_metadata("block_ok"))+ conforming["claimed_outcomes"][0]["goal_value_paths"] = [_GOAL_PATH_PLACEHOLDER]++ result = await wu._update_workflow(+ {+ "workflow_yaml": _TWO_BLOCK_YAML,+ "code_artifact_metadata": [conforming, {"block_label": "block_bad", "declared_goal": "g"}],+ },+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ assert result["ok"] is True+ assert ctx.latest_recorded_build_test_outcome is None+ assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"+ # block_ok normalizes on the enforcing pass; the seam must not drop it while relieving block_bad.+ assert set(ctx.code_artifact_metadata or {}) == {"block_ok"}+++async def test_schema_incompatibility_still_rejects_and_records_no_ablation_event(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_successful_update(monkeypatch)+ ctx = make_copilot_ctx()+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))+ incompatible = copy.deepcopy(_valid_metadata("collect_records"))+ incompatible["terminal_verifier_expectations"] = [+ {+ "goal_value_paths": ["records[].number"],+ "extraction_schema": json.dumps(+ {"type": "object", "properties": {"unrelated": {"type": "string"}}, "required": ["unrelated"]}+ ),+ }+ ]++ result = await wu._update_workflow(+ {"workflow_yaml": _NORMALIZATION_YAML, "code_artifact_metadata": [incompatible]},+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ if result["ok"] is False and ctx.author_time_gate_ablation_events:+ raise AssertionError("a rejected save must not also report a log-only ablation for the same call")+++async def test_undefaultable_normalization_contradiction_saves_no_row_under_log_only(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_successful_update(monkeypatch)+ ctx = make_copilot_ctx()+ cache_copilot_author_time_gate_log_only_ids(ctx, frozenset({OUTPUT_CONTRACT_ACTUATION_GATE_ID}))++ result = await wu._update_workflow(+ {+ "workflow_yaml": _NORMALIZATION_YAML,+ "code_artifact_metadata": [+ {+ "block_label": "collect_records",+ "declared_goal": "collect records",+ "claimed_outcomes": [{"id": "claim:x", "scope": "outcome", "text": "x", "status": "satisfied"}],+ }+ ],+ },+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ assert result["ok"] is True+ assert ctx.author_time_gate_ablation_events[-1].reason_code == "metadata_normalization"+ assert not ctx.code_artifact_metadata+++async def test_normalization_reject_stays_enforcing_when_gate_unselected(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_successful_update(monkeypatch)+ ctx = make_copilot_ctx()++ result = await wu._update_workflow(+ {+ "workflow_yaml": _NORMALIZATION_YAML,+ "code_artifact_metadata": [_IMPOSABLE_METADATA],+ },+ ctx,+ allow_missing_credentials=True,+ allow_static_output_uncertainty=True,+ )++ assert result["ok"] is False+ assert ctx.author_time_gate_ablation_events == []+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+++def test_eligible_gate_set_matches_the_seamed_gates_exactly() -> None:+ """Gate ids reaching an ablation recorder are the ground truth for what can be disabled:+ the flag's list must cover all of them, and every listed id must emit ablation events."""+ recorders = r"(?:record_author_time_gate_ablation_event|_record_output_contract_ablation_event)"+ seamed: set[str] = set()+ for path in pathlib.Path("skyvern/forge/sdk/copilot").rglob("*.py"):+ source = path.read_text()+ for match in re.finditer(recorders + r"\((.*?)\)", source, re.S):+ gate = re.search(r"gate_id=([A-Za-z_][A-Za-z_0-9]*|\"[a-z_]+\")", match.group(1))+ if gate is None:+ continue+ token = gate.group(1)+ if token == "gate_id":+ continue+ resolved = (+ token.strip('"') if token.startswith('"') else getattr(runtime, token, None) or getattr(wu, token, None)+ )+ if isinstance(resolved, str):+ seamed.add(resolved)++ assert seamed, "found no gate ids; the seam-detection regex needs updating"+ unreachable = seamed - AUTHOR_TIME_GATE_LOG_ONLY_IDS+ assert not unreachable, (+ f"these gates have a suppression seam but the flag cannot disable them: {sorted(unreachable)}. "+ "Add them to AUTHOR_TIME_GATE_LOG_ONLY_IDS."+ )+ eventless = AUTHOR_TIME_GATE_LOG_ONLY_IDS - seamed+ assert not eventless, (+ f"these flag-eligible gates emit no ablation events when suppressed: {sorted(eventless)}. "+ "Suppression without events cannot be graded or monitored — add a recording seam."+ )+++_SECURITY_GATE_IDS = frozenset(+ {+ "raw_secret_leak",+ "raw_secret_handling",+ "code_safety_reject",+ "credential_reference_presence",+ "unapproved_credential_reference",+ "credential_scout_reopen",+ }+)+++def test_eligible_gate_ids_are_exactly_the_sanctioned_seven() -> None:+ """Security-critical: OutputPolicy security gates must never become flag-suppressible.+ Exact-content assertion is intentional — any change to this set is a security decision."""+ assert AUTHOR_TIME_GATE_LOG_ONLY_IDS == frozenset(+ {+ "definition_contract_unsatisfied",+ "recorded_outcome_grounding_binder_ceiling",+ "synthesized_parameter_binding_ambiguous",+ "output_contract_actuation",+ "metadata_run_preflight_reject",+ "uncovered_output_rescout_steer",+ "recorded_outcome_grounding",+ }+ )+ assert not (AUTHOR_TIME_GATE_LOG_ONLY_IDS & _SECURITY_GATE_IDS)