Skip to content

Automate browser based workflows with AI

Remove three dead Task V3 experiment arms: error-code choice read, unanswerable-field remedy, ATS defaults (SKY-17012) (#8655)

pedrohsdba8dd93c2b162
diff --git a/skyvern/config.py b/skyvern/config.py
index a8954fc98..cb7f14652 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -559,17 +559,9 @@ class Settings(BaseSettings):
# Force-on term only: the arm is randomized per run by the flag of the same name and read through
# run_arm_enabled() in skyvern/forge/taskv3/run_arms.py (SKY-16501).
TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED: bool = False
- # What the system prompt tells the model to do about ONE required SENSITIVE field (government
- # ID, financial detail, legal/eligibility attestation) the payload does not carry: stop the run
- # (off) or leave that field and finish the page (on). An ordinary required field is out of
- # scope -- the ungated sentence above the clause already tells the model to enter the most
- # reasonable value. This arm does not gate the do-not-invent rule, which only
- # TASK_V3_REQUIRED_FIELD_ANSWERS rewrites (SKY-16651). Force-on term only:
- # runs are randomized per run by the flag of the same name, read through run_arm_enabled().
- TASK_V3_UNANSWERABLE_FIELD_REMEDY: bool = False
# Swap in the fill-rule text an AgentFunction supplies (task_v3_required_field_answers_text); with no
- # text supplied the run gets the control prompt. Force-on term only: runs are randomized per run by the flag of the same name, read through
- # run_arm_enabled().
+ # text supplied the run gets the control prompt. Force-on term only: runs are randomized per run by the
+ # flag of the same name, read through run_arm_enabled().
TASK_V3_REQUIRED_FIELD_ANSWERS: bool = False
# State in the system prompt that the task's own instructions win over its general rules. Force-on term only:
# runs are randomized per run by the flag of the same name, read through run_arm_enabled().
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 290c1dc88..731491bb9 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -201,7 +201,6 @@ from skyvern.forge.taskv3.run_arms import (
OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,
REQUIRED_FIELD_ANSWERS_FLAG,
TYPE_COORDINATE_CLICK_FLAG,
- UNANSWERABLE_FIELD_REMEDY_FLAG,
resolve_run_arm,
run_arm_enabled,
)
@@ -2266,13 +2265,6 @@ class ForgeAgent:
organization_id=task.organization_id,
forced=settings.TASK_V3_DATE_SEGMENT_AIM,
)
- await resolve_run_arm(
- context,
- UNANSWERABLE_FIELD_REMEDY_FLAG,
- distinct_id=task.workflow_run_id or task.task_id,
- organization_id=task.organization_id,
- forced=settings.TASK_V3_UNANSWERABLE_FIELD_REMEDY,
- )
await resolve_run_arm(
context,
REQUIRED_FIELD_ANSWERS_FLAG,
@@ -2936,19 +2928,6 @@ class ForgeAgent:
captcha_tools, captcha_guidance = build_captcha_tools(
task, _page_provider, organization_id=organization.organization_id
)
- # Withheld in page-free mode: form-filling defaults must not bias a data-only judgment.
- ats_guidance = None
- if not page_free_validation:
- try:
- ats_guidance = await app.AGENT_FUNCTION.resolve_task_v3_extra_guidance(
- task=task, organization=organization
- )
- except Exception:
- LOG.warning(
- "resolve_task_v3_extra_guidance failed; continuing without it",
- task_id=task.task_id,
- exc_info=True,
- )
# Page-free runs never get the precedence paragraph, so the label would have nothing to refer to.
workflow_system_guidance = task.workflow_system_prompt
if (
@@ -3014,9 +2993,9 @@ class ForgeAgent:
goal_judge=goal_judge,
goal_check_enforce=goal_check_enforce,
extraction_requested=extraction_requested,
- # Only the customer's own instructions can define what counts as done. Auth, captcha and
- # ATS guidance are how-to and answer policy, and the judge is told an outcome its
- # instructions allow is no contradiction.
+ # Only the customer's own instructions can define what counts as done. Auth and captcha
+ # guidance are how-to, and the judge is told an outcome its instructions allow is no
+ # contradiction.
goal_instructions=task.workflow_system_prompt or "",
secret_on_page_at_start=_task_v3_secret_may_be_on_page(
task,
@@ -3059,12 +3038,8 @@ class ForgeAgent:
on_action_round=_on_action_round,
on_pre_action=pre_submit_ring.capture if pre_submit_ring is not None else None,
extra_tools=auth_tools + captcha_tools,
- # ats_guidance before workflow_system_prompt keeps the customer's own text later in
- # the message; position in one system message is a weak signal, not precedence — the
- # real contract is the guidance's own scoping prose (its NEVER list and the explicit
- # link to the base stop-don't-guess rule).
extra_system_guidance="\n\n".join(
- part for part in (auth_guidance, captcha_guidance, ats_guidance, workflow_system_guidance) if part
+ part for part in (auth_guidance, captcha_guidance, workflow_system_guidance) if part
),
completion_probe=completion_probe,
completion_blocker=completion_blocker,
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index d03c156d0..e01b34896 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -974,20 +974,14 @@ class AgentFunction:
async def resolve_billing_tier(self, organization_id: str | None) -> BillingTier:
return BillingTier.UNKNOWN
- # OSS has no ATS-scoped guidance; cloud overrides to supply pre-authorized eligibility defaults
- # behind a flag when the task targets a gated application-tracking-system host.
- async def resolve_task_v3_extra_guidance(self, *, task: Task, organization: Organization) -> str | None:
- return None
-
# (fill text, self-screen bullet) for the v3 required-field-answers arm's treatment prompt. OSS
# supplies none, so a run in that arm's treatment renders the control prompt.
def task_v3_required_field_answers_text(self) -> tuple[str, str] | None:
return None
# Whether v3 offers the task's configured error codes to the model, so a terminal verdict names
- # its own business outcome instead of having one matched on afterwards (SKY-15586). Cloud
- # overrides behind a flag; False keeps codes out of the loop entirely, which makes the whole
- # feature byte-identical to before it existed.
+ # its own business outcome instead of having one matched on afterwards (SKY-15586). No deployment
+ # overrides it; False keeps codes out of the loop entirely.
async def resolve_task_v3_error_code_choice(self, *, task: Task, organization: Organization) -> bool:
return False
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index c417ab51e..e9d62ac0e 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -68,7 +68,6 @@ from skyvern.forge.taskv3.run_arms import (
CUSTOMER_PRECEDENCE_FLAG,
NO_ACTION_HOLD_FLAG,
REQUIRED_FIELD_ANSWERS_FLAG,
- UNANSWERABLE_FIELD_REMEDY_FLAG,
run_arm_enabled,
)
from skyvern.forge.taskv3.tools import (
@@ -120,35 +119,10 @@ GOAL_CHECK_DEADLINE_MARGIN_SECONDS = 2.0
# allowlist scoping the population, not authorizing it: bare tasks (block_type=None) fail closed.
NO_ACTION_HOLD_BLOCK_TYPES = frozenset({BlockType.TASK, BlockType.NAVIGATION})
-# SKY-16651: what the prompt tells the model to do about ONE required SENSITIVE field the payload
-# cannot fill -- the clause's antecedent is "one of those", i.e. the do-not-invent categories, NOT
-# any required field (an ordinary one is handled by the ungated sentence before it). The control
-# remedy stops the whole run, so a single unanswerable field abandons every other field on the
-# page. This arm gates only the remedy; the do-not-invent rule it hangs off is rewritten only in the
-# required-field-answers treatment cells.
-# "every other field you have a basis to fill", not "every other field on the page": the clause sits
-# INSIDE the do-not-invent sentence and one sentence from "Leave optional fields blank when you have
-# no basis to fill them". An unqualified imperative there reads, on a page with a SECOND unanswerable
-# sensitive field, as instructing the model to fill that one -- the exact act the sentence it lives
-# in forbids. "a basis to fill" is the prompt's own existing phrase for the same test.
-#
-# The treatment carries NO completion constraint, deliberately. An earlier revision ended with
-# "report the task complete only if the page itself accepted the submission", to stop the arm being
-# won by relabelling an honest stop as a completion. It is not needed for that: the ungated
-# how-to-work bullet already requires "every required field holds its intended value" before
-# status=completed, in BOTH arms, and a blank required field fails it. What the clause did add was a
-# CARVE-OUT -- completed is reachable if a submission was accepted -- which on a goal that never
-# asked for a submission reads as the only route to success running through one. That competes with
-# "Do not submit forms or take irreversible actions unless the goal explicitly instructs it" while
-# that prose rule is the only guard on the submit path, which the charter forbids. Anti-relabelling
-# belongs in a finish-gate veto applying to both arms, not in one arm's prompt.
-UNANSWERABLE_FIELD_REMEDY_CONTROL = "stop and report it rather than guessing"
-UNANSWERABLE_FIELD_REMEDY_TREATMENT = (
- "leave that field blank and do not fill it on a later pass, complete every other field you have a basis to "
- "fill, and name it in your finish reason"
-)
+# The rule for a required sensitive field the task's data cannot fill; the required-field-answers prompt must keep it.
+SENSITIVE_FIELD_STOP_CLAUSE = "stop and report it rather than guessing"
-# The anchor ends before UNANSWERABLE_FIELD_REMEDY_CONTROL so the required-field-answers prompt keeps that stop.
+# The anchor ends before SENSITIVE_FIELD_STOP_CLAUSE so the required-field-answers prompt keeps that stop.
REQUIRED_FIELD_ANSWERS_ANCHOR = (
"prefer the provided values, and for an ordinary required field with no exact value, enter the most reasonable "
"value you can. Do not invent sensitive or identifying values (government IDs, financial details, or "
@@ -191,47 +165,28 @@ Rules:
- Do not submit forms or take irreversible actions unless the goal explicitly instructs it."""
-def _with_unanswerable_field_remedy(prompt: str) -> str:
- """Derived once at import, so a prompt edit cannot make the two arms diverge mid-ramp.
-
- A clause that is no longer uniquely present means the prompt was edited without this arm and the
- swap would be a silent no-op; the treatment prompt then IS the prompt it was derived from, which
- the call site detects by identity and logs. The arm row is written before that check, so a run in
- this state still logs `arm=treatment`: the error line carries `workflow_run_id`, and the analyst
- joins on it to drop those runs.
- """
- if prompt.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) != 1:
- return prompt
- return prompt.replace(UNANSWERABLE_FIELD_REMEDY_CONTROL, UNANSWERABLE_FIELD_REMEDY_TREATMENT)
-
-
@functools.lru_cache(maxsize=4)
def _build_required_field_answers_prompt(fill_text: str, self_screen_bullet: str) -> str:
- """Falls back to `SYSTEM_PROMPT` itself unless every anchor is uniquely present and the remedy clause survives."""
+ """Falls back to `SYSTEM_PROMPT` itself unless every anchor is uniquely present and the stop clause survives."""
if SYSTEM_PROMPT.count(REQUIRED_FIELD_ANSWERS_ANCHOR) != 1 or SYSTEM_PROMPT.count(SELF_SCREEN_ANCHOR) != 1:
return SYSTEM_PROMPT
prompt = SYSTEM_PROMPT.replace(REQUIRED_FIELD_ANSWERS_ANCHOR, fill_text).replace(
SELF_SCREEN_ANCHOR, self_screen_bullet + SELF_SCREEN_ANCHOR
)
- if prompt.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) != 1:
+ if prompt.count(SENSITIVE_FIELD_STOP_CLAUSE) != 1:
return SYSTEM_PROMPT
return prompt
-UNANSWERABLE_FIELD_REMEDY_PROMPT = _with_unanswerable_field_remedy(SYSTEM_PROMPT)
-
-
def system_prompt_for_run_arms(
- *, required_field_answers_text: tuple[str, str] | None, unanswerable_field_remedy: bool, customer_precedence: bool
+ *, required_field_answers_text: tuple[str, str] | None, customer_precedence: bool
) -> str:
- """The v3 system prompt for this run's required-field-answers, remedy and customer-precedence arms.
+ """The v3 system prompt for this run's required-field-answers and customer-precedence arms.
`required_field_answers_text` is (fill text, self-screen bullet) for a run in that arm's treatment, else None.
With every arm off this is `SYSTEM_PROMPT` itself, not a copy, so the off arms cannot drift from today's prompt.
"""
- prompt = _system_prompt_for_fill_arms(
- required_field_answers_text=required_field_answers_text, unanswerable_field_remedy=unanswerable_field_remedy
- )
+ prompt = _system_prompt_for_fill_arms(required_field_answers_text=required_field_answers_text)
if not customer_precedence:
return prompt
if prompt.count(CUSTOMER_PRECEDENCE_ANCHOR) != 1:
@@ -240,21 +195,13 @@ def system_prompt_for_run_arms(
return prompt.replace(CUSTOMER_PRECEDENCE_ANCHOR, CUSTOMER_PRECEDENCE_TEXT + CUSTOMER_PRECEDENCE_ANCHOR)
-def _system_prompt_for_fill_arms(
- *, required_field_answers_text: tuple[str, str] | None, unanswerable_field_remedy: bool
-) -> str:
- if required_field_answers_text is not None:
- # The remedy is not applied here: its leave-blank clause, once page validation rejected the blank
- # required legal-status field, led the model to fill in an answer the data never gave.
- prompt = _build_required_field_answers_prompt(*required_field_answers_text)
- if prompt is SYSTEM_PROMPT:
- LOG.error("Task V3 required-field-answers clause is not uniquely present; sent control")
- return prompt
- if not unanswerable_field_remedy:
+def _system_prompt_for_fill_arms(*, required_field_answers_text: tuple[str, str] | None) -> str:
+ if required_field_answers_text is None:
return SYSTEM_PROMPT
- if UNANSWERABLE_FIELD_REMEDY_PROMPT is SYSTEM_PROMPT:
- LOG.error("Task V3 unanswerable-field remedy clause is not uniquely present; sent control")
- return UNANSWERABLE_FIELD_REMEDY_PROMPT
+ prompt = _build_required_field_answers_prompt(*required_field_answers_text)
+ if prompt is SYSTEM_PROMPT:
+ LOG.error("Task V3 required-field-answers clause is not uniquely present; sent control")
+ return prompt
OPAQUE_URL_GUIDANCE = """
@@ -585,14 +532,11 @@ async def run_task_v3_agent_loop(
# The COMPLETE dispatch list, not just the browser tools: auth / captcha / code tools and finish
# are appended here and would otherwise be able to inspect and act on a blank page.
apply_blank_page_guard(tools, blank_page_guard)
- # A page-free run has no page and no fields, so its prompt carries no remedy clause to swap.
+ # A page-free run has no page and no fields, so no prompt arm applies to it.
if page_free:
base_system_prompt = PAGE_FREE_SYSTEM_PROMPT
else:
required_field_answers = run_arm_enabled(REQUIRED_FIELD_ANSWERS_FLAG, settings.TASK_V3_REQUIRED_FIELD_ANSWERS)
- unanswerable_field_remedy = run_arm_enabled(
- UNANSWERABLE_FIELD_REMEDY_FLAG, settings.TASK_V3_UNANSWERABLE_FIELD_REMEDY
- )
required_field_answers_text = (
app.AGENT_FUNCTION.task_v3_required_field_answers_text() if required_field_answers else None
)
@@ -602,15 +546,8 @@ async def run_task_v3_agent_loop(
workflow_run_id=ctx.workflow_run_id if ctx else None,
task_id=ctx.task_id if ctx else None,
)
- if required_field_answers_text is not None and unanswerable_field_remedy:
- LOG.info(
- "Task V3 unanswerable-field remedy suppressed by required-field-answers arm",
- workflow_run_id=ctx.workflow_run_id if ctx else None,
- task_id=ctx.task_id if ctx else None,
- )
base_system_prompt = system_prompt_for_run_arms(
required_field_answers_text=required_field_answers_text,
- unanswerable_field_remedy=unanswerable_field_remedy,
customer_precedence=run_arm_enabled(CUSTOMER_PRECEDENCE_FLAG, settings.TASK_V3_CUSTOMER_PRECEDENCE),
)
# Keyed on which hooks are present, not completion_probe alone: an extraction blocker-only
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 1b3f1cd65..d08ea2d7a 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -2534,12 +2534,8 @@ def make_finish_tool(
perceptions=activity.perceptions,
)
# Deliberately NEUTRAL: it reports the fact and returns the turn, and does not tell the
- # model to act. Two reasons, and either alone is sufficient. (1) Nothing in the block
- # schema establishes authorization to mutate a page, so a directive to act cannot be
- # safely gated -- a read-only task block carries a `navigation_goal` too. (2) Naming the
- # per-field alternative here would deliver the UNANSWERABLE_FIELD_REMEDY arm's treatment
- # clause to runs whose system prompt carries its control clause, which is what made the
- # (remedy=control, hold=on) cell unreadable.
+ # model to act. Nothing in the block schema establishes authorization to mutate a page, so a
+ # directive to act cannot be safely gated -- a read-only task block carries a `navigation_goal` too.
return ToolResult.error(
"verdict held once: this run is ending without having attempted a single action on "
"the page, and budget remains. Look at the page once more and confirm that verdict "
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index 5d0e6a62f..ba7beea67 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -19,7 +19,6 @@ LOG = structlog.get_logger()
OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG = "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED"
TYPE_COORDINATE_CLICK_FLAG = "TASK_V3_TYPE_COORDINATE_CLICK"
DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"
-UNANSWERABLE_FIELD_REMEDY_FLAG = "TASK_V3_UNANSWERABLE_FIELD_REMEDY"
REQUIRED_FIELD_ANSWERS_FLAG = "TASK_V3_REQUIRED_FIELD_ANSWERS"
CUSTOMER_PRECEDENCE_FLAG = "TASK_V3_CUSTOMER_PRECEDENCE"
NO_ACTION_HOLD_FLAG = "TASK_V3_NO_ACTION_HOLD"
diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.py
index 17d96d49a..74fffe21d 100644
--- a/tests/unit/force_stub_app.py
+++ b/tests/unit/force_stub_app.py
@@ -66,9 +66,6 @@ def create_forge_stub_app() -> ForgeApp:
# Class constant, not a method — _LazyNamespace would auto-mock it into a non-iterable AsyncMock
# and break every caller that scans it for close-page phrases.
fake_app_module.AGENT_FUNCTION.MAGIC_LINK_CLOSE_SIGNALS = base_agent_function.MAGIC_LINK_CLOSE_SIGNALS
- # Same footgun: _LazyNamespace's truthy AsyncMock would inject extra guidance text into every
- # v3 task's extra_system_guidance join.
- fake_app_module.AGENT_FUNCTION.resolve_task_v3_extra_guidance = base_agent_function.resolve_task_v3_extra_guidance
fake_app_module.AGENT_FUNCTION.task_v3_required_field_answers_text = (
base_agent_function.task_v3_required_field_answers_text
)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index e73623533..84282b3ab 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -82,7 +82,6 @@ from skyvern.forge.taskv3.run_arms import (
OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,
REQUIRED_FIELD_ANSWERS_FLAG,
TYPE_COORDINATE_CLICK_FLAG,
- UNANSWERABLE_FIELD_REMEDY_FLAG,
run_arm_enabled,
)
from skyvern.forge.taskv3.tools import PageProvider
@@ -181,7 +180,6 @@ async def _run_execute_task_v3(
)
loop_mock.type_coordinate_click_enabled_during_loop = run_arm_enabled(TYPE_COORDINATE_CLICK_FLAG, forced=False)
loop_mock.date_segment_aim_enabled_during_loop = run_arm_enabled(DATE_SEGMENT_AIM_FLAG, forced=False)
- loop_mock.unanswerable_field_remedy_during_loop = run_arm_enabled(UNANSWERABLE_FIELD_REMEDY_FLAG, forced=False)
loop_mock.required_field_answers_during_loop = run_arm_enabled(REQUIRED_FIELD_ANSWERS_FLAG, forced=False)
loop_mock.customer_precedence_during_loop = run_arm_enabled(CUSTOMER_PRECEDENCE_FLAG, forced=False)
loop_mock.no_action_hold_during_loop = run_arm_enabled(NO_ACTION_HOLD_FLAG, forced=False)
@@ -415,35 +413,6 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa
)
-async def test_execute_task_v3_resolves_the_unanswerable_field_remedy_arm_before_the_loop_reads_it(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # Without this the resolve call can be deleted and every run reads control forever -- the arm
- # returns a null for the wrong reason. Bucketed by workflow run, like its siblings.
- monkeypatch.setattr(settings, "TASK_V3_UNANSWERABLE_FIELD_REMEDY", False)
- provider = AsyncMock(return_value="treatment")
- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)
-
- outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])
- _step, task, loop_mock, _post = await _run_execute_task_v3(
- monkeypatch,
- outcome,
- workflow_run_id="wr_unanswerable_field_remedy_reach",
- data_extraction_goal=None,
- extracted_information_schema=None,
- )
-
- assert task.workflow_run_id != task.task_id
- assert loop_mock.unanswerable_field_remedy_during_loop is True
- assert loop_mock.context.run_arms[UNANSWERABLE_FIELD_REMEDY_FLAG] == (task.workflow_run_id, "treatment")
- provider.assert_any_await(
- UNANSWERABLE_FIELD_REMEDY_FLAG,
- task.workflow_run_id,
- properties={"organization_id": task.organization_id},
- )
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize(
("workflow_permanent_id", "targeted_wpid"),
@@ -4034,29 +4003,6 @@ async def test_execute_task_v3_threads_workflow_system_prompt(monkeypatch: pytes
assert "Always use formal salutations." in loop_mock.await_args.kwargs["extra_system_guidance"]
-async def test_goal_judge_gets_the_workflow_system_prompt_and_not_the_ats_guidance(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(
- "skyvern.forge.agent.app.AGENT_FUNCTION.resolve_task_v3_extra_guidance",
- AsyncMock(return_value="If the posting is closed, finish completed."),
- )
- block = _make_block(NavigationBlock, navigation_goal="Open the page")
- _step, _task, loop_mock, _post = await _run_execute_task_v3(
- monkeypatch,
- LoopOutcome(status="completed", reason="done", billable_actions=[]),
- task_block=block,
- workflow_system_prompt="Always use formal salutations.",
- data_extraction_goal=None,
- extracted_information_schema=None,
- )
-
- # The ATS guidance is answer policy, not a definition of done: told that an outcome the
- # instructions allow is no contradiction, the judge would excuse a submit the page shows failed.
- assert loop_mock.await_args.kwargs["goal_instructions"] == "Always use formal salutations."
-
-
@pytest.mark.asyncio
async def test_execute_task_v3_page_free_validation_prompt_has_no_perception_instructions(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 07bf5831e..dd7a4e4a6 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -43,10 +43,8 @@ from skyvern.forge.taskv3.engine import (
OPAQUE_URL_GUIDANCE,
REQUIRED_FIELD_ANSWERS_ANCHOR,
SELF_SCREEN_ANCHOR,
+ SENSITIVE_FIELD_STOP_CLAUSE,
SYSTEM_PROMPT,
- UNANSWERABLE_FIELD_REMEDY_CONTROL,
- UNANSWERABLE_FIELD_REMEDY_PROMPT,
- UNANSWERABLE_FIELD_REMEDY_TREATMENT,
coerce_v3_parameters,
run_task_v3_agent_loop,
system_prompt_for_run_arms,
@@ -67,7 +65,6 @@ from skyvern.forge.taskv3.opaque_refs import OpaqueUrlRefs, mask_opaque_urls
from skyvern.forge.taskv3.run_arms import (
CUSTOMER_PRECEDENCE_FLAG,
REQUIRED_FIELD_ANSWERS_FLAG,
- UNANSWERABLE_FIELD_REMEDY_FLAG,
)
from skyvern.forge.taskv3.tools import PAGE_UNAVAILABLE_ERROR
from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
@@ -2064,12 +2061,10 @@ def stub_required_field_answers_text(monkeypatch: pytest.MonkeyPatch) -> tuple[s
async def _system_prompt_for_run(
- *, arm: str | None, required_field_answers_arm: str | None = None, precedence_arm: str | None = None
+ *, required_field_answers_arm: str | None = None, precedence_arm: str | None = None
) -> str:
"""The system message an actual engine run sends, with the fill-rule and precedence arms pinned."""
context = SkyvernContext()
- if arm is not None:
- context.run_arms = {**context.run_arms, UNANSWERABLE_FIELD_REMEDY_FLAG: ("wr_1", arm)}
if required_field_answers_arm is not None:
context.run_arms = {**context.run_arms, REQUIRED_FIELD_ANSWERS_FLAG: ("wr_1", required_field_answers_arm)}
if precedence_arm is not None:
@@ -2091,59 +2086,14 @@ _DATE_MARKER = "\n\nToday's date is "
@pytest.mark.asyncio
@pytest.mark.parametrize("required_field_answers_arm", [None, "control", "unrandomized"])
[email protected]("arm", [None, "control", "unrandomized"])
-async def test_unanswerable_field_remedy_off_arms_send_todays_prompt_unchanged(
- arm: str | None, required_field_answers_arm: str | None, stub_required_field_answers_text: tuple[str, str]
+async def test_required_field_answers_off_arms_send_todays_prompt_unchanged(
+ required_field_answers_arm: str | None, stub_required_field_answers_text: tuple[str, str]
) -> None:
# The off arms are the deployed prompt, byte for byte: a run outside the experiment must not be
# able to drift because the experiment exists.
- system_prompt = await _system_prompt_for_run(arm=arm, required_field_answers_arm=required_field_answers_arm)
+ system_prompt = await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm)
assert system_prompt.startswith(SYSTEM_PROMPT)
- assert UNANSWERABLE_FIELD_REMEDY_CONTROL in system_prompt
- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT not in system_prompt
- assert system_prompt_for_run_arms(
- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=False
- ) is (SYSTEM_PROMPT)
-
-
-async def test_unanswerable_field_remedy_treatment_swaps_the_remedy_and_nothing_else() -> None:
- control = await _system_prompt_for_run(arm="control")
- treatment = await _system_prompt_for_run(arm="treatment")
-
- assert control.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1
- assert UNANSWERABLE_FIELD_REMEDY_CONTROL not in treatment
- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT in treatment
- # The ONLY difference between the arms is the remedy clause. Anything else the arm changed --
- # including the do-not-invent rule the clause hangs off -- reds here. The date suffix the engine
- # appends is dropped: the two prompts are built by separate calls, so a midnight crossing
- # between them would otherwise red this on wall-clock rather than on a real difference.
- control_body = control.split(_DATE_MARKER)[0]
- treatment_body = treatment.split(_DATE_MARKER)[0]
- assert _DATE_MARKER in control and _DATE_MARKER in treatment
- assert control_body != treatment_body
- assert control_body.replace(UNANSWERABLE_FIELD_REMEDY_CONTROL, UNANSWERABLE_FIELD_REMEDY_TREATMENT) == (
- treatment_body
- )
-
-
-async def test_required_field_answers_treatment_suppresses_the_remedy_swap(
- stub_required_field_answers_text: tuple[str, str],
-) -> None:
- # The remedy's leave-blank clause collided with page validation and the model refilled a legal-status
- # answer; under required-field-answers its own stop clause must hold, so C4 sends C3's prompt.
- with capture_logs() as logs:
- both = await _system_prompt_for_run(arm="treatment", required_field_answers_arm="treatment")
- required_only = await _system_prompt_for_run(arm="control", required_field_answers_arm="treatment")
-
- assert both.split(_DATE_MARKER)[0] == required_only.split(_DATE_MARKER)[0]
- assert STUB_SELF_SCREEN_BULLET in both
- assert both.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1
- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT not in both
- assert [e["event"] for e in logs].count(
- "Task V3 unanswerable-field remedy suppressed by required-field-answers arm"
- ) == 1
+ assert system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=False) is SYSTEM_PROMPT
@pytest.mark.asyncio
@@ -2152,13 +2102,13 @@ async def test_required_field_answers_treatment_is_not_the_silent_fallback(
) -> None:
# The builder falls back to today's prompt when an anchor stops matching; a green suite must not
# hide a treatment arm that is byte-identical to control.
- treatment = await _system_prompt_for_run(arm=None, required_field_answers_arm="treatment")
- control = await _system_prompt_for_run(arm=None, required_field_answers_arm="control")
+ treatment = await _system_prompt_for_run(required_field_answers_arm="treatment")
+ control = await _system_prompt_for_run(required_field_answers_arm="control")
treatment_body = treatment.split(_DATE_MARKER)[0]
assert not treatment.startswith(SYSTEM_PROMPT)
assert STUB_REQUIRED_FIELD_ANSWERS_FILL in treatment_body and STUB_SELF_SCREEN_BULLET in treatment_body
- assert treatment_body.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1
+ assert treatment_body.count(SENSITIVE_FIELD_STOP_CLAUSE) == 1
assert (
treatment_body.replace(STUB_REQUIRED_FIELD_ANSWERS_FILL, REQUIRED_FIELD_ANSWERS_ANCHOR).replace(
STUB_SELF_SCREEN_BULLET, ""
@@ -2168,25 +2118,17 @@ async def test_required_field_answers_treatment_is_not_the_silent_fallback(
@pytest.mark.asyncio
- ("remedy_arm", "expected_prompt"),
- [("control", SYSTEM_PROMPT), ("treatment", UNANSWERABLE_FIELD_REMEDY_PROMPT)],
- ids=["remedy_control", "remedy_treatment"],
-)
-async def test_required_field_answers_treatment_without_supplied_text_renders_control(
- remedy_arm: str, expected_prompt: str
-) -> None:
+async def test_required_field_answers_treatment_without_supplied_text_renders_control() -> None:
# OSS supplies no required-field-answers text, so that arm's treatment must be indistinguishable from
- # its control -- including leaving the remedy arm free to apply -- and say so once for the read to drop.
+ # its control, and say so once for the read to drop.
with capture_logs() as logs:
- system_prompt = await _system_prompt_for_run(arm=remedy_arm, required_field_answers_arm="treatment")
+ system_prompt = await _system_prompt_for_run(required_field_answers_arm="treatment")
- assert system_prompt.split(_DATE_MARKER)[0] == expected_prompt
+ assert system_prompt.split(_DATE_MARKER)[0] == SYSTEM_PROMPT
events = [e["event"] for e in logs]
assert (
events.count("Task V3 required-field-answers arm resolved treatment but no text is supplied; sent control") == 1
)
- assert "Task V3 unanswerable-field remedy suppressed by required-field-answers arm" not in events
@pytest.mark.parametrize(
@@ -2206,7 +2148,6 @@ def test_required_field_answers_falls_back_to_control_when_an_anchor_drifts(
with capture_logs() as logs:
prompt = system_prompt_for_run_arms(
required_field_answers_text=(STUB_REQUIRED_FIELD_ANSWERS_FILL, STUB_SELF_SCREEN_BULLET),
- unanswerable_field_remedy=False,
customer_precedence=False,
)
finally:
@@ -2215,54 +2156,42 @@ def test_required_field_answers_falls_back_to_control_when_an_anchor_drifts(
assert [e["event"] for e in logs] == ["Task V3 required-field-answers clause is not uniquely present; sent control"]
-def test_unanswerable_field_remedy_clause_stays_uniquely_present_and_the_rule_is_not_gated() -> None:
- # If the prompt is edited so the clause no longer matches, the treatment arm silently becomes a
- # no-op and the experiment reads null for the wrong reason. This reds on that edit.
- assert SYSTEM_PROMPT.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1
- # The rule the remedy hangs off is the safety property and is NOT part of the variable.
- rule = "Do not invent sensitive or identifying values (government IDs, financial details, or legal/eligibility attestations)"
- assert rule in SYSTEM_PROMPT
- assert rule in system_prompt_for_run_arms(
- required_field_answers_text=None, unanswerable_field_remedy=True, customer_precedence=False
- )
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("precedence_arm", [None, "treatment"])
@pytest.mark.parametrize("required_field_answers_arm", [None, "treatment"])
-async def test_unanswerable_field_remedy_treatment_adds_no_submit_pressure(
+async def test_prompt_arms_add_no_submit_pressure(
required_field_answers_arm: str | None,
precedence_arm: str | None,
stub_required_field_answers_text: tuple[str, str],
) -> None:
# The charter's non-negotiable: while the only thing standing between a model error and an
# unauthorized submit is a line of system prompt, no arm may add prose that competes with it.
- # An earlier revision ended the treatment with "report the task complete only if the page itself
- # accepted the submission", which on a goal that never asked for a submission reads as the only
- # route to success running through one. Anti-relabelling never needed it: the ungated
- # how-to-work bullet below already requires every required field to hold its value before
- # completed, in BOTH arms. Asserted on the prompt the engine actually sends, not the constant.
- # The supplied required-field-answers wording itself is pinned against this where it lives.
+ # Asserted on the prompt the engine actually sends, not the constant. The supplied
+ # required-field-answers wording itself is pinned against this where it lives.
treatment = await _system_prompt_for_run(
- arm="treatment", required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm
- )
- control = await _system_prompt_for_run(
- arm="control", required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm
+ required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm
)
- base_control = await _system_prompt_for_run(arm="control")
+ base_control = await _system_prompt_for_run()
bullet = next(line for line in treatment.splitlines() if line.startswith("- Fill fields from the task's data"))
assert "submission" not in bullet and "accepted" not in bullet
- # The optional-fields instruction shares the bullet and must stay untouched by the remedy.
assert "Leave optional fields blank" in bullet
- # The anti-relabelling property the removed clause used to carry, in its real home: ungated, and
- # byte-identical across the arms, so neither arm carries a completion rule the other does not.
+ # The completion rule is ungated and byte-identical across the arms.
contract = next(line for line in treatment.splitlines() if "status=completed" in line)
assert "every required field holds its intended value" in contract
- assert contract in control and contract in base_control
- # The no-submit rule is the guard the charter is protecting; it must survive the swap intact.
+ assert contract in base_control
no_submit = "Do not submit forms or take irreversible actions unless the goal explicitly instructs it."
- assert no_submit in treatment and no_submit in control and no_submit in base_control
+ assert no_submit in treatment and no_submit in base_control
+ # Pinned as a literal so an edit weakening SYSTEM_PROMPT cannot pass by weakening the constant too. Only the
+ # required-field-answers arm rewrites this rule; every other arm sends it verbatim.
+ do_not_invent = (
+ "Do not invent sensitive or identifying values (government IDs, financial details, or "
+ "legal/eligibility attestations); if one of those is required and not provided, stop and report it "
+ "rather than guessing."
+ )
+ assert do_not_invent in base_control
+ if required_field_answers_arm is None:
+ assert do_not_invent in treatment
# The precedence paragraph sits beside that guard, so it may name submitting only to exempt that guard.
if precedence_arm == "treatment":
paragraph = treatment.split(CUSTOMER_PRECEDENCE_ANCHOR)[0].split("\n\n")[-1]
@@ -2279,22 +2208,17 @@ def _body(prompt: str) -> str:
@pytest.mark.asyncio
@pytest.mark.parametrize("precedence_arm", [None, "control", "unrandomized"])
async def test_customer_precedence_off_arms_send_todays_prompt(precedence_arm: str | None) -> None:
- system_prompt = await _system_prompt_for_run(arm=None, precedence_arm=precedence_arm)
+ system_prompt = await _system_prompt_for_run(precedence_arm=precedence_arm)
assert system_prompt.startswith(SYSTEM_PROMPT)
assert CUSTOMER_PRECEDENCE_TEXT not in system_prompt
- assert (
- system_prompt_for_run_arms(
- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=False
- )
- is SYSTEM_PROMPT
- )
+ assert system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=False) is SYSTEM_PROMPT
@pytest.mark.asyncio
async def test_customer_precedence_treatment_adds_the_paragraph_before_how_to_work_and_nothing_else() -> None:
- control = _body(await _system_prompt_for_run(arm=None, precedence_arm="control"))
- treatment = _body(await _system_prompt_for_run(arm=None, precedence_arm="treatment"))
+ control = _body(await _system_prompt_for_run(precedence_arm="control"))
+ treatment = _body(await _system_prompt_for_run(precedence_arm="treatment"))
assert SYSTEM_PROMPT.count(CUSTOMER_PRECEDENCE_ANCHOR) == 1
assert control != treatment
@@ -2313,9 +2237,7 @@ def test_customer_precedence_sends_the_prompt_unchanged_when_its_anchor_drifts(
) -> None:
monkeypatch.setattr(engine_mod, "SYSTEM_PROMPT", drifted_prompt)
with capture_logs() as logs:
- prompt = system_prompt_for_run_arms(
- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=True
- )
+ prompt = system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=True)
assert prompt is drifted_prompt
assert [e["event"] for e in logs] == [
"Task V3 customer-precedence anchor is not uniquely present; sent the prompt without it"
@@ -2324,32 +2246,24 @@ def test_customer_precedence_sends_the_prompt_unchanged_when_its_anchor_drifts(
@pytest.mark.asyncio
@pytest.mark.parametrize("required_field_answers_arm", [None, "treatment"])
[email protected]("remedy_arm", [None, "treatment"])
async def test_customer_precedence_composes_with_the_fill_rule_arms(
- remedy_arm: str | None,
required_field_answers_arm: str | None,
stub_required_field_answers_text: tuple[str, str],
) -> None:
without = _body(
- await _system_prompt_for_run(
- arm=remedy_arm, required_field_answers_arm=required_field_answers_arm, precedence_arm="control"
- )
+ await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm, precedence_arm="control")
)
with_precedence = _body(
- await _system_prompt_for_run(
- arm=remedy_arm, required_field_answers_arm=required_field_answers_arm, precedence_arm="treatment"
- )
+ await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm, precedence_arm="treatment")
)
# Above "How to work:" (named literally, not through the anchor constant), outside every span another arm rewrites.
assert with_precedence.count(CUSTOMER_PRECEDENCE_TEXT) == 1
assert with_precedence.split("\n\nHow to work:\n")[0].endswith(CUSTOMER_PRECEDENCE_TEXT)
assert with_precedence.replace(CUSTOMER_PRECEDENCE_TEXT, "", 1) == without
- remedy_applied = remedy_arm == "treatment" and required_field_answers_arm is None
- assert with_precedence.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == (0 if remedy_applied else 1)
- assert with_precedence.count(UNANSWERABLE_FIELD_REMEDY_TREATMENT) == (1 if remedy_applied else 0)
+ assert with_precedence.count(SENSITIVE_FIELD_STOP_CLAUSE) == 1
for other_arms_anchor in (
- UNANSWERABLE_FIELD_REMEDY_CONTROL,
+ SENSITIVE_FIELD_STOP_CLAUSE,
REQUIRED_FIELD_ANSWERS_ANCHOR,
SELF_SCREEN_ANCHOR,
CUSTOMER_PRECEDENCE_ANCHOR,
@@ -2362,7 +2276,7 @@ async def test_customer_precedence_keeps_page_text_out_of_the_users_reach() -> N
# Security-critical wording, so pinned exactly: page text never becomes the user's instruction, and the two
# prose guards stay outside the precedence while they are the only guards. Both halves refer to the rules by
# their own conditions rather than restating them: a paraphrase narrows or widens what the rule covers.
- treatment = _body(await _system_prompt_for_run(arm=None, precedence_arm="treatment"))
+ treatment = _body(await _system_prompt_for_run(precedence_arm="treatment"))
assert "where they conflict with a general rule in this prompt, follow the user" in treatment
assert (