Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Remove three dead Task V3 experiment arms: error-code choice read, unanswerable-field remedy, ATS defaults (SKY-17012) (#8655)
pedrohsdba8dd93c2b162
diff --git a/skyvern/config.py b/skyvern/config.pyindex a8954fc98..cb7f14652 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -559,17 +559,9 @@ class Settings(BaseSettings):# Force-on term only: the arm is randomized per run by the flag of the same name and read through# run_arm_enabled() in skyvern/forge/taskv3/run_arms.py (SKY-16501).TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED: bool = False- # What the system prompt tells the model to do about ONE required SENSITIVE field (government- # ID, financial detail, legal/eligibility attestation) the payload does not carry: stop the run- # (off) or leave that field and finish the page (on). An ordinary required field is out of- # scope -- the ungated sentence above the clause already tells the model to enter the most- # reasonable value. This arm does not gate the do-not-invent rule, which only- # TASK_V3_REQUIRED_FIELD_ANSWERS rewrites (SKY-16651). Force-on term only:- # runs are randomized per run by the flag of the same name, read through run_arm_enabled().- TASK_V3_UNANSWERABLE_FIELD_REMEDY: bool = False# Swap in the fill-rule text an AgentFunction supplies (task_v3_required_field_answers_text); with no- # text supplied the run gets the control prompt. Force-on term only: runs are randomized per run by the flag of the same name, read through- # run_arm_enabled().+ # text supplied the run gets the control prompt. Force-on term only: runs are randomized per run by the+ # flag of the same name, read through run_arm_enabled().TASK_V3_REQUIRED_FIELD_ANSWERS: bool = False# State in the system prompt that the task's own instructions win over its general rules. Force-on term only:# runs are randomized per run by the flag of the same name, read through run_arm_enabled().diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 290c1dc88..731491bb9 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -201,7 +201,6 @@ from skyvern.forge.taskv3.run_arms import (OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,REQUIRED_FIELD_ANSWERS_FLAG,TYPE_COORDINATE_CLICK_FLAG,- UNANSWERABLE_FIELD_REMEDY_FLAG,resolve_run_arm,run_arm_enabled,)@@ -2266,13 +2265,6 @@ class ForgeAgent:organization_id=task.organization_id,forced=settings.TASK_V3_DATE_SEGMENT_AIM,)- await resolve_run_arm(- context,- UNANSWERABLE_FIELD_REMEDY_FLAG,- distinct_id=task.workflow_run_id or task.task_id,- organization_id=task.organization_id,- forced=settings.TASK_V3_UNANSWERABLE_FIELD_REMEDY,- )await resolve_run_arm(context,REQUIRED_FIELD_ANSWERS_FLAG,@@ -2936,19 +2928,6 @@ class ForgeAgent:captcha_tools, captcha_guidance = build_captcha_tools(task, _page_provider, organization_id=organization.organization_id)- # Withheld in page-free mode: form-filling defaults must not bias a data-only judgment.- ats_guidance = None- if not page_free_validation:- try:- ats_guidance = await app.AGENT_FUNCTION.resolve_task_v3_extra_guidance(- task=task, organization=organization- )- except Exception:- LOG.warning(- "resolve_task_v3_extra_guidance failed; continuing without it",- task_id=task.task_id,- exc_info=True,- )# Page-free runs never get the precedence paragraph, so the label would have nothing to refer to.workflow_system_guidance = task.workflow_system_promptif (@@ -3014,9 +2993,9 @@ class ForgeAgent:goal_judge=goal_judge,goal_check_enforce=goal_check_enforce,extraction_requested=extraction_requested,- # Only the customer's own instructions can define what counts as done. Auth, captcha and- # ATS guidance are how-to and answer policy, and the judge is told an outcome its- # instructions allow is no contradiction.+ # Only the customer's own instructions can define what counts as done. Auth and captcha+ # guidance are how-to, and the judge is told an outcome its instructions allow is no+ # contradiction.goal_instructions=task.workflow_system_prompt or "",secret_on_page_at_start=_task_v3_secret_may_be_on_page(task,@@ -3059,12 +3038,8 @@ class ForgeAgent:on_action_round=_on_action_round,on_pre_action=pre_submit_ring.capture if pre_submit_ring is not None else None,extra_tools=auth_tools + captcha_tools,- # ats_guidance before workflow_system_prompt keeps the customer's own text later in- # the message; position in one system message is a weak signal, not precedence — the- # real contract is the guidance's own scoping prose (its NEVER list and the explicit- # link to the base stop-don't-guess rule).extra_system_guidance="\n\n".join(- part for part in (auth_guidance, captcha_guidance, ats_guidance, workflow_system_guidance) if part+ part for part in (auth_guidance, captcha_guidance, workflow_system_guidance) if part),completion_probe=completion_probe,completion_blocker=completion_blocker,diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex d03c156d0..e01b34896 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -974,20 +974,14 @@ class AgentFunction:async def resolve_billing_tier(self, organization_id: str | None) -> BillingTier:return BillingTier.UNKNOWN- # OSS has no ATS-scoped guidance; cloud overrides to supply pre-authorized eligibility defaults- # behind a flag when the task targets a gated application-tracking-system host.- async def resolve_task_v3_extra_guidance(self, *, task: Task, organization: Organization) -> str | None:- return None-# (fill text, self-screen bullet) for the v3 required-field-answers arm's treatment prompt. OSS# supplies none, so a run in that arm's treatment renders the control prompt.def task_v3_required_field_answers_text(self) -> tuple[str, str] | None:return None# Whether v3 offers the task's configured error codes to the model, so a terminal verdict names- # its own business outcome instead of having one matched on afterwards (SKY-15586). Cloud- # overrides behind a flag; False keeps codes out of the loop entirely, which makes the whole- # feature byte-identical to before it existed.+ # its own business outcome instead of having one matched on afterwards (SKY-15586). No deployment+ # overrides it; False keeps codes out of the loop entirely.async def resolve_task_v3_error_code_choice(self, *, task: Task, organization: Organization) -> bool:return Falsediff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex c417ab51e..e9d62ac0e 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -68,7 +68,6 @@ from skyvern.forge.taskv3.run_arms import (CUSTOMER_PRECEDENCE_FLAG,NO_ACTION_HOLD_FLAG,REQUIRED_FIELD_ANSWERS_FLAG,- UNANSWERABLE_FIELD_REMEDY_FLAG,run_arm_enabled,)from skyvern.forge.taskv3.tools import (@@ -120,35 +119,10 @@ GOAL_CHECK_DEADLINE_MARGIN_SECONDS = 2.0# allowlist scoping the population, not authorizing it: bare tasks (block_type=None) fail closed.NO_ACTION_HOLD_BLOCK_TYPES = frozenset({BlockType.TASK, BlockType.NAVIGATION})-# SKY-16651: what the prompt tells the model to do about ONE required SENSITIVE field the payload-# cannot fill -- the clause's antecedent is "one of those", i.e. the do-not-invent categories, NOT-# any required field (an ordinary one is handled by the ungated sentence before it). The control-# remedy stops the whole run, so a single unanswerable field abandons every other field on the-# page. This arm gates only the remedy; the do-not-invent rule it hangs off is rewritten only in the-# required-field-answers treatment cells.-# "every other field you have a basis to fill", not "every other field on the page": the clause sits-# INSIDE the do-not-invent sentence and one sentence from "Leave optional fields blank when you have-# no basis to fill them". An unqualified imperative there reads, on a page with a SECOND unanswerable-# sensitive field, as instructing the model to fill that one -- the exact act the sentence it lives-# in forbids. "a basis to fill" is the prompt's own existing phrase for the same test.-#-# The treatment carries NO completion constraint, deliberately. An earlier revision ended with-# "report the task complete only if the page itself accepted the submission", to stop the arm being-# won by relabelling an honest stop as a completion. It is not needed for that: the ungated-# how-to-work bullet already requires "every required field holds its intended value" before-# status=completed, in BOTH arms, and a blank required field fails it. What the clause did add was a-# CARVE-OUT -- completed is reachable if a submission was accepted -- which on a goal that never-# asked for a submission reads as the only route to success running through one. That competes with-# "Do not submit forms or take irreversible actions unless the goal explicitly instructs it" while-# that prose rule is the only guard on the submit path, which the charter forbids. Anti-relabelling-# belongs in a finish-gate veto applying to both arms, not in one arm's prompt.-UNANSWERABLE_FIELD_REMEDY_CONTROL = "stop and report it rather than guessing"-UNANSWERABLE_FIELD_REMEDY_TREATMENT = (- "leave that field blank and do not fill it on a later pass, complete every other field you have a basis to "- "fill, and name it in your finish reason"-)+# The rule for a required sensitive field the task's data cannot fill; the required-field-answers prompt must keep it.+SENSITIVE_FIELD_STOP_CLAUSE = "stop and report it rather than guessing"-# The anchor ends before UNANSWERABLE_FIELD_REMEDY_CONTROL so the required-field-answers prompt keeps that stop.+# The anchor ends before SENSITIVE_FIELD_STOP_CLAUSE so the required-field-answers prompt keeps that stop.REQUIRED_FIELD_ANSWERS_ANCHOR = ("prefer the provided values, and for an ordinary required field with no exact value, enter the most reasonable ""value you can. Do not invent sensitive or identifying values (government IDs, financial details, or "@@ -191,47 +165,28 @@ Rules:- Do not submit forms or take irreversible actions unless the goal explicitly instructs it."""-def _with_unanswerable_field_remedy(prompt: str) -> str:- """Derived once at import, so a prompt edit cannot make the two arms diverge mid-ramp.-- A clause that is no longer uniquely present means the prompt was edited without this arm and the- swap would be a silent no-op; the treatment prompt then IS the prompt it was derived from, which- the call site detects by identity and logs. The arm row is written before that check, so a run in- this state still logs `arm=treatment`: the error line carries `workflow_run_id`, and the analyst- joins on it to drop those runs.- """- if prompt.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) != 1:- return prompt- return prompt.replace(UNANSWERABLE_FIELD_REMEDY_CONTROL, UNANSWERABLE_FIELD_REMEDY_TREATMENT)--@functools.lru_cache(maxsize=4)def _build_required_field_answers_prompt(fill_text: str, self_screen_bullet: str) -> str:- """Falls back to `SYSTEM_PROMPT` itself unless every anchor is uniquely present and the remedy clause survives."""+ """Falls back to `SYSTEM_PROMPT` itself unless every anchor is uniquely present and the stop clause survives."""if SYSTEM_PROMPT.count(REQUIRED_FIELD_ANSWERS_ANCHOR) != 1 or SYSTEM_PROMPT.count(SELF_SCREEN_ANCHOR) != 1:return SYSTEM_PROMPTprompt = SYSTEM_PROMPT.replace(REQUIRED_FIELD_ANSWERS_ANCHOR, fill_text).replace(SELF_SCREEN_ANCHOR, self_screen_bullet + SELF_SCREEN_ANCHOR)- if prompt.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) != 1:+ if prompt.count(SENSITIVE_FIELD_STOP_CLAUSE) != 1:return SYSTEM_PROMPTreturn prompt-UNANSWERABLE_FIELD_REMEDY_PROMPT = _with_unanswerable_field_remedy(SYSTEM_PROMPT)--def system_prompt_for_run_arms(- *, required_field_answers_text: tuple[str, str] | None, unanswerable_field_remedy: bool, customer_precedence: bool+ *, required_field_answers_text: tuple[str, str] | None, customer_precedence: bool) -> str:- """The v3 system prompt for this run's required-field-answers, remedy and customer-precedence arms.+ """The v3 system prompt for this run's required-field-answers and customer-precedence arms.`required_field_answers_text` is (fill text, self-screen bullet) for a run in that arm's treatment, else None.With every arm off this is `SYSTEM_PROMPT` itself, not a copy, so the off arms cannot drift from today's prompt."""- prompt = _system_prompt_for_fill_arms(- required_field_answers_text=required_field_answers_text, unanswerable_field_remedy=unanswerable_field_remedy- )+ prompt = _system_prompt_for_fill_arms(required_field_answers_text=required_field_answers_text)if not customer_precedence:return promptif prompt.count(CUSTOMER_PRECEDENCE_ANCHOR) != 1:@@ -240,21 +195,13 @@ def system_prompt_for_run_arms(return prompt.replace(CUSTOMER_PRECEDENCE_ANCHOR, CUSTOMER_PRECEDENCE_TEXT + CUSTOMER_PRECEDENCE_ANCHOR)-def _system_prompt_for_fill_arms(- *, required_field_answers_text: tuple[str, str] | None, unanswerable_field_remedy: bool-) -> str:- if required_field_answers_text is not None:- # The remedy is not applied here: its leave-blank clause, once page validation rejected the blank- # required legal-status field, led the model to fill in an answer the data never gave.- prompt = _build_required_field_answers_prompt(*required_field_answers_text)- if prompt is SYSTEM_PROMPT:- LOG.error("Task V3 required-field-answers clause is not uniquely present; sent control")- return prompt- if not unanswerable_field_remedy:+def _system_prompt_for_fill_arms(*, required_field_answers_text: tuple[str, str] | None) -> str:+ if required_field_answers_text is None:return SYSTEM_PROMPT- if UNANSWERABLE_FIELD_REMEDY_PROMPT is SYSTEM_PROMPT:- LOG.error("Task V3 unanswerable-field remedy clause is not uniquely present; sent control")- return UNANSWERABLE_FIELD_REMEDY_PROMPT+ prompt = _build_required_field_answers_prompt(*required_field_answers_text)+ if prompt is SYSTEM_PROMPT:+ LOG.error("Task V3 required-field-answers clause is not uniquely present; sent control")+ return promptOPAQUE_URL_GUIDANCE = """@@ -585,14 +532,11 @@ async def run_task_v3_agent_loop(# The COMPLETE dispatch list, not just the browser tools: auth / captcha / code tools and finish# are appended here and would otherwise be able to inspect and act on a blank page.apply_blank_page_guard(tools, blank_page_guard)- # A page-free run has no page and no fields, so its prompt carries no remedy clause to swap.+ # A page-free run has no page and no fields, so no prompt arm applies to it.if page_free:base_system_prompt = PAGE_FREE_SYSTEM_PROMPTelse:required_field_answers = run_arm_enabled(REQUIRED_FIELD_ANSWERS_FLAG, settings.TASK_V3_REQUIRED_FIELD_ANSWERS)- unanswerable_field_remedy = run_arm_enabled(- UNANSWERABLE_FIELD_REMEDY_FLAG, settings.TASK_V3_UNANSWERABLE_FIELD_REMEDY- )required_field_answers_text = (app.AGENT_FUNCTION.task_v3_required_field_answers_text() if required_field_answers else None)@@ -602,15 +546,8 @@ async def run_task_v3_agent_loop(workflow_run_id=ctx.workflow_run_id if ctx else None,task_id=ctx.task_id if ctx else None,)- if required_field_answers_text is not None and unanswerable_field_remedy:- LOG.info(- "Task V3 unanswerable-field remedy suppressed by required-field-answers arm",- workflow_run_id=ctx.workflow_run_id if ctx else None,- task_id=ctx.task_id if ctx else None,- )base_system_prompt = system_prompt_for_run_arms(required_field_answers_text=required_field_answers_text,- unanswerable_field_remedy=unanswerable_field_remedy,customer_precedence=run_arm_enabled(CUSTOMER_PRECEDENCE_FLAG, settings.TASK_V3_CUSTOMER_PRECEDENCE),)# Keyed on which hooks are present, not completion_probe alone: an extraction blocker-onlydiff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 1b3f1cd65..d08ea2d7a 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -2534,12 +2534,8 @@ def make_finish_tool(perceptions=activity.perceptions,)# Deliberately NEUTRAL: it reports the fact and returns the turn, and does not tell the- # model to act. Two reasons, and either alone is sufficient. (1) Nothing in the block- # schema establishes authorization to mutate a page, so a directive to act cannot be- # safely gated -- a read-only task block carries a `navigation_goal` too. (2) Naming the- # per-field alternative here would deliver the UNANSWERABLE_FIELD_REMEDY arm's treatment- # clause to runs whose system prompt carries its control clause, which is what made the- # (remedy=control, hold=on) cell unreadable.+ # model to act. Nothing in the block schema establishes authorization to mutate a page, so a+ # directive to act cannot be safely gated -- a read-only task block carries a `navigation_goal` too.return ToolResult.error("verdict held once: this run is ending without having attempted a single action on ""the page, and budget remains. Look at the page once more and confirm that verdict "diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.pyindex 5d0e6a62f..ba7beea67 100644--- a/skyvern/forge/taskv3/run_arms.py+++ b/skyvern/forge/taskv3/run_arms.py@@ -19,7 +19,6 @@ LOG = structlog.get_logger()OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG = "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED"TYPE_COORDINATE_CLICK_FLAG = "TASK_V3_TYPE_COORDINATE_CLICK"DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"-UNANSWERABLE_FIELD_REMEDY_FLAG = "TASK_V3_UNANSWERABLE_FIELD_REMEDY"REQUIRED_FIELD_ANSWERS_FLAG = "TASK_V3_REQUIRED_FIELD_ANSWERS"CUSTOMER_PRECEDENCE_FLAG = "TASK_V3_CUSTOMER_PRECEDENCE"NO_ACTION_HOLD_FLAG = "TASK_V3_NO_ACTION_HOLD"diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.pyindex 17d96d49a..74fffe21d 100644--- a/tests/unit/force_stub_app.py+++ b/tests/unit/force_stub_app.py@@ -66,9 +66,6 @@ def create_forge_stub_app() -> ForgeApp:# Class constant, not a method — _LazyNamespace would auto-mock it into a non-iterable AsyncMock# and break every caller that scans it for close-page phrases.fake_app_module.AGENT_FUNCTION.MAGIC_LINK_CLOSE_SIGNALS = base_agent_function.MAGIC_LINK_CLOSE_SIGNALS- # Same footgun: _LazyNamespace's truthy AsyncMock would inject extra guidance text into every- # v3 task's extra_system_guidance join.- fake_app_module.AGENT_FUNCTION.resolve_task_v3_extra_guidance = base_agent_function.resolve_task_v3_extra_guidancefake_app_module.AGENT_FUNCTION.task_v3_required_field_answers_text = (base_agent_function.task_v3_required_field_answers_text)diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex e73623533..84282b3ab 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -82,7 +82,6 @@ from skyvern.forge.taskv3.run_arms import (OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,REQUIRED_FIELD_ANSWERS_FLAG,TYPE_COORDINATE_CLICK_FLAG,- UNANSWERABLE_FIELD_REMEDY_FLAG,run_arm_enabled,)from skyvern.forge.taskv3.tools import PageProvider@@ -181,7 +180,6 @@ async def _run_execute_task_v3()loop_mock.type_coordinate_click_enabled_during_loop = run_arm_enabled(TYPE_COORDINATE_CLICK_FLAG, forced=False)loop_mock.date_segment_aim_enabled_during_loop = run_arm_enabled(DATE_SEGMENT_AIM_FLAG, forced=False)- loop_mock.unanswerable_field_remedy_during_loop = run_arm_enabled(UNANSWERABLE_FIELD_REMEDY_FLAG, forced=False)loop_mock.required_field_answers_during_loop = run_arm_enabled(REQUIRED_FIELD_ANSWERS_FLAG, forced=False)loop_mock.customer_precedence_during_loop = run_arm_enabled(CUSTOMER_PRECEDENCE_FLAG, forced=False)loop_mock.no_action_hold_during_loop = run_arm_enabled(NO_ACTION_HOLD_FLAG, forced=False)@@ -415,35 +413,6 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa)-async def test_execute_task_v3_resolves_the_unanswerable_field_remedy_arm_before_the_loop_reads_it(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # Without this the resolve call can be deleted and every run reads control forever -- the arm- # returns a null for the wrong reason. Bucketed by workflow run, like its siblings.- monkeypatch.setattr(settings, "TASK_V3_UNANSWERABLE_FIELD_REMEDY", False)- provider = AsyncMock(return_value="treatment")- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)-- outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])- _step, task, loop_mock, _post = await _run_execute_task_v3(- monkeypatch,- outcome,- workflow_run_id="wr_unanswerable_field_remedy_reach",- data_extraction_goal=None,- extracted_information_schema=None,- )-- assert task.workflow_run_id != task.task_id- assert loop_mock.unanswerable_field_remedy_during_loop is True- assert loop_mock.context.run_arms[UNANSWERABLE_FIELD_REMEDY_FLAG] == (task.workflow_run_id, "treatment")- provider.assert_any_await(- UNANSWERABLE_FIELD_REMEDY_FLAG,- task.workflow_run_id,- properties={"organization_id": task.organization_id},- )--@pytest.mark.asyncio@pytest.mark.parametrize(("workflow_permanent_id", "targeted_wpid"),@@ -4034,29 +4003,6 @@ async def test_execute_task_v3_threads_workflow_system_prompt(monkeypatch: pytesassert "Always use formal salutations." in loop_mock.await_args.kwargs["extra_system_guidance"]-async def test_goal_judge_gets_the_workflow_system_prompt_and_not_the_ats_guidance(- monkeypatch: pytest.MonkeyPatch,-) -> None:- monkeypatch.setattr(- "skyvern.forge.agent.app.AGENT_FUNCTION.resolve_task_v3_extra_guidance",- AsyncMock(return_value="If the posting is closed, finish completed."),- )- block = _make_block(NavigationBlock, navigation_goal="Open the page")- _step, _task, loop_mock, _post = await _run_execute_task_v3(- monkeypatch,- LoopOutcome(status="completed", reason="done", billable_actions=[]),- task_block=block,- workflow_system_prompt="Always use formal salutations.",- data_extraction_goal=None,- extracted_information_schema=None,- )-- # The ATS guidance is answer policy, not a definition of done: told that an outcome the- # instructions allow is no contradiction, the judge would excuse a submit the page shows failed.- assert loop_mock.await_args.kwargs["goal_instructions"] == "Always use formal salutations."--@pytest.mark.asyncioasync def test_execute_task_v3_page_free_validation_prompt_has_no_perception_instructions(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 07bf5831e..dd7a4e4a6 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -43,10 +43,8 @@ from skyvern.forge.taskv3.engine import (OPAQUE_URL_GUIDANCE,REQUIRED_FIELD_ANSWERS_ANCHOR,SELF_SCREEN_ANCHOR,+ SENSITIVE_FIELD_STOP_CLAUSE,SYSTEM_PROMPT,- UNANSWERABLE_FIELD_REMEDY_CONTROL,- UNANSWERABLE_FIELD_REMEDY_PROMPT,- UNANSWERABLE_FIELD_REMEDY_TREATMENT,coerce_v3_parameters,run_task_v3_agent_loop,system_prompt_for_run_arms,@@ -67,7 +65,6 @@ from skyvern.forge.taskv3.opaque_refs import OpaqueUrlRefs, mask_opaque_urlsfrom skyvern.forge.taskv3.run_arms import (CUSTOMER_PRECEDENCE_FLAG,REQUIRED_FIELD_ANSWERS_FLAG,- UNANSWERABLE_FIELD_REMEDY_FLAG,)from skyvern.forge.taskv3.tools import PAGE_UNAVAILABLE_ERRORfrom skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig@@ -2064,12 +2061,10 @@ def stub_required_field_answers_text(monkeypatch: pytest.MonkeyPatch) -> tuple[sasync def _system_prompt_for_run(- *, arm: str | None, required_field_answers_arm: str | None = None, precedence_arm: str | None = None+ *, required_field_answers_arm: str | None = None, precedence_arm: str | None = None) -> str:"""The system message an actual engine run sends, with the fill-rule and precedence arms pinned."""context = SkyvernContext()- if arm is not None:- context.run_arms = {**context.run_arms, UNANSWERABLE_FIELD_REMEDY_FLAG: ("wr_1", arm)}if required_field_answers_arm is not None:context.run_arms = {**context.run_arms, REQUIRED_FIELD_ANSWERS_FLAG: ("wr_1", required_field_answers_arm)}if precedence_arm is not None:@@ -2091,59 +2086,14 @@ _DATE_MARKER = "\n\nToday's date is "@pytest.mark.asyncio@pytest.mark.parametrize("required_field_answers_arm", [None, "control", "unrandomized"])[email protected]("arm", [None, "control", "unrandomized"])-async def test_unanswerable_field_remedy_off_arms_send_todays_prompt_unchanged(- arm: str | None, required_field_answers_arm: str | None, stub_required_field_answers_text: tuple[str, str]+async def test_required_field_answers_off_arms_send_todays_prompt_unchanged(+ required_field_answers_arm: str | None, stub_required_field_answers_text: tuple[str, str]) -> None:# The off arms are the deployed prompt, byte for byte: a run outside the experiment must not be# able to drift because the experiment exists.- system_prompt = await _system_prompt_for_run(arm=arm, required_field_answers_arm=required_field_answers_arm)+ system_prompt = await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm)assert system_prompt.startswith(SYSTEM_PROMPT)- assert UNANSWERABLE_FIELD_REMEDY_CONTROL in system_prompt- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT not in system_prompt- assert system_prompt_for_run_arms(- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=False- ) is (SYSTEM_PROMPT)---async def test_unanswerable_field_remedy_treatment_swaps_the_remedy_and_nothing_else() -> None:- control = await _system_prompt_for_run(arm="control")- treatment = await _system_prompt_for_run(arm="treatment")-- assert control.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1- assert UNANSWERABLE_FIELD_REMEDY_CONTROL not in treatment- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT in treatment- # The ONLY difference between the arms is the remedy clause. Anything else the arm changed --- # including the do-not-invent rule the clause hangs off -- reds here. The date suffix the engine- # appends is dropped: the two prompts are built by separate calls, so a midnight crossing- # between them would otherwise red this on wall-clock rather than on a real difference.- control_body = control.split(_DATE_MARKER)[0]- treatment_body = treatment.split(_DATE_MARKER)[0]- assert _DATE_MARKER in control and _DATE_MARKER in treatment- assert control_body != treatment_body- assert control_body.replace(UNANSWERABLE_FIELD_REMEDY_CONTROL, UNANSWERABLE_FIELD_REMEDY_TREATMENT) == (- treatment_body- )---async def test_required_field_answers_treatment_suppresses_the_remedy_swap(- stub_required_field_answers_text: tuple[str, str],-) -> None:- # The remedy's leave-blank clause collided with page validation and the model refilled a legal-status- # answer; under required-field-answers its own stop clause must hold, so C4 sends C3's prompt.- with capture_logs() as logs:- both = await _system_prompt_for_run(arm="treatment", required_field_answers_arm="treatment")- required_only = await _system_prompt_for_run(arm="control", required_field_answers_arm="treatment")-- assert both.split(_DATE_MARKER)[0] == required_only.split(_DATE_MARKER)[0]- assert STUB_SELF_SCREEN_BULLET in both- assert both.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1- assert UNANSWERABLE_FIELD_REMEDY_TREATMENT not in both- assert [e["event"] for e in logs].count(- "Task V3 unanswerable-field remedy suppressed by required-field-answers arm"- ) == 1+ assert system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=False) is SYSTEM_PROMPT@pytest.mark.asyncio@@ -2152,13 +2102,13 @@ async def test_required_field_answers_treatment_is_not_the_silent_fallback() -> None:# The builder falls back to today's prompt when an anchor stops matching; a green suite must not# hide a treatment arm that is byte-identical to control.- treatment = await _system_prompt_for_run(arm=None, required_field_answers_arm="treatment")- control = await _system_prompt_for_run(arm=None, required_field_answers_arm="control")+ treatment = await _system_prompt_for_run(required_field_answers_arm="treatment")+ control = await _system_prompt_for_run(required_field_answers_arm="control")treatment_body = treatment.split(_DATE_MARKER)[0]assert not treatment.startswith(SYSTEM_PROMPT)assert STUB_REQUIRED_FIELD_ANSWERS_FILL in treatment_body and STUB_SELF_SCREEN_BULLET in treatment_body- assert treatment_body.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1+ assert treatment_body.count(SENSITIVE_FIELD_STOP_CLAUSE) == 1assert (treatment_body.replace(STUB_REQUIRED_FIELD_ANSWERS_FILL, REQUIRED_FIELD_ANSWERS_ANCHOR).replace(STUB_SELF_SCREEN_BULLET, ""@@ -2168,25 +2118,17 @@ async def test_required_field_answers_treatment_is_not_the_silent_fallback(@pytest.mark.asyncio- ("remedy_arm", "expected_prompt"),- [("control", SYSTEM_PROMPT), ("treatment", UNANSWERABLE_FIELD_REMEDY_PROMPT)],- ids=["remedy_control", "remedy_treatment"],-)-async def test_required_field_answers_treatment_without_supplied_text_renders_control(- remedy_arm: str, expected_prompt: str-) -> None:+async def test_required_field_answers_treatment_without_supplied_text_renders_control() -> None:# OSS supplies no required-field-answers text, so that arm's treatment must be indistinguishable from- # its control -- including leaving the remedy arm free to apply -- and say so once for the read to drop.+ # its control, and say so once for the read to drop.with capture_logs() as logs:- system_prompt = await _system_prompt_for_run(arm=remedy_arm, required_field_answers_arm="treatment")+ system_prompt = await _system_prompt_for_run(required_field_answers_arm="treatment")- assert system_prompt.split(_DATE_MARKER)[0] == expected_prompt+ assert system_prompt.split(_DATE_MARKER)[0] == SYSTEM_PROMPTevents = [e["event"] for e in logs]assert (events.count("Task V3 required-field-answers arm resolved treatment but no text is supplied; sent control") == 1)- assert "Task V3 unanswerable-field remedy suppressed by required-field-answers arm" not in events@pytest.mark.parametrize(@@ -2206,7 +2148,6 @@ def test_required_field_answers_falls_back_to_control_when_an_anchor_drifts(with capture_logs() as logs:prompt = system_prompt_for_run_arms(required_field_answers_text=(STUB_REQUIRED_FIELD_ANSWERS_FILL, STUB_SELF_SCREEN_BULLET),- unanswerable_field_remedy=False,customer_precedence=False,)finally:@@ -2215,54 +2156,42 @@ def test_required_field_answers_falls_back_to_control_when_an_anchor_drifts(assert [e["event"] for e in logs] == ["Task V3 required-field-answers clause is not uniquely present; sent control"]-def test_unanswerable_field_remedy_clause_stays_uniquely_present_and_the_rule_is_not_gated() -> None:- # If the prompt is edited so the clause no longer matches, the treatment arm silently becomes a- # no-op and the experiment reads null for the wrong reason. This reds on that edit.- assert SYSTEM_PROMPT.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == 1- # The rule the remedy hangs off is the safety property and is NOT part of the variable.- rule = "Do not invent sensitive or identifying values (government IDs, financial details, or legal/eligibility attestations)"- assert rule in SYSTEM_PROMPT- assert rule in system_prompt_for_run_arms(- required_field_answers_text=None, unanswerable_field_remedy=True, customer_precedence=False- )--@pytest.mark.asyncio@pytest.mark.parametrize("precedence_arm", [None, "treatment"])@pytest.mark.parametrize("required_field_answers_arm", [None, "treatment"])-async def test_unanswerable_field_remedy_treatment_adds_no_submit_pressure(+async def test_prompt_arms_add_no_submit_pressure(required_field_answers_arm: str | None,precedence_arm: str | None,stub_required_field_answers_text: tuple[str, str],) -> None:# The charter's non-negotiable: while the only thing standing between a model error and an# unauthorized submit is a line of system prompt, no arm may add prose that competes with it.- # An earlier revision ended the treatment with "report the task complete only if the page itself- # accepted the submission", which on a goal that never asked for a submission reads as the only- # route to success running through one. Anti-relabelling never needed it: the ungated- # how-to-work bullet below already requires every required field to hold its value before- # completed, in BOTH arms. Asserted on the prompt the engine actually sends, not the constant.- # The supplied required-field-answers wording itself is pinned against this where it lives.+ # Asserted on the prompt the engine actually sends, not the constant. The supplied+ # required-field-answers wording itself is pinned against this where it lives.treatment = await _system_prompt_for_run(- arm="treatment", required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm- )- control = await _system_prompt_for_run(- arm="control", required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm+ required_field_answers_arm=required_field_answers_arm, precedence_arm=precedence_arm)- base_control = await _system_prompt_for_run(arm="control")+ base_control = await _system_prompt_for_run()bullet = next(line for line in treatment.splitlines() if line.startswith("- Fill fields from the task's data"))assert "submission" not in bullet and "accepted" not in bullet- # The optional-fields instruction shares the bullet and must stay untouched by the remedy.assert "Leave optional fields blank" in bullet- # The anti-relabelling property the removed clause used to carry, in its real home: ungated, and- # byte-identical across the arms, so neither arm carries a completion rule the other does not.+ # The completion rule is ungated and byte-identical across the arms.contract = next(line for line in treatment.splitlines() if "status=completed" in line)assert "every required field holds its intended value" in contract- assert contract in control and contract in base_control- # The no-submit rule is the guard the charter is protecting; it must survive the swap intact.+ assert contract in base_controlno_submit = "Do not submit forms or take irreversible actions unless the goal explicitly instructs it."- assert no_submit in treatment and no_submit in control and no_submit in base_control+ assert no_submit in treatment and no_submit in base_control+ # Pinned as a literal so an edit weakening SYSTEM_PROMPT cannot pass by weakening the constant too. Only the+ # required-field-answers arm rewrites this rule; every other arm sends it verbatim.+ do_not_invent = (+ "Do not invent sensitive or identifying values (government IDs, financial details, or "+ "legal/eligibility attestations); if one of those is required and not provided, stop and report it "+ "rather than guessing."+ )+ assert do_not_invent in base_control+ if required_field_answers_arm is None:+ assert do_not_invent in treatment# The precedence paragraph sits beside that guard, so it may name submitting only to exempt that guard.if precedence_arm == "treatment":paragraph = treatment.split(CUSTOMER_PRECEDENCE_ANCHOR)[0].split("\n\n")[-1]@@ -2279,22 +2208,17 @@ def _body(prompt: str) -> str:@pytest.mark.asyncio@pytest.mark.parametrize("precedence_arm", [None, "control", "unrandomized"])async def test_customer_precedence_off_arms_send_todays_prompt(precedence_arm: str | None) -> None:- system_prompt = await _system_prompt_for_run(arm=None, precedence_arm=precedence_arm)+ system_prompt = await _system_prompt_for_run(precedence_arm=precedence_arm)assert system_prompt.startswith(SYSTEM_PROMPT)assert CUSTOMER_PRECEDENCE_TEXT not in system_prompt- assert (- system_prompt_for_run_arms(- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=False- )- is SYSTEM_PROMPT- )+ assert system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=False) is SYSTEM_PROMPT@pytest.mark.asyncioasync def test_customer_precedence_treatment_adds_the_paragraph_before_how_to_work_and_nothing_else() -> None:- control = _body(await _system_prompt_for_run(arm=None, precedence_arm="control"))- treatment = _body(await _system_prompt_for_run(arm=None, precedence_arm="treatment"))+ control = _body(await _system_prompt_for_run(precedence_arm="control"))+ treatment = _body(await _system_prompt_for_run(precedence_arm="treatment"))assert SYSTEM_PROMPT.count(CUSTOMER_PRECEDENCE_ANCHOR) == 1assert control != treatment@@ -2313,9 +2237,7 @@ def test_customer_precedence_sends_the_prompt_unchanged_when_its_anchor_drifts() -> None:monkeypatch.setattr(engine_mod, "SYSTEM_PROMPT", drifted_prompt)with capture_logs() as logs:- prompt = system_prompt_for_run_arms(- required_field_answers_text=None, unanswerable_field_remedy=False, customer_precedence=True- )+ prompt = system_prompt_for_run_arms(required_field_answers_text=None, customer_precedence=True)assert prompt is drifted_promptassert [e["event"] for e in logs] == ["Task V3 customer-precedence anchor is not uniquely present; sent the prompt without it"@@ -2324,32 +2246,24 @@ def test_customer_precedence_sends_the_prompt_unchanged_when_its_anchor_drifts(@pytest.mark.asyncio@pytest.mark.parametrize("required_field_answers_arm", [None, "treatment"])[email protected]("remedy_arm", [None, "treatment"])async def test_customer_precedence_composes_with_the_fill_rule_arms(- remedy_arm: str | None,required_field_answers_arm: str | None,stub_required_field_answers_text: tuple[str, str],) -> None:without = _body(- await _system_prompt_for_run(- arm=remedy_arm, required_field_answers_arm=required_field_answers_arm, precedence_arm="control"- )+ await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm, precedence_arm="control"))with_precedence = _body(- await _system_prompt_for_run(- arm=remedy_arm, required_field_answers_arm=required_field_answers_arm, precedence_arm="treatment"- )+ await _system_prompt_for_run(required_field_answers_arm=required_field_answers_arm, precedence_arm="treatment"))# Above "How to work:" (named literally, not through the anchor constant), outside every span another arm rewrites.assert with_precedence.count(CUSTOMER_PRECEDENCE_TEXT) == 1assert with_precedence.split("\n\nHow to work:\n")[0].endswith(CUSTOMER_PRECEDENCE_TEXT)assert with_precedence.replace(CUSTOMER_PRECEDENCE_TEXT, "", 1) == without- remedy_applied = remedy_arm == "treatment" and required_field_answers_arm is None- assert with_precedence.count(UNANSWERABLE_FIELD_REMEDY_CONTROL) == (0 if remedy_applied else 1)- assert with_precedence.count(UNANSWERABLE_FIELD_REMEDY_TREATMENT) == (1 if remedy_applied else 0)+ assert with_precedence.count(SENSITIVE_FIELD_STOP_CLAUSE) == 1for other_arms_anchor in (- UNANSWERABLE_FIELD_REMEDY_CONTROL,+ SENSITIVE_FIELD_STOP_CLAUSE,REQUIRED_FIELD_ANSWERS_ANCHOR,SELF_SCREEN_ANCHOR,CUSTOMER_PRECEDENCE_ANCHOR,@@ -2362,7 +2276,7 @@ async def test_customer_precedence_keeps_page_text_out_of_the_users_reach() -> N# Security-critical wording, so pinned exactly: page text never becomes the user's instruction, and the two# prose guards stay outside the precedence while they are the only guards. Both halves refer to the rules by# their own conditions rather than restating them: a paraphrase narrows or widens what the rule covers.- treatment = _body(await _system_prompt_for_run(arm=None, precedence_arm="treatment"))+ treatment = _body(await _system_prompt_for_run(precedence_arm="treatment"))assert "where they conflict with a general rule in this prompt, follow the user" in treatmentassert (