Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-10075): exempt check-user-goal from output slimming (extract-actions-only relaunch) (#6468)
pedrohsdb246ede4499a2
diff --git a/skyvern/forge/sdk/experimentation/prompt_families.py b/skyvern/forge/sdk/experimentation/prompt_families.pyindex 8600d032f..c5ed45bc7 100644--- a/skyvern/forge/sdk/experimentation/prompt_families.py+++ b/skyvern/forge/sdk/experimentation/prompt_families.py@@ -38,9 +38,12 @@ PROMPT_NAMES_BY_FAMILY: dict[PromptFamily, frozenset[str]] = {PromptFamily.PARSE_INPUT_OR_SELECT_CONTEXT: frozenset({"parse-input-or-select-context"}),}-# Families whose templates honor the slim variants at runtime. A family is added-# only after passing its offline backtest gate (SKY-10075).-SLIM_ENABLED_FAMILIES: frozenset[PromptFamily] = frozenset({PromptFamily.EXTRACT_ACTIONS, PromptFamily.CHECK_USER_GOAL})+# Families whose templates honor the slim variants at runtime. A family is enabled+# only after it passes validation; an excluded family keeps its in-template slim+# conditionals for a later re-test. check-user-goal is excluded deliberately —+# slimming the verification prompt degrades its complete/terminate judgment (it+# relies on page_info and full thoughts), so only extract-actions is slimmed.+SLIM_ENABLED_FAMILIES: frozenset[PromptFamily] = frozenset({PromptFamily.EXTRACT_ACTIONS})_FAMILY_BY_TEMPLATE: dict[str, PromptFamily] = {template: family for family, templates in TEMPLATES_BY_FAMILY.items() for template in templatesdiff --git a/tests/unit/test_slim_llm_output.py b/tests/unit/test_slim_llm_output.pyindex ad16ab9f2..2f7dd92dd 100644--- a/tests/unit/test_slim_llm_output.py+++ b/tests/unit/test_slim_llm_output.py@@ -261,8 +261,12 @@ def test_family_lookups() -> None:def test_effective_prompt_schema_variant() -> None:assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "extract-actions") == SLIM_VARIANT_SAFE- assert effective_prompt_schema_variant(SLIM_VARIANT_TERSE, "check-user-goal-after-click") == SLIM_VARIANT_TERSE- # Families outside the allowlist render control, so their calls must be labeled control.+ assert effective_prompt_schema_variant(SLIM_VARIANT_TERSE, "extract-actions") == SLIM_VARIANT_TERSE+ # check-user-goal is intentionally excluded from the allowlist — its calls must+ # label control even under a treatment run.+ assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "check-user-goal") is None+ assert effective_prompt_schema_variant(SLIM_VARIANT_TERSE, "check-user-goal-after-click") is None+ # Other families outside the allowlist render control, so their calls must be labeled control.assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "auto-completion-choose-option") is Noneassert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "parse-input-or-select-context") is None# Unknown prompt_names and non-slim assignments fail safe to control.@@ -298,8 +302,8 @@ async def test_resolver_maps_flag_variant_to_template_value(async def test_resolver_resolves_once_per_run(monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext) -> None:provider = _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)assert await get_slim_output_template_value("extract-action") == "safe"- assert await get_slim_output_template_value("check-user-goal") == "safe"- assert await get_slim_output_template_value("check-user-goal-with-termination") == "safe"+ assert await get_slim_output_template_value("extract-action-static") == "safe"+ assert await get_slim_output_template_value("extract-action-dynamic") == "safe"assert provider.get_value_cached.await_count == 1assert run_context.slim_output_variant_assigned == SLIM_VARIANT_SAFEassert run_context.slim_output_variant_resolved is True@@ -312,6 +316,10 @@ async def test_resolver_excludes_families_outside_allowlist(_mock_provider(monkeypatch, SLIM_VARIANT_SAFE)assert await get_slim_output_template_value("auto-completion-choose-option") is Noneassert await get_slim_output_template_value("parse-input-or-select-context") is None+ # check-user-goal is intentionally excluded — its templates must render control+ # even under a treatment run.+ assert await get_slim_output_template_value("check-user-goal") is None+ assert await get_slim_output_template_value("check-user-goal-with-termination") is None# The assignment is still recorded for telemetry even though these render control.assert run_context.slim_output_variant_assigned == SLIM_VARIANT_SAFE@@ -367,7 +375,7 @@ async def test_concurrent_first_use_resolves_exactly_once(results = await asyncio.gather(get_slim_output_template_value("extract-action"),- get_slim_output_template_value("check-user-goal"),+ get_slim_output_template_value("extract-action-static"),get_slim_output_template_value("extract-action"),)assert results == ["safe", "safe", "safe"]@@ -394,7 +402,7 @@ async def test_concurrent_first_use_with_flag_error_pins_all_callers_to_control(results = await asyncio.gather(get_slim_output_template_value("extract-action"),- get_slim_output_template_value("check-user-goal"),+ get_slim_output_template_value("extract-action-static"),)assert results == [None, None]assert provider.get_value_cached.await_count == 1