Skip to content

Automate browser based workflows with AI

Persist the model's turn reasoning on Task V3 action rows (#8354)

pedrohsdbbde7e27b3510
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index d550f3a2a..86bed0ad6 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -44,6 +44,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
from skyvern.schemas.workflows import FileStorageType
from skyvern.utils.strings import sanitize_identifier
from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import reasoning_is_turn_scoped
LOG = structlog.get_logger(__name__)
GENERATE_CODE_AI_MODE_PROACTIVE = "proactive"
@@ -739,6 +740,14 @@ def safe_name(label: str) -> str:
return s
+def _per_action_reasoning(act: dict) -> str:
+ """A v3 row's reasoning is the whole turn's text, shared across the round — never a per-action
+ prompt source. Only pre-v3 rows carry per-action reasoning."""
+ if reasoning_is_turn_scoped(act.get("description")):
+ return ""
+ return act.get("reasoning") or ""
+
+
def _value(value: Any) -> cst.BaseExpression:
"""Convert simple Python objects to CST expressions."""
if isinstance(value, str):
@@ -1107,7 +1116,7 @@ def _action_to_stmt(
value = option.get("value")
label = option.get("label")
value = value or label
- if not value and use_semantic_selectors and (act.get("intention") or act.get("reasoning")):
+ if not value and use_semantic_selectors and (act.get("intention") or _per_action_reasoning(act)):
args.append(
cst.Arg(
keyword=cst.Name("ai"),
@@ -1346,7 +1355,7 @@ def _action_to_stmt(
)
)
- intention = act.get("intention") or act.get("reasoning") or ""
+ intention = act.get("intention") or _per_action_reasoning(act) or ""
if intention and method not in ACTIONS_OPT_OUT_INTENTION_FOR_PROMPT:
# Try to parameterize the prompt for click actions so cached scripts
# don't embed recording-specific values (e.g. a patient ID).
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 412d5914f..2ba2ca150 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -195,6 +195,7 @@ from skyvern.utils.token_counter import count_tokens
from skyvern.utils.url_validators import strip_query_params
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
+ TASK_V3_ACTION_DESCRIPTION_PREFIX,
Action,
ActionStatus,
ClickAction,
@@ -370,6 +371,11 @@ def _redact_tool_arg(value: Any, secret_values: set[str]) -> Any:
return value
+# Cap on the persisted turn text: actions is a high-traffic table and readers clamp far lower for
+# display; a runaway multi-paragraph turn must not become a per-row payload.
+_TASKV3_REASONING_MAX_CHARS = 1000
+
+
def _taskv3_action_for_tool_call(tool_name: str, args: dict[str, Any], **fields: Any) -> Action:
"""Build the Action a v3 tool call persists as, carrying the fields its typed subclass requires so
the row hydrates as that subclass instead of falling back to base Action on every read."""
@@ -392,7 +398,11 @@ def _taskv3_action_for_tool_call(tool_name: str, args: dict[str, Any], **fields:
if tool_name == "navigate":
return GotoUrlAction(url=str(args.get("url") or ""), **fields)
if tool_name == "reload_page":
- return ReloadPageAction(reasoning=str(args.get("reason") or ""), **fields)
+ # fields carries reasoning=turn_reasoning; popped to avoid a duplicate kwarg. Loop-injected
+ # reloads always set a "reason" arg, so the turn-text fallback is future-proofing only.
+ turn_reasoning = fields.pop("reasoning", None)
+ reload_reason = str(args.get("reason") or "") or (turn_reasoning or "")
+ return ReloadPageAction(reasoning=reload_reason, **fields)
return Action(action_type=_TASKV3_TOOL_ACTION_TYPES.get(tool_name, ActionType.CLICK), **fields)
@@ -1759,7 +1769,9 @@ class ForgeAgent:
v3_persisted_actions: list[Action] = []
v3_round_index = 0
- async def _on_action_round(round_actions: list[tuple[str, dict[str, Any], bool]]) -> None:
+ async def _on_action_round(
+ round_actions: list[tuple[str, dict[str, Any], bool]], turn_reasoning: str | None
+ ) -> None:
nonlocal v3_round_index
screenshot_artifact_id: str | None = None
try:
@@ -1769,9 +1781,19 @@ class ForgeAgent:
)
except Exception:
LOG.warning("task_v3 failed to capture post-action screenshot", task_id=task.task_id, exc_info=True)
- secret_values: set[str] = set()
- if app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled(task.workflow_run_id):
- secret_values = app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run(task.workflow_run_id)
+ # Opted-out runs still floor runtime-resolved secrets (e.g. verification codes),
+ # matching the recording/HAR finalization sites.
+ secret_values = (
+ app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run(task.workflow_run_id)
+ if app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled(task.workflow_run_id)
+ else app.WORKFLOW_CONTEXT_MANAGER.runtime_secret_values_for_artifacts()
+ )
+ if turn_reasoning:
+ if secret_values:
+ # Default (substring) matching: turn text is free-form prose, where a long secret
+ # can be glued to adjacent alphanumerics — boundary anchoring would let it through.
+ turn_reasoning = redact_secrets_from_text(turn_reasoning, secret_values)
+ turn_reasoning = turn_reasoning[:_TASKV3_REASONING_MAX_CHARS]
for name, args, succeeded in round_actions:
try:
tool_args = _redact_tool_args(args if isinstance(args, dict) else {}, secret_values)
@@ -1788,8 +1810,9 @@ class ForgeAgent:
# as one unit of the workflow-run step budget (distinct (task, order) pairs).
step_order=v3_round_index,
action_order=len(v3_persisted_actions),
- description=f"task_v3 {name} {selector}".strip(),
+ description=f"{TASK_V3_ACTION_DESCRIPTION_PREFIX}{name} {selector}".strip(),
screenshot_artifact_id=screenshot_artifact_id,
+ reasoning=turn_reasoning,
)
v3_persisted_actions.append(action)
await app.DATABASE.workflow_params.create_action(action=action)
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 20211d0f7..728e58f34 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -203,7 +203,7 @@ async def run_task_v3_agent_loop(
prompt_name: str = "taskv3-agent-loop",
step: Any = None,
should_cancel: Callable[[], Awaitable[bool]] | None = None,
- on_action_round: Callable[[list[tuple[str, dict[str, Any], bool]]], Awaitable[None]] | None = None,
+ on_action_round: Callable[[list[tuple[str, dict[str, Any], bool]], str | None], Awaitable[None]] | None = None,
on_pre_action: Callable[[str, dict[str, Any]], Awaitable[None]] | None = None,
extra_tools: list[ToolSpec] | None = None,
extra_system_guidance: str = "",
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 047d0c0b9..5d8896fff 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -1278,7 +1278,7 @@ async def run_agent_tool_loop(
organization_id: str | None = None,
call_kwargs: dict[str, Any] | None = None,
should_cancel: Callable[[], Awaitable[bool]] | None = None,
- on_action_round: Callable[[list[tuple[str, dict[str, Any], bool]]], Awaitable[None]] | None = None,
+ on_action_round: Callable[[list[tuple[str, dict[str, Any], bool]], str | None], Awaitable[None]] | None = None,
on_pre_action: Callable[[str, dict[str, Any]], Awaitable[None]] | None = None,
max_tokens: int | None = None,
deadline_seconds: float | None = None,
@@ -2617,7 +2617,7 @@ async def run_agent_tool_loop(
# persistence hiccup must not abort an otherwise-good run, so failures are contained here.
if round_actions and on_action_round is not None:
try:
- await on_action_round(round_actions)
+ await on_action_round(round_actions, text or None)
except Exception:
LOG.warning("taskv3 on_action_round callback failed", turn=turns, exc_info=True)
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 9038c2c53..44fba37ff 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -118,7 +118,7 @@ from skyvern.utils.css_selector import build_action_summaries_with_timing
from skyvern.utils.script_file_paths import SCRIPT_FILE_PATH_ERROR, normalize_script_file_path
from skyvern.utils.url_validators import validate_fetch_url
from skyvern.webeye.actions.action_types import ActionType
-from skyvern.webeye.actions.actions import Action, DecisiveAction
+from skyvern.webeye.actions.actions import Action, DecisiveAction, reasoning_is_turn_scoped
from skyvern.webeye.cdp_download_interceptor import download_filename_from_suffix
from skyvern.webeye.scraper.scraped_page import ElementTreeFormat
@@ -1181,7 +1181,10 @@ async def _prepare_cached_block_inputs(cache_key: str, prompt: str | None, step_
for action, field_name in action_entries:
if not field_name:
continue
- prompt_text = action.intention or action.reasoning or ""
+ # A v3 row's reasoning is the whole turn's text, shared across the round — not a
+ # per-field prompt; using it would give N fields one prompt naming all N.
+ per_action_reasoning = None if reasoning_is_turn_scoped(action.description) else action.reasoning
+ prompt_text = action.intention or per_action_reasoning or ""
if action.input_or_select_context and action.input_or_select_context.intention:
prompt_text = action.input_or_select_context.intention
field_prompts.append({"name": field_name, "prompt": prompt_text})
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index a254338e7..e7c37d87a 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -128,6 +128,16 @@ class ClickContext(BaseModel):
desired_state: bool | None = None
+# Stamp on Task V3-persisted action rows. Their `reasoning` is the whole TURN's text, shared by
+# every action of the round — readers that assume per-action reasoning (cached-script prompt
+# inference) must key off this stamp and skip it.
+TASK_V3_ACTION_DESCRIPTION_PREFIX = "task_v3 "
+
+
+def reasoning_is_turn_scoped(description: str | None) -> bool:
+ return bool(description) and str(description).startswith(TASK_V3_ACTION_DESCRIPTION_PREFIX)
+
+
class Action(BaseModel):
model_config = ConfigDict(from_attributes=True)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index befce37a7..8fd8478cb 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -49,6 +49,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunS
from skyvern.forge.taskv3.engine import MIN_ACTION_STEPS
from skyvern.forge.taskv3.loop import LoopOutcome
from skyvern.schemas.workflows import BlockStatus, BlockType
+from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER
from skyvern.webeye.actions.actions import (
ActionStatus,
ActionType,
@@ -68,6 +69,7 @@ async def _run_execute_task_v3(
outcome: LoopOutcome,
post_step_side_effect: BaseException | None = None,
action_rounds: list[list[tuple[str, dict[str, Any]]]] | None = None,
+ action_round_texts: list[str | None] | None = None,
screenshot_raises: bool = False,
task_block: BaseTaskBlock | None = None,
validation_without_page_information: bool = False,
@@ -108,8 +110,9 @@ async def _run_execute_task_v3(
loop_mock.active_credential_parameter_key_during_loop = context.active_credential_parameter_key
cb = kwargs.get("on_action_round")
if cb is not None and action_rounds:
- for round_actions in action_rounds:
- await cb(round_actions)
+ for i, round_actions in enumerate(action_rounds):
+ turn_text = action_round_texts[i] if action_round_texts and i < len(action_round_texts) else None
+ await cb(round_actions, turn_text)
if provider_probe_calls:
provider = kwargs["page_provider"]
loop_mock.resolved_pages = [await provider() for _ in range(provider_probe_calls)]
@@ -546,6 +549,36 @@ async def test_execute_task_v3_scrubs_registered_secret_from_extracted_informati
assert "482913" not in task.extracted_information["confirmation"]
+async def test_execute_task_v3_scrubs_registered_secret_from_persisted_action_reasoning(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.forge import agent as agent_mod
+
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True
+ )
+ # A long (>=8 char) secret glued to adjacent alphanumerics: only substring matching (the
+ # free-form-prose mode) catches it — a boundary-anchored scrub would leak it.
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run",
+ lambda *_a, **_k: {"sk4829137765"},
+ )
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["type"])
+ _step, task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=[[("type", {"selector": "#otp", "text": "sk4829137765"}, True)]],
+ action_round_texts=["typing the keysk4829137765into the field"],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert task.status == TaskStatus.completed
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert "sk4829137765" not in (persisted.reasoning or "")
+ assert REDACTED_SECRET_PLACEHOLDER in (persisted.reasoning or "")
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_leaves_extraction_unscrubbed_when_redaction_disabled(
monkeypatch: pytest.MonkeyPatch,
@@ -762,10 +795,12 @@ async def test_execute_task_v3_persists_per_action_screenshots_and_rows(monkeypa
[("click", {"selector": "#a"}, True), ("type", {"selector": "#b", "text": "x"}, True)],
[("click", {"selector": "#submit"}, True)],
]
+ round_texts = ["clicking the field then typing into it", "submitting the form"]
step, task, _loop, _post = await _run_execute_task_v3(
monkeypatch,
outcome,
action_rounds=rounds,
+ action_round_texts=round_texts,
workflow_run_id="wr_v3test",
data_extraction_goal=None,
extracted_information_schema=None,
@@ -782,6 +817,11 @@ async def test_execute_task_v3_persists_per_action_screenshots_and_rows(monkeypa
assert all(a.task_id == task.task_id and a.step_id == step.step_id for a in persisted)
assert all(a.screenshot_artifact_id == "artifact-1" for a in persisted)
assert [a.action_type for a in persisted] == [ActionType.CLICK, ActionType.INPUT_TEXT, ActionType.CLICK]
+ # Every action in a round carries that round's turn text as its reasoning -- neither
+ # intention nor response, which the turn has no per-action value for.
+ assert [a.reasoning for a in persisted] == [round_texts[0], round_texts[0], round_texts[1]]
+ assert all(a.intention is None for a in persisted)
+ assert all(a.response is None for a in persisted)
@pytest.mark.asyncio
@@ -3318,3 +3358,72 @@ async def test_execute_task_v3_persisted_final_url_is_stripped_to_bare(monkeypat
extracted_information_schema=None,
)
assert update_block_mock.await_args.kwargs["final_url"] == "https://example.test/callback"
+
+
+async def test_execute_task_v3_floors_runtime_secrets_when_redaction_disabled(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.forge import agent as agent_mod
+
+ # Org opted out of artifact redaction — runtime-resolved secrets (e.g. a verification code)
+ # must still be floored out of the persisted turn text.
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: False
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.runtime_secret_values_for_artifacts",
+ lambda *_a, **_k: {"73914268"},
+ )
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["type"])
+ _step, task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=[[("type", {"selector": "#otp", "text": "73914268"}, True)]],
+ action_round_texts=["typing the verification code 73914268 into the field"],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert task.status == TaskStatus.completed
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert "73914268" not in (persisted.reasoning or "")
+ assert REDACTED_SECRET_PLACEHOLDER in (persisted.reasoning or "")
+
+
+async def test_execute_task_v3_caps_persisted_reasoning_length(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge import agent as agent_mod
+
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+ _step, task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=[[("click", {"selector": "#a"}, True)]],
+ action_round_texts=["x" * 5000],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert task.status == TaskStatus.completed
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert len(persisted.reasoning or "") == agent_mod._TASKV3_REASONING_MAX_CHARS
+
+
+async def test_execute_task_v3_persists_reload_row_with_its_own_reason(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge import agent as agent_mod
+
+ # Guards the reasoning kwarg-pop in the reload branch: a regression there raises inside the
+ # per-action try and the reload row silently vanishes behind a warning.
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+ _step, task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=[[("reload_page", {"reason": "a page-level handler requested a refresh"}, True)]],
+ action_round_texts=["retrying after the handler asked for a refresh"],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert task.status == TaskStatus.completed
+ assert agent_mod.app.DATABASE.workflow_params.create_action.await_count == 1
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert persisted.reasoning == "a page-level handler requested a refresh"
diff --git a/tests/unit/test_click_prompt_parameterization.py b/tests/unit/test_click_prompt_parameterization.py
index fc3cbc36c..4ea88cbf1 100644
--- a/tests/unit/test_click_prompt_parameterization.py
+++ b/tests/unit/test_click_prompt_parameterization.py
@@ -664,3 +664,32 @@ class TestCollectSecretParamKeys:
def test_returns_empty_when_no_definition(self) -> None:
assert _collect_secret_param_keys({}) == frozenset()
assert _collect_secret_param_keys({"workflow_definition": None}) == frozenset()
+
+
+class TestTurnScopedReasoningExcludedFromPrompts:
+ """A v3-persisted row's reasoning is the whole turn's text (stamped via its description);
+ it must never become a per-action prompt in generated scripts."""
+
+ def _render(self, stmt: cst.BaseStatement) -> str:
+ return cst.Module(body=[stmt]).code
+
+ def test_v3_stamped_reasoning_is_not_a_prompt(self) -> None:
+ act: dict[str, Any] = {
+ "action_type": "click",
+ "xpath": "//button",
+ "description": "task_v3 click [data-tv3='t1']",
+ "reasoning": "Filling name, email and phone, then submitting the form",
+ }
+ stmt = _action_to_stmt(act, {}, value_to_param=None)
+ code = self._render(stmt)
+ assert "Filling name, email and phone" not in code
+
+ def test_unstamped_reasoning_still_prompts(self) -> None:
+ act: dict[str, Any] = {
+ "action_type": "click",
+ "xpath": "//button",
+ "reasoning": "Click the submit button",
+ }
+ stmt = _action_to_stmt(act, {}, value_to_param=None)
+ code = self._render(stmt)
+ assert "Click the submit button" in code
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 17fe3c2a0..3440a8042 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -64,8 +64,11 @@ from skyvern.forge.taskv3.opaque_refs import mask_opaque_urls
class _ScriptedCaller:
"""Emits one queued turn per ``call``. Each turn is a list of (tool_name, args)."""
- def __init__(self, script: list[list[tuple[str, dict[str, Any]]]]) -> None:
+ def __init__(self, script: list[list[tuple[str, dict[str, Any]]]], texts: list[str] | None = None) -> None:
self._script = script
+ # Per-turn assistant text, indexed like `script`; falls back to a fixed placeholder so
+ # existing callers that don't care about the text still get a non-empty one.
+ self._texts = texts
self.calls = 0
self.message_history: list[dict[str, Any]] = []
self.sent_tools: list[dict[str, Any]] | None = None
@@ -103,9 +106,11 @@ class _ScriptedCaller:
if isinstance(part, dict) and part.get("type") in ("image_url", "image")
)
self.image_blocks_per_call.append(history_images + (len(screenshots) if screenshots else 0))
- turn = self._script[self.calls] if self.calls < len(self._script) else []
+ idx = self.calls
+ turn = self._script[idx] if idx < len(self._script) else []
+ text = self._texts[idx] if self._texts and idx < len(self._texts) else "reasoning..."
self.calls += 1
- message: dict[str, Any] = {"content": "reasoning..."}
+ message: dict[str, Any] = {"content": text}
if turn:
message["tool_calls"] = [
{"id": f"call_{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}
@@ -154,8 +159,14 @@ def _erroring_tool(
)
-async def _run(script: list[list[tuple[str, dict[str, Any]]]], tools: list[ToolSpec], **kwargs: Any):
- caller = _ScriptedCaller(script)
+async def _run(
+ script: list[list[tuple[str, dict[str, Any]]]],
+ tools: list[ToolSpec],
+ *,
+ texts: list[str] | None = None,
+ **kwargs: Any,
+):
+ caller = _ScriptedCaller(script, texts=texts)
defaults = {"max_turns": 20, "max_tool_calls": 100}
defaults.update(kwargs)
outcome = await run_agent_tool_loop(
@@ -2305,11 +2316,14 @@ def test_budget_extension_gate_vetoes_fire_independently() -> None:
@pytest.mark.asyncio
async def test_on_action_round_fires_once_per_action_round() -> None:
# The callback fires once per action ROUND (a turn with >=1 successful billable action), not per
- # tool and not on perception-only turns, and receives that round's (name, args) list.
+ # tool and not on perception-only turns, and receives that round's (name, args) list plus the
+ # assistant text the SAME turn produced.
rounds: list[list[tuple[str, dict[str, Any]]]] = []
+ round_texts: list[str | None] = []
- async def _on_round(actions: list[tuple[str, dict[str, Any]]]) -> None:
+ async def _on_round(actions: list[tuple[str, dict[str, Any]]], turn_text: str | None) -> None:
rounds.append(actions)
+ round_texts.append(turn_text)
obs, clk, typ = [], [], []
observe = _recording_tool("observe", obs) # perception, not billable
@@ -2322,10 +2336,14 @@ async def test_on_action_round_fires_once_per_action_round() -> None:
[("click", {"selector": "#a"}), ("type", {"selector": "#b", "text": "x"})], # 1 round, 2 tools -> 1 call
[("finish", {"status": "completed", "reason": "ok"})],
]
- outcome, _ = await _run(script, [observe, click, type_, make_finish_tool()], on_action_round=_on_round)
+ texts = ["looking around", "clicking the field and typing into it", "done"]
+ outcome, _ = await _run(script, [observe, click, type_, make_finish_tool()], on_action_round=_on_round, texts=texts)
assert outcome.status == "completed"
assert len(rounds) == 1
assert rounds[0] == [("click", {"selector": "#a"}, True), ("type", {"selector": "#b", "text": "x"}, True)]
+ # The action round's text is the SECOND turn's ("clicking the field..."), not the first
+ # (perception-only) or third (finish) turn's text.
+ assert round_texts == [texts[1]]
@pytest.mark.asyncio
@@ -2334,7 +2352,7 @@ async def test_on_action_round_fires_for_all_failed_round_with_failure_flag() ->
# callback (flagged unsuccessful) so the round persists into the workflow-run step budget.
rounds: list[list[tuple[str, dict[str, Any], bool]]] = []
- async def _on_round(actions: list[tuple[str, dict[str, Any], bool]]) -> None:
+ async def _on_round(actions: list[tuple[str, dict[str, Any], bool]], _turn_text: str | None) -> None:
rounds.append(actions)
clk: list[tuple[str, dict[str, Any]]] = []
@@ -2349,7 +2367,7 @@ async def test_on_action_round_fires_for_all_failed_round_with_failure_flag() ->
@pytest.mark.asyncio
async def test_on_action_round_failure_does_not_abort_run() -> None:
- async def _boom(actions: list[tuple[str, dict[str, Any]]]) -> None:
+ async def _boom(actions: list[tuple[str, dict[str, Any]]], _turn_text: str | None) -> None:
raise RuntimeError("persist boom")
clk: list[tuple[str, dict[str, Any]]] = []
@@ -5444,7 +5462,7 @@ async def test_completion_probe_ends_loop_mid_batch_without_finish() -> None:
script = [[("click", {"selector": "#a"}), ("click", {"selector": "#b"})]]
recorded_rounds: list[list[tuple[str, dict[str, Any], bool]]] = []
- async def on_action_round(round_actions: list[tuple[str, dict[str, Any], bool]]) -> None:
+ async def on_action_round(round_actions: list[tuple[str, dict[str, Any], bool]], _turn_text: str | None) -> None:
recorded_rounds.append(round_actions)
outcome, _ = await _run(script, tools, completion_probe=probe, on_action_round=on_action_round)
@@ -6421,7 +6439,7 @@ async def test_refresh_reload_is_recorded_in_the_action_round() -> None:
rounds: list[list[tuple[str, dict[str, Any], bool]]] = []
- async def on_round(actions: list[tuple[str, dict[str, Any], bool]]) -> None:
+ async def on_round(actions: list[tuple[str, dict[str, Any], bool]], _turn_text: str | None) -> None:
rounds.append(list(actions))
click_calls: list[tuple[str, dict[str, Any]]] = []