Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add an always-on per-iteration TODO list to the copilot agent prompt (#7717)
Andrew Neilsoncfafaae4a28b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex a9fc49e98..74c2740e0 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -194,6 +194,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import (maybe_emit_design_end,)from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, reason_in_reply_shadow+from skyvern.forge.sdk.copilot.todo_list import todo_list_promptfrom skyvern.forge.sdk.copilot.tools.guardrails import _record_output_policy_guardrail_churnfrom skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabledfrom skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket@@ -1244,6 +1245,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -+ _recorded_build_test_outcome_prompt(ctx)+ _code_authoring_repair_context_prompt(ctx)+ _synthesized_block_offer_prompt(ctx)+ + todo_list_prompt(ctx)+ _docs_answer_turn_directive(ctx.turn_intent))diff --git a/skyvern/forge/sdk/copilot/todo_list.py b/skyvern/forge/sdk/copilot/todo_list.pynew file mode 100644index 000000000..b49dbd17d--- /dev/null+++ b/skyvern/forge/sdk/copilot/todo_list.py@@ -0,0 +1,139 @@+"""Per-iteration TODO list rendered from typed turn state; adds context, blocks nothing."""++from __future__ import annotations++from typing import Any++import structlog++from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.runtime import AgentContext++LOG = structlog.get_logger(__name__)+++def _page_key(url: object) -> str | None:+ if not isinstance(url, str) or not url.strip():+ return None+ return url.strip().split("#", 1)[0].rstrip("/")+++def _credential_fill_page_keys(ctx: AgentContext) -> set[str] | None:+ """Pages where a credential fill happened, or None when no fill has happened at all."""+ keys: set[str] = set()+ fills = 0+ entries: list[dict[str, Any]] = [dict(interaction) for interaction in ctx.scout_trajectory]+ entries.extend(entry for entry in ctx.prior_fill_carry if isinstance(entry, dict))+ for entry in entries:+ if entry.get("tool_name") != "fill_credential_field":+ continue+ fills += 1+ key = _page_key(entry.get("source_url"))+ if key:+ keys.add(key)+ return keys if fills else None+++def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:+ """Pages reached by actually interacting, not just looking — the only evidence that counts as login progress.++ Heuristic ceiling: any interaction-reached page off the fill page counts, so a non-submit+ navigation (e.g. a forgot-password link) can suppress the login line without a real login.+ """+ keys: set[str] = set()+ for page in ctx.prior_observed_acted_pages:+ if isinstance(page, dict) and page.get("reached_via") == "interaction":+ key = _page_key(page.get("url"))+ if key:+ keys.add(key)+ for entry in ctx.flow_evidence:+ if entry.get("reached_via") != "interaction":+ continue+ url = entry.get("url")+ evidence = entry.get("evidence")+ if not _page_key(url) and isinstance(evidence, dict):+ url = evidence.get("current_url") or evidence.get("inspected_url")+ key = _page_key(url)+ if key:+ keys.add(key)+ return keys+++def _login_line(ctx: AgentContext) -> str | None:+ policy = ctx.request_policy+ if not isinstance(policy, RequestPolicy) or not policy.login_intent or not policy.resolved_credentials:+ return None+ fill_pages = _credential_fill_page_keys(ctx)+ if fill_pages is None:+ return "Login: credential resolved but login not yet attempted"+ if _interaction_reached_page_keys(ctx) - fill_pages:+ return None+ return "Login: credential resolved but login not completed (no page reached by interaction yet)"+++def _minted_criteria(ctx: AgentContext) -> list[CompletionCriterion]:+ turn_state = ctx.completion_criteria_turn_state+ if turn_state is not None and turn_state.decision is not None:+ return list(turn_state.decision.criteria)+ policy = ctx.request_policy+ if isinstance(policy, RequestPolicy):+ return policy.graded_completion_criteria()+ return []+++def _satisfied_output_paths(ctx: AgentContext) -> set[str]:+ result = ctx.completion_verification_result+ if result is None:+ return set()+ paths: set[str] = set()+ for verdict in result.verdicts:+ if not verdict.satisfied:+ continue+ # A definition-plane satisfied verdict proves the workflow is configurable, never that+ # a run produced the output (same discriminator as is_fully_satisfied).+ if verdict.reason_code.startswith("definition_"):+ continue+ for path in (verdict.output_path, result.criterion_output_path_by_id.get(verdict.criterion_id)):+ if path:+ paths.add(path)+ return paths+++def _outputs_line(ctx: AgentContext) -> str | None:+ satisfied = _satisfied_output_paths(ctx)+ pending: list[str] = []+ for criterion in _minted_criteria(ctx):+ # Definition-plane criteria are graded against the YAML and only ever get+ # definition_* verdicts, so a run-plane pending check would nag forever.+ if criterion.level == "definition":+ continue+ path = criterion.output_path+ if path and path not in satisfied and path not in pending:+ pending.append(path)+ if not pending:+ return None+ return "Outputs not yet observed: " + ", ".join(pending)+++def _interactions_line(ctx: AgentContext) -> str | None:+ if ctx.scout_trajectory or ctx.prior_fill_carry or _interaction_reached_page_keys(ctx):+ return None+ return "The site has not been acted on yet (0 interactions recorded)"+++def render_todo_list(ctx: AgentContext) -> str | None:+ lines = [line for line in (_login_line(ctx), _outputs_line(ctx)) if line]+ if not lines:+ return None+ interactions = _interactions_line(ctx)+ if interactions:+ lines.append(interactions)+ return "\n".join(f"- {line}" for line in lines)+++def todo_list_prompt(ctx: AgentContext) -> str:+ todo = render_todo_list(ctx)+ if not todo:+ return ""+ LOG.debug("copilot_todo_list_rendered", line_count=todo.count("\n") + 1)+ return "\n\nTODO — outstanding before you reply:\n" + tododiff --git a/tests/unit/test_copilot_todo_list.py b/tests/unit/test_copilot_todo_list.pynew file mode 100644index 000000000..74d9161a5--- /dev/null+++ b/tests/unit/test_copilot_todo_list.py@@ -0,0 +1,189 @@+from datetime import datetime, timezone+from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.todo_list import render_todo_list, todo_list_prompt+from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType+from tests.unit.copilot_test_helpers import make_copilot_ctx+++def _login_policy() -> RequestPolicy:+ now = datetime(2026, 1, 1, tzinfo=timezone.utc)+ credential = Credential(+ credential_id="cred-1",+ organization_id="org-1",+ name="Example Login",+ vault_type=CredentialVaultType.SKYVERN,+ item_id="item_1",+ credential_type=CredentialType.PASSWORD,+ username="[email protected]",+ card_last4=None,+ card_brand=None,+ created_at=now,+ modified_at=now,+ )+ return RequestPolicy(login_intent=True, resolved_credentials=[credential])+++def _output_criterion(path: str) -> CompletionCriterion:+ return CompletionCriterion(id=f"crit-{path}", outcome=f"observe {path}", output_path=path)+++def test_login_not_attempted() -> None:+ ctx = make_copilot_ctx(request_policy=_login_policy())+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "- Login: credential resolved but login not yet attempted" in todo+ assert "- The site has not been acted on yet (0 interactions recorded)" in todo+++def test_login_filled_but_no_page_reached_by_interaction() -> None:+ ctx = make_copilot_ctx(request_policy=_login_policy())+ ctx.scout_trajectory = [+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}+ ]+ assert (+ render_todo_list(ctx)+ == "- Login: credential resolved but login not completed (no page reached by interaction yet)"+ )+++def test_read_only_page_observations_do_not_complete_login() -> None:+ ctx = make_copilot_ctx(request_policy=_login_policy())+ ctx.scout_trajectory = [+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}+ ]+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "inspection"}]+ ctx.observed_browser_urls = ["https://example.com/dashboard"]+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "login not completed" in todo+++def test_login_completed_via_interaction_reached_page() -> None:+ ctx = make_copilot_ctx(request_policy=_login_policy())+ ctx.prior_fill_carry = [+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}+ ]+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "interaction"}]+ assert render_todo_list(ctx) is None+++def test_outputs_not_yet_observed() -> None:+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.number_of_visitors_last_week")])+ ctx = make_copilot_ctx(request_policy=policy)+ ctx.scout_trajectory = [{"tool_name": "click", "source_url": "https://example.com/stats"}]+ assert render_todo_list(ctx) == "- Outputs not yet observed: output.number_of_visitors_last_week"+++def test_credential_id_on_non_fill_tool_is_not_a_login_attempt() -> None:+ ctx = make_copilot_ctx(request_policy=_login_policy())+ ctx.scout_trajectory = [+ {"tool_name": "click", "source_url": "https://example.com/login", "credential_id": "cred-1"}+ ]+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "login not yet attempted" in todo+++def test_interaction_reached_page_suppresses_zero_interactions_line() -> None:+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])+ ctx = make_copilot_ctx(request_policy=policy)+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/stats", "reached_via": "interaction"}]+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "has not been acted on" not in todo+++def test_zero_interactions_line_appended_when_work_remains() -> None:+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])+ ctx = make_copilot_ctx(request_policy=policy)+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "- Outputs not yet observed: output.total" in todo+ assert "- The site has not been acted on yet (0 interactions recorded)" in todo+++def test_outputs_satisfied_by_verification() -> None:+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])+ ctx = make_copilot_ctx(request_policy=policy)+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["crit-output.total"],+ verdicts=[+ CriterionVerdict(+ criterion_id="crit-output.total",+ state="satisfied",+ reason_code="observed",+ output_path="output.total",+ )+ ],+ )+ assert render_todo_list(ctx) is None+++def test_definition_plane_satisfied_verdict_does_not_clear_output() -> None:+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])+ ctx = make_copilot_ctx(request_policy=policy)+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["crit-output.total"],+ verdicts=[+ CriterionVerdict(+ criterion_id="crit-output.total",+ state="satisfied",+ reason_code="definition_parameters_referenced",+ output_path="output.total",+ )+ ],+ )+ todo = render_todo_list(ctx)+ assert todo is not None+ assert "- Outputs not yet observed: output.total" in todo+++def test_definition_level_criterion_output_is_never_pending() -> None:+ criterion = CompletionCriterion(+ id="crit-def", outcome="workflow references the total parameter", level="definition", output_path="output.total"+ )+ policy = RequestPolicy(completion_criteria=[criterion])+ ctx = make_copilot_ctx(request_policy=policy)+ ctx.scout_trajectory = [{"tool_name": "click", "source_url": "https://example.com/stats"}]+ assert render_todo_list(ctx) is None+++def test_all_done_returns_none() -> None:+ assert render_todo_list(make_copilot_ctx()) is None+++def test_login_and_outputs_combined() -> None:+ policy = _login_policy()+ policy.completion_criteria = [_output_criterion("output.total")]+ todo = render_todo_list(make_copilot_ctx(request_policy=policy))+ assert todo is not None+ assert "- Login: credential resolved but login not yet attempted" in todo+ assert "- Outputs not yet observed: output.total" in todo+++def test_dynamic_system_prompt_injects_todo_list(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())+ ctx = make_copilot_ctx(request_policy=_login_policy())+ prompt = instructions(SimpleNamespace(context=ctx), None)+ assert "TODO — outstanding before you reply:" in prompt+ assert "Login: credential resolved but login not yet attempted" in prompt+++def test_dynamic_system_prompt_adds_nothing_when_nothing_pending(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())+ prompt = instructions(SimpleNamespace(context=make_copilot_ctx(request_policy=RequestPolicy())), None)+ assert "TODO — outstanding before you reply:" not in prompt+++def test_todo_list_prompt_empty_when_nothing_pending() -> None:+ assert todo_list_prompt(make_copilot_ctx()) == ""