Skip to content

Automate browser based workflows with AI

Add an always-on per-iteration TODO list to the copilot agent prompt (#7717)

Andrew Neilsoncfafaae4a28b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a9fc49e98..74c2740e0 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -194,6 +194,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import (
maybe_emit_design_end,
)
from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, reason_in_reply_shadow
+from skyvern.forge.sdk.copilot.todo_list import todo_list_prompt
from skyvern.forge.sdk.copilot.tools.guardrails import _record_output_policy_guardrail_churn
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket
@@ -1244,6 +1245,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ _recorded_build_test_outcome_prompt(ctx)
+ _code_authoring_repair_context_prompt(ctx)
+ _synthesized_block_offer_prompt(ctx)
+ + todo_list_prompt(ctx)
+ _docs_answer_turn_directive(ctx.turn_intent)
)
diff --git a/skyvern/forge/sdk/copilot/todo_list.py b/skyvern/forge/sdk/copilot/todo_list.py
new file mode 100644
index 000000000..b49dbd17d
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/todo_list.py
@@ -0,0 +1,139 @@
+"""Per-iteration TODO list rendered from typed turn state; adds context, blocks nothing."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+
+LOG = structlog.get_logger(__name__)
+
+
+def _page_key(url: object) -> str | None:
+ if not isinstance(url, str) or not url.strip():
+ return None
+ return url.strip().split("#", 1)[0].rstrip("/")
+
+
+def _credential_fill_page_keys(ctx: AgentContext) -> set[str] | None:
+ """Pages where a credential fill happened, or None when no fill has happened at all."""
+ keys: set[str] = set()
+ fills = 0
+ entries: list[dict[str, Any]] = [dict(interaction) for interaction in ctx.scout_trajectory]
+ entries.extend(entry for entry in ctx.prior_fill_carry if isinstance(entry, dict))
+ for entry in entries:
+ if entry.get("tool_name") != "fill_credential_field":
+ continue
+ fills += 1
+ key = _page_key(entry.get("source_url"))
+ if key:
+ keys.add(key)
+ return keys if fills else None
+
+
+def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:
+ """Pages reached by actually interacting, not just looking — the only evidence that counts as login progress.
+
+ Heuristic ceiling: any interaction-reached page off the fill page counts, so a non-submit
+ navigation (e.g. a forgot-password link) can suppress the login line without a real login.
+ """
+ keys: set[str] = set()
+ for page in ctx.prior_observed_acted_pages:
+ if isinstance(page, dict) and page.get("reached_via") == "interaction":
+ key = _page_key(page.get("url"))
+ if key:
+ keys.add(key)
+ for entry in ctx.flow_evidence:
+ if entry.get("reached_via") != "interaction":
+ continue
+ url = entry.get("url")
+ evidence = entry.get("evidence")
+ if not _page_key(url) and isinstance(evidence, dict):
+ url = evidence.get("current_url") or evidence.get("inspected_url")
+ key = _page_key(url)
+ if key:
+ keys.add(key)
+ return keys
+
+
+def _login_line(ctx: AgentContext) -> str | None:
+ policy = ctx.request_policy
+ if not isinstance(policy, RequestPolicy) or not policy.login_intent or not policy.resolved_credentials:
+ return None
+ fill_pages = _credential_fill_page_keys(ctx)
+ if fill_pages is None:
+ return "Login: credential resolved but login not yet attempted"
+ if _interaction_reached_page_keys(ctx) - fill_pages:
+ return None
+ return "Login: credential resolved but login not completed (no page reached by interaction yet)"
+
+
+def _minted_criteria(ctx: AgentContext) -> list[CompletionCriterion]:
+ turn_state = ctx.completion_criteria_turn_state
+ if turn_state is not None and turn_state.decision is not None:
+ return list(turn_state.decision.criteria)
+ policy = ctx.request_policy
+ if isinstance(policy, RequestPolicy):
+ return policy.graded_completion_criteria()
+ return []
+
+
+def _satisfied_output_paths(ctx: AgentContext) -> set[str]:
+ result = ctx.completion_verification_result
+ if result is None:
+ return set()
+ paths: set[str] = set()
+ for verdict in result.verdicts:
+ if not verdict.satisfied:
+ continue
+ # A definition-plane satisfied verdict proves the workflow is configurable, never that
+ # a run produced the output (same discriminator as is_fully_satisfied).
+ if verdict.reason_code.startswith("definition_"):
+ continue
+ for path in (verdict.output_path, result.criterion_output_path_by_id.get(verdict.criterion_id)):
+ if path:
+ paths.add(path)
+ return paths
+
+
+def _outputs_line(ctx: AgentContext) -> str | None:
+ satisfied = _satisfied_output_paths(ctx)
+ pending: list[str] = []
+ for criterion in _minted_criteria(ctx):
+ # Definition-plane criteria are graded against the YAML and only ever get
+ # definition_* verdicts, so a run-plane pending check would nag forever.
+ if criterion.level == "definition":
+ continue
+ path = criterion.output_path
+ if path and path not in satisfied and path not in pending:
+ pending.append(path)
+ if not pending:
+ return None
+ return "Outputs not yet observed: " + ", ".join(pending)
+
+
+def _interactions_line(ctx: AgentContext) -> str | None:
+ if ctx.scout_trajectory or ctx.prior_fill_carry or _interaction_reached_page_keys(ctx):
+ return None
+ return "The site has not been acted on yet (0 interactions recorded)"
+
+
+def render_todo_list(ctx: AgentContext) -> str | None:
+ lines = [line for line in (_login_line(ctx), _outputs_line(ctx)) if line]
+ if not lines:
+ return None
+ interactions = _interactions_line(ctx)
+ if interactions:
+ lines.append(interactions)
+ return "\n".join(f"- {line}" for line in lines)
+
+
+def todo_list_prompt(ctx: AgentContext) -> str:
+ todo = render_todo_list(ctx)
+ if not todo:
+ return ""
+ LOG.debug("copilot_todo_list_rendered", line_count=todo.count("\n") + 1)
+ return "\n\nTODO — outstanding before you reply:\n" + todo
diff --git a/tests/unit/test_copilot_todo_list.py b/tests/unit/test_copilot_todo_list.py
new file mode 100644
index 000000000..74d9161a5
--- /dev/null
+++ b/tests/unit/test_copilot_todo_list.py
@@ -0,0 +1,189 @@
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.todo_list import render_todo_list, todo_list_prompt
+from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType
+from tests.unit.copilot_test_helpers import make_copilot_ctx
+
+
+def _login_policy() -> RequestPolicy:
+ now = datetime(2026, 1, 1, tzinfo=timezone.utc)
+ credential = Credential(
+ credential_id="cred-1",
+ organization_id="org-1",
+ name="Example Login",
+ vault_type=CredentialVaultType.SKYVERN,
+ item_id="item_1",
+ credential_type=CredentialType.PASSWORD,
+ username="[email protected]",
+ card_last4=None,
+ card_brand=None,
+ created_at=now,
+ modified_at=now,
+ )
+ return RequestPolicy(login_intent=True, resolved_credentials=[credential])
+
+
+def _output_criterion(path: str) -> CompletionCriterion:
+ return CompletionCriterion(id=f"crit-{path}", outcome=f"observe {path}", output_path=path)
+
+
+def test_login_not_attempted() -> None:
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "- Login: credential resolved but login not yet attempted" in todo
+ assert "- The site has not been acted on yet (0 interactions recorded)" in todo
+
+
+def test_login_filled_but_no_page_reached_by_interaction() -> None:
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ ctx.scout_trajectory = [
+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
+ ]
+ assert (
+ render_todo_list(ctx)
+ == "- Login: credential resolved but login not completed (no page reached by interaction yet)"
+ )
+
+
+def test_read_only_page_observations_do_not_complete_login() -> None:
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ ctx.scout_trajectory = [
+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
+ ]
+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "inspection"}]
+ ctx.observed_browser_urls = ["https://example.com/dashboard"]
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "login not completed" in todo
+
+
+def test_login_completed_via_interaction_reached_page() -> None:
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ ctx.prior_fill_carry = [
+ {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
+ ]
+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "interaction"}]
+ assert render_todo_list(ctx) is None
+
+
+def test_outputs_not_yet_observed() -> None:
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.number_of_visitors_last_week")])
+ ctx = make_copilot_ctx(request_policy=policy)
+ ctx.scout_trajectory = [{"tool_name": "click", "source_url": "https://example.com/stats"}]
+ assert render_todo_list(ctx) == "- Outputs not yet observed: output.number_of_visitors_last_week"
+
+
+def test_credential_id_on_non_fill_tool_is_not_a_login_attempt() -> None:
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "source_url": "https://example.com/login", "credential_id": "cred-1"}
+ ]
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "login not yet attempted" in todo
+
+
+def test_interaction_reached_page_suppresses_zero_interactions_line() -> None:
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
+ ctx = make_copilot_ctx(request_policy=policy)
+ ctx.prior_observed_acted_pages = [{"url": "https://example.com/stats", "reached_via": "interaction"}]
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "has not been acted on" not in todo
+
+
+def test_zero_interactions_line_appended_when_work_remains() -> None:
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
+ ctx = make_copilot_ctx(request_policy=policy)
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "- Outputs not yet observed: output.total" in todo
+ assert "- The site has not been acted on yet (0 interactions recorded)" in todo
+
+
+def test_outputs_satisfied_by_verification() -> None:
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
+ ctx = make_copilot_ctx(request_policy=policy)
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["crit-output.total"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="crit-output.total",
+ state="satisfied",
+ reason_code="observed",
+ output_path="output.total",
+ )
+ ],
+ )
+ assert render_todo_list(ctx) is None
+
+
+def test_definition_plane_satisfied_verdict_does_not_clear_output() -> None:
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
+ ctx = make_copilot_ctx(request_policy=policy)
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["crit-output.total"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="crit-output.total",
+ state="satisfied",
+ reason_code="definition_parameters_referenced",
+ output_path="output.total",
+ )
+ ],
+ )
+ todo = render_todo_list(ctx)
+ assert todo is not None
+ assert "- Outputs not yet observed: output.total" in todo
+
+
+def test_definition_level_criterion_output_is_never_pending() -> None:
+ criterion = CompletionCriterion(
+ id="crit-def", outcome="workflow references the total parameter", level="definition", output_path="output.total"
+ )
+ policy = RequestPolicy(completion_criteria=[criterion])
+ ctx = make_copilot_ctx(request_policy=policy)
+ ctx.scout_trajectory = [{"tool_name": "click", "source_url": "https://example.com/stats"}]
+ assert render_todo_list(ctx) is None
+
+
+def test_all_done_returns_none() -> None:
+ assert render_todo_list(make_copilot_ctx()) is None
+
+
+def test_login_and_outputs_combined() -> None:
+ policy = _login_policy()
+ policy.completion_criteria = [_output_criterion("output.total")]
+ todo = render_todo_list(make_copilot_ctx(request_policy=policy))
+ assert todo is not None
+ assert "- Login: credential resolved but login not yet attempted" in todo
+ assert "- Outputs not yet observed: output.total" in todo
+
+
+def test_dynamic_system_prompt_injects_todo_list(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")
+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())
+ ctx = make_copilot_ctx(request_policy=_login_policy())
+ prompt = instructions(SimpleNamespace(context=ctx), None)
+ assert "TODO — outstanding before you reply:" in prompt
+ assert "Login: credential resolved but login not yet attempted" in prompt
+
+
+def test_dynamic_system_prompt_adds_nothing_when_nothing_pending(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")
+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())
+ prompt = instructions(SimpleNamespace(context=make_copilot_ctx(request_policy=RequestPolicy())), None)
+ assert "TODO — outstanding before you reply:" not in prompt
+
+
+def test_todo_list_prompt_empty_when_nothing_pending() -> None:
+ assert todo_list_prompt(make_copilot_ctx()) == ""