Skip to content

Automate browser based workflows with AI

copilot: split run-context read authority from mutation gates (SKY-10313) (#6176)

Andrew Neilson0ebc4fa8e507
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 2396371ab..900a2feec 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -70,6 +70,7 @@ from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_
from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_intent import (
NO_MUTATION_TURN_INTENT_MODES,
+ READ_CONTEXT_DENIED_MODES,
RequiredContextKey,
TurnIntent,
TurnIntentMode,
@@ -551,7 +552,16 @@ def _native_tools_for_turn(
turn_intent: TurnIntent | None,
request_policy: RequestPolicy | None = None,
) -> list[Any]:
+ # Deferred: matches the existing tools.py <-> routes.workflow_copilot <-> agent.py cycle workaround.
+ from skyvern.forge.sdk.copilot.tools import ANSWER_ONLY_CONTEXT_TOOLS
+
if _turn_intent_disables_tools(turn_intent):
+ if (
+ turn_intent is not None
+ and turn_intent.authority.may_read_run_context
+ and turn_intent.mode not in READ_CONTEXT_DENIED_MODES
+ ):
+ return [tool for tool in native_tools if getattr(tool, "name", None) in ANSWER_ONLY_CONTEXT_TOOLS]
return []
if _request_policy_requires_update_and_run_skip_path(request_policy):
return [tool for tool in native_tools if getattr(tool, "name", None) != "update_workflow"]
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 0834ad083..e80879b7a 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -88,7 +88,13 @@ class StructuredContext(BaseModel):
elif tool == "update_workflow":
self.workflow_state = summary
- elif tool in ("click", "evaluate", "run_blocks_and_collect_debug", "get_run_results"):
+ elif tool in (
+ "click",
+ "evaluate",
+ "run_blocks_and_collect_debug",
+ "update_and_run_blocks",
+ "get_run_results",
+ ):
self.decisions_made.append(f"{tool}: {summary}")
elif tool == "get_browser_screenshot":
@@ -98,7 +104,7 @@ class StructuredContext(BaseModel):
self.urls_visited.append(UrlVisit(url=url, summary="screenshot"))
output = entry.get("output_preview")
- if output and tool in ("run_blocks_and_collect_debug", "get_run_results"):
+ if output and tool in ("run_blocks_and_collect_debug", "update_and_run_blocks", "get_run_results"):
preview = output[:300] if len(output) > 300 else output
self.decisions_made.append(f" output: {preview}")
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 9c749e02b..7f1b58a27 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -70,6 +70,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_r
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_intent import (
NO_MUTATION_TURN_INTENT_MODES,
+ READ_CONTEXT_DENIED_MODES,
UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
TurnIntent,
TurnIntentMode,
@@ -1039,10 +1040,30 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
blocks_update = tool_name in WORKFLOW_MUTATION_TOOLS and not authority.may_update_workflow
blocks_run = tool_name in BLOCK_RUNNING_TOOLS and not authority.may_run_blocks
- blocks_context_read = tool_name in ANSWER_ONLY_CONTEXT_TOOLS
+ # Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:
+ # (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)
+ # (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog
+ may_read_run_context = authority.may_read_run_context and intent.mode not in READ_CONTEXT_DENIED_MODES
+ blocks_context_read = tool_name in ANSWER_ONLY_CONTEXT_TOOLS and not may_read_run_context
+
+ within_turn_read_override = False
+ if blocks_context_read and intent.mode not in READ_CONTEXT_DENIED_MODES:
+ pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
+ if isinstance(pending_run_id, str) and pending_run_id:
+ blocks_context_read = False
+ within_turn_read_override = True
+
if blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):
return None
if not blocks_update and not blocks_run and not blocks_context_read:
+ if within_turn_read_override:
+ LOG.info(
+ "copilot authority gate allowed tool via within-turn read override",
+ authority_gate_layer="turn_intent",
+ turn_intent_mode=intent.mode.value,
+ tool_name=tool_name,
+ turn_intent_within_turn_read_override=True,
+ )
return None
if intent.mode in NO_MUTATION_TURN_INTENT_MODES and blocks_run:
@@ -1054,7 +1075,7 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
elif blocks_update:
reason_code = "turn_intent_update_blocked"
elif blocks_context_read:
- reason_code = "turn_intent_no_mutation_context_read_blocked"
+ reason_code = "turn_intent_context_read_blocked"
else:
reason_code = "turn_intent_run_blocked"
LOG.info(
@@ -1064,6 +1085,7 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
turn_intent_target_entity_types=sorted(intent.target_entities),
turn_intent_may_update_workflow=authority.may_update_workflow,
turn_intent_may_run_blocks=authority.may_run_blocks,
+ turn_intent_may_read_run_context=authority.may_read_run_context,
blocked_tool=tool_name,
safe_reason_code=reason_code,
)
@@ -1075,6 +1097,12 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
f"to run browser blocks for the latest user message. Use `update_workflow` only and keep the draft "
f"unvalidated. safe_reason_code={reason_code}.{detail}"
)
+ if blocks_context_read and not blocks_update and not blocks_run:
+ return (
+ f"TurnIntent classified this turn as `{intent.mode.value}`, so `{tool_name}` is not allowed "
+ f"to read run context for the latest user message. Answer using the context already provided. "
+ f"safe_reason_code={reason_code}.{detail}"
+ )
return (
f"TurnIntent classified this turn as `{intent.mode.value}`, so `{tool_name}` is not allowed "
"for the latest user message. Do not update workflow YAML or run browser blocks, and do not "
@@ -2559,6 +2587,17 @@ async def _run_blocks_and_collect_debug(
async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
workflow_run_id = params.get("workflow_run_id")
+ pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
+ if isinstance(pending_run_id, str) and pending_run_id:
+ if workflow_run_id and workflow_run_id != pending_run_id:
+ return {
+ "ok": False,
+ "error": (
+ f"Run inspection is pending for {pending_run_id}; "
+ "call get_run_results with that workflow_run_id first."
+ ),
+ }
+ workflow_run_id = pending_run_id
if not workflow_run_id:
# Include every final state so the agent can inspect failures via the
@@ -2587,6 +2626,8 @@ async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[st
)
if not run:
return {"ok": False, "error": f"Workflow run not found: {workflow_run_id}"}
+ if getattr(run, "workflow_permanent_id", None) != ctx.workflow_permanent_id:
+ return {"ok": False, "error": f"Workflow run not found for this workflow: {workflow_run_id}"}
blocks = await app.DATABASE.observer.get_workflow_run_blocks(
workflow_run_id=workflow_run_id,
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index d2e490098..8bb6daacd 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -9,6 +9,7 @@ from typing import Any
import yaml
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from skyvern.forge.sdk.copilot.context import StructuredContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, redact_raw_secrets_for_prompt
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -39,6 +40,16 @@ NO_MUTATION_TURN_INTENT_MODES = frozenset(
}
)
+# Modes that have no legitimate use of run context. Used by tools.py to scope
+# the intra-turn read-context override away from explicitly answer-only turns.
+READ_CONTEXT_DENIED_MODES = frozenset(
+ {
+ TurnIntentMode.DOCS_ANSWER,
+ TurnIntentMode.REFUSE,
+ TurnIntentMode.CLARIFY,
+ }
+)
+
class RequiredContextKey(StrEnum):
CURRENT_WORKFLOW = "current_workflow"
@@ -73,6 +84,7 @@ class TurnIntentReasonCode(StrEnum):
CONFIRMATION_CARRYOVER = "confirmation_carryover"
RAW_SECRET_REFUSAL = "raw_secret_refusal"
USER_NON_PROGRESS = "user_non_progress"
+ RECOVERY_FROM_RUN_CONTEXT = "recovery_from_run_context"
class TurnIntentAuthority(BaseModel):
@@ -82,6 +94,8 @@ class TurnIntentAuthority(BaseModel):
may_run_blocks: bool = False
may_answer_without_mutation: bool = True
requires_user_input: bool = False
+ # Decoupled from mutation flags so DIAGNOSE turns may inspect run state without write authority.
+ may_read_run_context: bool = False
class TurnIntent(BaseModel):
@@ -140,6 +154,7 @@ class TurnIntent(BaseModel):
"may_run_blocks": self.authority.may_run_blocks,
"may_answer_without_mutation": self.authority.may_answer_without_mutation,
"requires_user_input": self.authority.requires_user_input,
+ "may_read_run_context": self.authority.may_read_run_context,
"confidence": self.confidence,
"reason_codes": [reason.value for reason in self.reason_codes],
}
@@ -155,6 +170,7 @@ _BUILD_TERMS = ("build", "create", "make", "generate")
_NEW_BROWSER_TASK_TERMS = ("go to", "navigate to", "open", "visit", "search for")
_EDIT_TERMS = ("edit", "update", "change", "modify", "replace", "fix")
_DIAGNOSE_TERMS = ("debug", "diagnose", "failed", "failure", "error", "result")
+_RUN_CONTEXT_REQUEST_TERMS = ("get_run_results", "workflow_run_id", "run result", "run results")
_DOCS_TERMS = (
"explain",
"how do",
@@ -205,6 +221,10 @@ def _contains_any(text: str, terms: tuple[str, ...]) -> bool:
return _word_boundary_pattern(terms).search(text.lower()) is not None
+def _is_explicit_run_context_request(user_message: str) -> bool:
+ return _contains_any(user_message, _RUN_CONTEXT_REQUEST_TERMS)
+
+
def _workflow_block_labels(workflow_yaml: str | None) -> set[str]:
if not workflow_yaml:
return set()
@@ -304,6 +324,37 @@ def _mode_from_keywords(
return None
+_PRIOR_RUN_OUTPUT_PREFIX = " output:"
+_RUN_CONTEXT_DECISION_PREFIXES = (
+ "run_blocks_and_collect_debug:",
+ "update_and_run_blocks:",
+ "get_run_results:",
+)
+_PRIOR_RUN_FAILURE_MARKERS = (
+ "Run ID:",
+ "workflow_run_id",
+ "Outcome is uncertain",
+ "per-tool-call budget",
+)
+
+
+def _decision_records_prior_run(entry: str) -> bool:
+ if entry.startswith(_PRIOR_RUN_OUTPUT_PREFIX):
+ return True
+ if not entry.startswith(_RUN_CONTEXT_DECISION_PREFIXES):
+ return False
+ if any(marker in entry for marker in _PRIOR_RUN_FAILURE_MARKERS):
+ return True
+ return entry.startswith(("run_blocks_and_collect_debug: Run ", "update_and_run_blocks: Run "))
+
+
+def _has_structured_prior_run_signal(global_llm_context: str) -> bool:
+ if not (global_llm_context or "").strip():
+ return False
+ structured = StructuredContext.from_json_str(global_llm_context)
+ return any(isinstance(entry, str) and _decision_records_prior_run(entry) for entry in structured.decisions_made)
+
+
_AFFIRMATIVE_REPLIES = frozenset(
{
"yes",
@@ -466,6 +517,8 @@ def build_turn_intent(
confidence = 0.2 if (has_workflow or has_prior_context or chat_history) else 0.0
missing_context_question = None
+ has_prior_run_signal = workflow_run_id is not None or _has_structured_prior_run_signal(global_llm_context)
+
if request_policy.raw_secret_detected and request_policy.raw_secret_handling != "redacted_draft":
mode = TurnIntentMode.REFUSE
expected_output = TurnIntentExpectedOutput.REFUSAL
@@ -479,6 +532,11 @@ def build_turn_intent(
confidence = 0.8
missing_context_question = request_policy.clarification_question
reason_codes.append(TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION)
+ elif has_prior_run_signal and _is_explicit_run_context_request(user_message):
+ mode = TurnIntentMode.DIAGNOSE
+ expected_output = TurnIntentExpectedOutput.RUN_RESULT
+ confidence = 0.45
+ reason_codes.append(TurnIntentReasonCode.KEYWORD_HEURISTIC)
elif request_policy.testing_intent == "skip_test":
mode = TurnIntentMode.DRAFT_ONLY
expected_output = TurnIntentExpectedOutput.WORKFLOW_DRAFT
@@ -503,6 +561,11 @@ def build_turn_intent(
if unresolved_block_refs:
target_entities[UNRESOLVED_BLOCK_REF_TARGET_ENTITY] = unresolved_block_refs
+ if mode == TurnIntentMode.UNKNOWN and has_prior_run_signal:
+ mode = TurnIntentMode.DIAGNOSE
+ expected_output = TurnIntentExpectedOutput.RUN_RESULT
+ reason_codes.append(TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT)
+
if mode == TurnIntentMode.DOCS_ANSWER:
authority.may_update_workflow = False
authority.may_run_blocks = False
@@ -515,6 +578,8 @@ def build_turn_intent(
authority.may_update_workflow = False
authority.may_run_blocks = False
+ authority.may_read_run_context = mode == TurnIntentMode.DIAGNOSE
+
return TurnIntent(
mode=mode,
user_goal=_normalize_user_goal(user_message),
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 649ff8e18..20d711f33 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -13,7 +13,7 @@ from skyvern.forge.sdk.copilot.agent import (
_native_tools_for_turn,
_store_request_policy_on_context,
)
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.context import CopilotContext, StructuredContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.turn_intent import (
UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
@@ -22,6 +22,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (
TurnIntentAuthority,
TurnIntentMode,
TurnIntentReasonCode,
+ _has_structured_prior_run_signal,
build_turn_intent,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
@@ -83,6 +84,7 @@ def test_turn_intent_trace_data_omits_raw_goal_and_target_values() -> None:
"may_run_blocks": False,
"may_answer_without_mutation": True,
"requires_user_input": False,
+ "may_read_run_context": False,
"confidence": 0.75,
"reason_codes": ["request_policy_derived"],
"target_entity_types": ["credential", "workflow"],
@@ -526,3 +528,163 @@ def test_answer_only_turn_intent_hides_get_run_results_tool() -> None:
filtered = _native_tools_for_turn(tools, intent)
assert filtered == []
+
+
+def _structured_context_with_run_decision() -> str:
+ return StructuredContext(
+ decisions_made=[
+ "run_blocks_and_collect_debug: ran 3 blocks, hit per-tool-call budget",
+ " output: block_extract: {'rbt_certified': true}",
+ ],
+ ).to_json_str()
+
+
+def test_build_turn_intent_unknown_with_workflow_run_id_upgrades_to_diagnose() -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ workflow_run_id="wr_test_123",
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert intent.authority.may_read_run_context is True
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+
+
+def test_build_turn_intent_unknown_with_persisted_prior_run_signal_upgrades_to_diagnose() -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert intent.authority.may_read_run_context is True
+
+
+def test_build_turn_intent_unknown_with_persisted_prior_failed_run_upgrades_to_diagnose() -> None:
+ structured_context = StructuredContext()
+ structured_context.merge_turn_summary(
+ [
+ {
+ "tool": "update_and_run_blocks",
+ "summary": "Failed: The run exceeded the 240s per-tool-call budget. Run ID: wr_budget_123.",
+ }
+ ]
+ )
+ global_llm_context = structured_context.to_json_str()
+ assert _has_structured_prior_run_signal(global_llm_context) is True
+
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=global_llm_context,
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert intent.authority.may_read_run_context is True
+
+
+def test_build_turn_intent_unknown_without_run_signals_stays_unknown() -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.UNKNOWN
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT not in intent.reason_codes
+ assert intent.authority.may_read_run_context is False
+
+
+def test_build_turn_intent_docs_answer_keeps_read_context_false_even_with_run_id() -> None:
+ intent = build_turn_intent(
+ user_message="How do parameters work?",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ workflow_run_id="wr_test_123",
+ )
+
+ assert intent.mode == TurnIntentMode.DOCS_ANSWER
+ assert intent.authority.may_read_run_context is False
+
+
+def test_build_turn_intent_explicit_run_results_outranks_skip_test_policy() -> None:
+ intent = build_turn_intent(
+ user_message="Call get_run_results with workflow_run_id wr_test_123. Do not run blocks.",
+ workflow_yaml="title: Existing workflow\nworkflow_definition:\n blocks: []\n",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(testing_intent="skip_test", allow_update_workflow=True, allow_run_blocks=False),
+ workflow_run_id="wr_test_123",
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_read_run_context is True
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+
+
+def test_build_turn_intent_diagnose_grants_may_read_run_context_without_run_id() -> None:
+ intent = build_turn_intent(
+ user_message="Diagnose the failure.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_read_run_context is True
+
+
+def test_has_structured_prior_run_signal_ignores_workflow_state_only() -> None:
+ payload = StructuredContext(workflow_state="updated workflow yaml diff summary").to_json_str()
+ assert _has_structured_prior_run_signal(payload) is False
+
+
+def test_has_structured_prior_run_signal_ignores_unrelated_decisions() -> None:
+ payload = StructuredContext(
+ decisions_made=["click: clicked submit", "evaluate: returned true"],
+ ).to_json_str()
+ assert _has_structured_prior_run_signal(payload) is False
+
+
+def test_has_structured_prior_run_signal_handles_malformed_json() -> None:
+ assert _has_structured_prior_run_signal("{not valid json") is False
+ assert _has_structured_prior_run_signal("") is False
+
+
+def test_has_structured_prior_run_signal_ignores_failed_tool_attempt_without_output() -> None:
+ # The merger appends a `<tool>: Failed: ...` entry for blocked/loop-rejected
+ # calls, but no ` output:` line (hooks.py only sets output_preview on
+ # ok=True). Failed attempts must not trigger recovery.
+ payload = StructuredContext(
+ decisions_made=["run_blocks_and_collect_debug: Failed: blocked by policy"],
+ ).to_json_str()
+ assert _has_structured_prior_run_signal(payload) is False
+
+
+def test_has_structured_prior_run_signal_matches_output_preview_line() -> None:
+ payload = StructuredContext(
+ decisions_made=[
+ "get_run_results: workflow completed",
+ " output: block_extract: {'result': 'ok'}",
+ ],
+ ).to_json_str()
+ assert _has_structured_prior_run_signal(payload) is True
diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.py
index 4700d51a7..fd2d88b56 100644
--- a/tests/unit/test_copilot_turn_intent_tool_gate.py
+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py
@@ -5,9 +5,10 @@ from unittest.mock import AsyncMock, patch
import pytest
+from skyvern.forge.sdk.copilot.agent import _native_tools_for_turn
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
-from skyvern.forge.sdk.copilot.tools import _turn_intent_tool_error, _update_workflow
+from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _get_run_results, _turn_intent_tool_error, _update_workflow
from skyvern.forge.sdk.copilot.turn_intent import (
UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
TurnIntent,
@@ -16,8 +17,14 @@ from skyvern.forge.sdk.copilot.turn_intent import (
)
-def _ctx(turn_intent: TurnIntent, request_policy: RequestPolicy | None = None) -> CopilotContext:
- return CopilotContext(
+def _ctx(
+ turn_intent: TurnIntent,
+ request_policy: RequestPolicy | None = None,
+ *,
+ pending_reconciliation_run_id: str | None = None,
+ tool_activity: list[dict] | None = None,
+) -> CopilotContext:
+ ctx = CopilotContext(
organization_id="org-1",
workflow_id="wf-1",
workflow_permanent_id="wfp-1",
@@ -27,6 +34,11 @@ def _ctx(turn_intent: TurnIntent, request_policy: RequestPolicy | None = None) -
turn_intent=turn_intent,
request_policy=request_policy,
)
+ if pending_reconciliation_run_id is not None:
+ ctx.pending_reconciliation_run_id = pending_reconciliation_run_id
+ if tool_activity is not None:
+ ctx.tool_activity = tool_activity
+ return ctx
@pytest.mark.parametrize(
@@ -242,16 +254,165 @@ def test_update_and_run_blocks_reports_both_blocked_authorities() -> None:
assert "turn_intent_no_mutation_run_blocked" in error
-def test_answer_only_diagnose_blocks_get_run_results_tool() -> None:
+def test_diagnose_allows_get_run_results_tool() -> None:
intent = TurnIntent(
mode=TurnIntentMode.DIAGNOSE,
- authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ authority=TurnIntentAuthority(
+ may_update_workflow=False,
+ may_run_blocks=False,
+ may_read_run_context=True,
+ ),
+ )
+
+ assert _turn_intent_tool_error(_ctx(intent), "get_run_results") is None
+
+
+def test_unknown_without_run_context_blocks_get_run_results() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.UNKNOWN,
+ authority=TurnIntentAuthority(),
)
error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
assert error is not None
- assert "`diagnose`" in error
+ assert "`unknown`" in error
assert "`get_run_results`" in error
- assert "fetch additional run context with tools" in error
- assert "turn_intent_no_mutation_context_read_blocked" in error
+ assert "turn_intent_context_read_blocked" in error
+ assert "read run context" in error
+ assert "Do not update workflow YAML or run browser blocks" not in error
+
+
+def test_docs_answer_blocks_get_run_results() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DOCS_ANSWER,
+ authority=TurnIntentAuthority(),
+ )
+
+ error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
+
+ assert error is not None
+ assert "`docs_answer`" in error
+ assert "turn_intent_context_read_blocked" in error
+
+
+def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DOCS_ANSWER,
+ authority=TurnIntentAuthority(may_read_run_context=True),
+ )
+
+ error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
+
+ assert error is not None
+ assert "`docs_answer`" in error
+ assert "turn_intent_context_read_blocked" in error
+ assert _native_tools_for_turn(list(NATIVE_TOOLS), intent) == []
+
+
+def test_within_turn_override_pending_reconciliation_allows_read() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.UNKNOWN,
+ authority=TurnIntentAuthority(),
+ )
+
+ ctx = _ctx(intent, pending_reconciliation_run_id="wr_pending_test")
+
+ assert _turn_intent_tool_error(ctx, "get_run_results") is None
+
+
+def test_within_turn_override_excluded_for_docs_answer() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DOCS_ANSWER,
+ authority=TurnIntentAuthority(),
+ )
+
+ ctx = _ctx(intent, pending_reconciliation_run_id="wr_pending_test")
+ error = _turn_intent_tool_error(ctx, "get_run_results")
+
+ assert error is not None
+ assert "turn_intent_context_read_blocked" in error
+
+
+def test_tool_activity_is_not_a_substitute_for_pending_reconciliation_run_id() -> None:
+ # tool_activity entries are appended for every completed tool call,
+ # including ones that failed the authority/loop gate before the run ever
+ # started. The override must key only on pending_reconciliation_run_id,
+ # which the watchdog sets only when a real run exited unfinalized.
+ intent = TurnIntent(
+ mode=TurnIntentMode.UNKNOWN,
+ authority=TurnIntentAuthority(),
+ )
+
+ ctx = _ctx(
+ intent,
+ tool_activity=[{"tool": "run_blocks_and_collect_debug", "summary": "Failed: blocked"}],
+ )
+
+ assert _turn_intent_tool_error(ctx, "get_run_results") is not None
+
+
+def test_recovery_diagnose_keeps_get_run_results_in_native_tools() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DIAGNOSE,
+ authority=TurnIntentAuthority(
+ may_update_workflow=False,
+ may_run_blocks=False,
+ may_read_run_context=True,
+ ),
+ )
+
+ filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)
+ names = {getattr(tool, "name", None) for tool in filtered}
+
+ assert names == {"get_run_results"}
+
+
+async def test_get_run_results_rejects_explicit_run_from_other_workflow() -> None:
+ ctx = _ctx(
+ TurnIntent(mode=TurnIntentMode.DIAGNOSE, authority=TurnIntentAuthority(may_read_run_context=True)),
+ pending_reconciliation_run_id="wr_other",
+ )
+ run = SimpleNamespace(workflow_run_id="wr_other", workflow_permanent_id="wfp-other", status="failed")
+
+ with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
+ mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock()
+ result = await _get_run_results({"workflow_run_id": "wr_other"}, ctx)
+
+ assert result == {"ok": False, "error": "Workflow run not found for this workflow: wr_other"}
+ mock_app.DATABASE.observer.get_workflow_run_blocks.assert_not_called()
+
+
+async def test_get_run_results_uses_pending_reconciliation_run_when_id_omitted() -> None:
+ ctx = _ctx(TurnIntent(mode=TurnIntentMode.UNKNOWN), pending_reconciliation_run_id="wr_pending")
+ run = SimpleNamespace(workflow_run_id="wr_pending", workflow_permanent_id="wfp-1", status="failed")
+
+ with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
+ mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[])
+ result = await _get_run_results({}, ctx)
+
+ assert result["ok"] is True
+ assert result["data"]["workflow_run_id"] == "wr_pending"
+ mock_app.DATABASE.workflow_runs.get_workflow_run.assert_awaited_once_with(
+ workflow_run_id="wr_pending",
+ organization_id="org-1",
+ )
+
+
+async def test_get_run_results_rejects_different_run_while_reconciliation_pending() -> None:
+ ctx = _ctx(TurnIntent(mode=TurnIntentMode.UNKNOWN), pending_reconciliation_run_id="wr_pending")
+
+ with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock()
+ result = await _get_run_results({"workflow_run_id": "wr_other"}, ctx)
+
+ assert result == {
+ "ok": False,
+ "error": "Run inspection is pending for wr_pending; call get_run_results with that workflow_run_id first.",
+ }
+ mock_app.DATABASE.workflow_runs.get_workflow_run.assert_not_called()