Skip to content

Automate browser based workflows with AI

Let Copilot page a long run's results instead of losing rows past the 50k cut (SKY-17095) (#8684)

Andrew Neilson1f1a764ceb35
diff --git a/skyvern/forge/sdk/agents/context.py b/skyvern/forge/sdk/agents/context.py
index a47ec0126..3b6f626e2 100644
--- a/skyvern/forge/sdk/agents/context.py
+++ b/skyvern/forge/sdk/agents/context.py
@@ -51,7 +51,8 @@ def compact_agent_messages_for_llm(
summarize_tool_output: Callable[[str], str] | None = None,
summarize_tool_arguments: Callable[[str], str] | None = None,
tool_output_truncation_suffix: str = DEFAULT_TRUNCATION_SUFFIX,
- on_recent_truncation: Callable[[int, int], None] | None = None,
+ on_recent_truncation: Callable[[int, int, list[str]], None] | None = None,
+ keep_whole_output_indices: Collection[int] = (),
token_budget: int | None = None,
estimate_tokens: Callable[[list[Any]], int] | None = None,
is_synthetic_message: Callable[[Any], bool] | None = None,
@@ -77,6 +78,7 @@ def compact_agent_messages_for_llm(
summarize_tool_arguments=summarize_tool_arguments,
suffix=tool_output_truncation_suffix,
on_recent_truncation=on_recent_truncation,
+ keep_whole_output_indices=keep_whole_output_indices,
)
if token_budget is None:
return items
@@ -200,16 +202,25 @@ def _compact_tool_payloads(
summarize_tool_output: Callable[[str], str] | None,
summarize_tool_arguments: Callable[[str], str] | None,
suffix: str,
- on_recent_truncation: Callable[[int, int], None] | None = None,
+ on_recent_truncation: Callable[[int, int, list[str]], None] | None = None,
+ keep_whole_output_indices: Collection[int] = (),
) -> list[Any]:
output_indices = [i for i, item in enumerate(items) if _tool_output_field(item) is not None]
call_indices = [i for i, item in enumerate(items) if get_agent_message_field(item, "type") == "function_call"]
recent_outputs = set(output_indices[-keep_recent_tool_outputs:]) if keep_recent_tool_outputs > 0 else set()
+ # Kept outputs are exempt from summarization only; the recent-output char cap still applies to them.
+ recent_outputs |= set(keep_whole_output_indices)
+ tool_names = {
+ get_agent_message_field(item, "call_id"): get_agent_message_field(item, "name")
+ for item in items
+ if get_agent_message_field(item, "type") == "function_call"
+ }
recent_calls = set(call_indices[-keep_recent_tool_outputs:]) if keep_recent_tool_outputs > 0 else set()
compacted: list[Any] = []
recent_truncated_count = 0
recent_truncated_largest = 0
+ recent_truncated_names: list[str] = []
for i, item in enumerate(items):
output_field = _tool_output_field(item)
if output_field is not None:
@@ -223,6 +234,7 @@ def _compact_tool_payloads(
if i in recent_outputs and new_output != output:
recent_truncated_count += 1
recent_truncated_largest = max(recent_truncated_largest, len(output))
+ recent_truncated_names.append(tool_names.get(get_agent_message_field(item, "call_id")) or "unknown")
item = replace_agent_message_field(item, output_field, new_output) if new_output != output else item
elif get_agent_message_field(item, "type") == "function_call" and i not in recent_calls:
arguments = get_agent_message_field(item, "arguments")
@@ -240,7 +252,7 @@ def _compact_tool_payloads(
)
compacted.append(item)
if recent_truncated_count and on_recent_truncation is not None:
- on_recent_truncation(recent_truncated_count, recent_truncated_largest)
+ on_recent_truncation(recent_truncated_count, recent_truncated_largest, recent_truncated_names)
return compacted
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 960132d79..01f0315f5 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -846,13 +846,19 @@ def _summarize_tool_output(output: str) -> str:
if isinstance(categories, list) and categories:
synopsis["failure_categories"] = categories
- blocks = data.get("blocks")
- if isinstance(blocks, list):
+ for list_key in ("blocks", "block_details"):
+ blocks = data.get(list_key)
+ if not isinstance(blocks, list):
+ continue
block_summary: list[dict[str, Any]] = []
for block in blocks:
if not isinstance(block, dict):
continue
entry: dict[str, Any] = {"label": block.get("label"), "status": block.get("status")}
+ # A paged run-results row keeps its key and iteration, so the model can map it and read it again.
+ for key in ("row_key", "current_index"):
+ if key in block:
+ entry[key] = block[key]
if block.get("failure_reason"):
entry["failure_reason"] = str(block["failure_reason"])[:120]
codes = block.get("error_codes")
@@ -860,7 +866,7 @@ def _summarize_tool_output(output: str) -> str:
entry["error_codes"] = _bounded_error_codes(codes)
block_summary.append(entry)
if block_summary:
- synopsis["blocks"] = block_summary
+ synopsis[list_key] = block_summary
synopsis["_summarized"] = "older tool output — only key fields retained"
try:
@@ -924,15 +930,33 @@ def _summarize_tool_arguments(args_json: str) -> str:
return args_json[:_SUMMARIZED_TOOL_ARGUMENT_CHAR_CAP] + _TOOL_OUTPUT_TRUNCATION_SUFFIX
-def log_recent_tool_output_truncation(truncated_count: int, largest_original_chars: int) -> None:
+def log_recent_tool_output_truncation(truncated_count: int, largest_original_chars: int, tool_names: list[str]) -> None:
LOG.warning(
"copilot_recent_tool_output_truncated",
truncated_count=truncated_count,
cap=_RECENT_TOOL_OUTPUT_CHAR_CAP,
largest_original_chars=largest_original_chars,
+ tool_name=sorted(set(tool_names)),
)
+def _is_model_authored(item: Any) -> bool:
+ return _item_field(item, "type") in ("reasoning", "function_call") or _item_field(item, "role") == "assistant"
+
+
+def unread_tool_output_indices(items: Sequence[Any]) -> set[int]:
+ """Indices of tool outputs no model call has seen yet: those after the last model-authored item. The SDK
+ appends a response's calls, then all of their outputs, so any later model item means the model read them.
+ User, synthetic and screenshot messages are not reads."""
+ unread: set[int] = set()
+ for index in range(len(items) - 1, -1, -1):
+ if _is_model_authored(items[index]):
+ break
+ if _item_field(items[index], "type") == "function_call_output":
+ unread.add(index)
+ return unread
+
+
def _prune_input_list(items: list[Any]) -> list[Any]:
"""Drop all but the most recent screenshot, compress older tool outputs,
and summarize the arguments of older tool CALLS so bulky payloads (like
@@ -948,12 +972,16 @@ def _prune_input_list(items: list[Any]) -> list[Any]:
fco_indices = [i for i, item in enumerate(items) if _item_field(item, "type") == "function_call_output"]
recent_fco_set = set(fco_indices[-KEEP_RECENT_TOOL_OUTPUTS:])
+ recent_fco_set |= unread_tool_output_indices(items)
+
fc_indices = [i for i, item in enumerate(items) if _item_field(item, "type") == "function_call"]
recent_fc_set = set(fc_indices[-KEEP_RECENT_TOOL_OUTPUTS:])
+ tool_names = {_item_field(items[i], "call_id"): _item_field(items[i], "name") for i in fc_indices}
result: list[Any] = []
recent_truncated_count = 0
recent_truncated_largest = 0
+ recent_truncated_names: list[str] = []
for i, item in enumerate(items):
if i in drop_indices:
result.append({"role": "user", "content": SCREENSHOT_PLACEHOLDER})
@@ -969,6 +997,7 @@ def _prune_input_list(items: list[Any]) -> list[Any]:
if new_output != output:
recent_truncated_count += 1
recent_truncated_largest = max(recent_truncated_largest, len(output))
+ recent_truncated_names.append(tool_names.get(_item_field(item, "call_id")) or "unknown")
else:
new_output = output
else:
@@ -984,7 +1013,7 @@ def _prune_input_list(items: list[Any]) -> list[Any]:
result.append(item)
if recent_truncated_count:
- log_recent_tool_output_truncation(recent_truncated_count, recent_truncated_largest)
+ log_recent_tool_output_truncation(recent_truncated_count, recent_truncated_largest, recent_truncated_names)
return result
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index 8e1a3cd44..9e2dd456f 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -220,8 +220,8 @@ class CopilotRunHooks(RunHooksBase):
blocks = []
output_parts = []
for b in blocks:
- if b.get("output") or b.get("extracted_data"):
- out = b.get("output") or b.get("extracted_data")
+ if b.get("output") or b.get("extracted_data") or b.get("output_preview"):
+ out = b.get("output") or b.get("extracted_data") or b.get("output_preview")
out_str = json.dumps(out, default=str) if not isinstance(out, str) else out
if len(out_str) > 500:
out_str = out_str[:500] + "..."
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 0a8c03de2..7721be2ef 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -1620,15 +1620,6 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
else block
for block in blocks
]
- if tool_name == "get_run_results" and not had_build_test_packet:
- blocks = data.get("blocks")
- if isinstance(blocks, list):
- data["blocks"] = [
- {**block, "output": truncate_output(block["output"])}
- if isinstance(block, dict) and "output" in block
- else block
- for block in blocks
- ]
repair_context = data.get("authoring_repair_context")
if isinstance(repair_context, dict):
repair_context = dict(repair_context)
@@ -2185,20 +2176,3 @@ def format_tool_result_for_user(
if tool_name in _USER_FACING_EMPTY_SUCCESS_TOOLS:
return ""
return summarize_tool_result(tool_name, result, for_display=True)
-
-
-def truncate_output(output: Any, max_chars: int = 2000) -> str | None:
- if output is None:
- return None
-
- if isinstance(output, str):
- text = output
- else:
- try:
- text = json.dumps(output, default=str)
- except (TypeError, ValueError):
- text = str(output)
-
- if len(text) > max_chars:
- return text[:max_chars] + "\n... [truncated]"
- return text
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.py
index a287123ac..79cb4f513 100644
--- a/skyvern/forge/sdk/copilot/session_factory.py
+++ b/skyvern/forge/sdk/copilot/session_factory.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
is_synthetic_user_message,
log_recent_tool_output_truncation,
pending_screenshot_message,
+ unread_tool_output_indices,
)
from skyvern.forge.sdk.copilot.model_input_capture import (
clear_pending_model_input_capture,
@@ -61,9 +62,14 @@ TOOL_OUTPUT_TRUNCATE_EMERGENCY = 300
def _emergency_truncate_all(items: list[Any], cap: int) -> list[Any]:
truncated_items = [_truncate_tool_output(item, cap) for item in items]
- truncated_count = sum(1 for old, new in zip(items, truncated_items) if new is not old)
- if truncated_count:
- LOG.warning("copilot_tool_output_emergency_truncated", truncated_count=truncated_count, cap=cap)
+ truncated = {i for i, (old, new) in enumerate(zip(items, truncated_items)) if new is not old}
+ if truncated:
+ LOG.warning(
+ "copilot_tool_output_emergency_truncated",
+ truncated_count=len(truncated),
+ unread_count=len(truncated & unread_tool_output_indices(items)),
+ cap=cap,
+ )
return truncated_items
@@ -72,9 +78,41 @@ def create_copilot_session(chat_id: str) -> SQLiteSession:
return SQLiteSession(session_id=chat_id, db_path=":memory:")
+_DEFERRED_ARGUMENTS_MAX_CHARS = 2_000
+
+
+def _defer_unread_outputs_over_budget(items: list[Any], token_budget: int) -> tuple[list[Any], list[str]]:
+ """Swap the latest unread outputs for a notice that the call ran until the input fits; the first stays whole."""
+ calls = {
+ get_agent_message_field(item, "call_id"): item
+ for item in items
+ if get_agent_message_field(item, "type") == "function_call"
+ }
+ unread = sorted(unread_tool_output_indices(items))
+ items = list(items)
+ deferred: list[str] = []
+ while len(unread) > 1 and estimate_tokens(items) > token_budget:
+ index = unread.pop()
+ call = calls.get(get_agent_message_field(items[index], "call_id"))
+ tool_name = get_agent_message_field(call, "name") if call is not None else None
+ arguments = get_agent_message_field(call, "arguments") if call is not None else None
+ # The call may have changed state, so the notice reports that it ran and never asks for a repeat.
+ notice = {
+ "not_shown": "This call already ran and finished, but its result did not fit the model input with the "
+ "rest of its batch. Calling the tool again runs it again.",
+ "already_ran": True,
+ "tool_name": tool_name,
+ "arguments": arguments[:_DEFERRED_ARGUMENTS_MAX_CHARS] if isinstance(arguments, str) else None,
+ }
+ items[index] = replace_agent_message_field(items[index], "output", json.dumps(notice))
+ deferred.append(tool_name or "unknown")
+ return items, deferred
+
+
def _compact_tool_items(items: list[Any]) -> list[Any]:
return compact_agent_messages_for_llm(
items,
+ keep_whole_output_indices=unread_tool_output_indices(items),
keep_recent_tool_outputs=KEEP_RECENT_TOOL_OUTPUTS,
max_recent_tool_output_chars=_RECENT_TOOL_OUTPUT_CHAR_CAP,
summarize_tool_output=_summarize_tool_output,
@@ -312,7 +350,8 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
Graduated pruning:
1. Compact older tool outputs + function-call arguments using the
KEEP_RECENT_TOOL_OUTPUTS rule (mirrors ``enforcement._prune_input_list``).
- 2. If still over budget: drop all screenshots except the most recent.
+ 2. If still over budget: drop all screenshots except the most recent,
+ then replace the latest unread outputs that do not fit with a notice that the call ran.
3. If still over budget: truncate ALL tool outputs — first to 2000 chars,
then to 300 only if the softer pass was not enough.
4. If still over budget: aggressive prune as last resort.
@@ -349,6 +388,20 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
LOG.info("Within budget after screenshot drop", tokens=est)
return ModelInputData(input=items, instructions=instructions)
+ # Layer 2b: split an unread batch that does not fit. Outputs stay whole in call order; the rest become a
+ # notice that the call ran, so nothing the model has not read is summarized or cut.
+ items, deferred_tools = _defer_unread_outputs_over_budget(items, token_budget)
+ if deferred_tools:
+ est = estimate_tokens(items)
+ LOG.warning(
+ "copilot_unread_tool_outputs_deferred_for_budget",
+ deferred_count=len(deferred_tools),
+ tool_name=sorted(set(deferred_tools)),
+ tokens=est,
+ )
+ if est <= token_budget:
+ return ModelInputData(input=items, instructions=instructions)
+
# Layer 3a: bring every tool output down to the pre-raise bound before resorting
# to the harsher pass, so a code-bearing recent output degrades gracefully.
items = _emergency_truncate_all(items, TOOL_OUTPUT_TRUNCATE_SOFT)
@@ -367,7 +420,12 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
return ModelInputData(input=items, instructions=instructions)
# Layer 4: Aggressive prune as last resort
- LOG.warning("Aggressive prune needed", tokens=est, budget=token_budget)
+ LOG.warning(
+ "Aggressive prune needed",
+ tokens=est,
+ budget=token_budget,
+ unread_count=len(unread_tool_output_indices(items)),
+ )
items = aggressive_prune(items)
est = estimate_tokens(items)
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index c4f4a8b25..f6d51e681 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -206,6 +206,9 @@ from .page_challenge import FRESH_BROWSER_TOOL_NAME, SOLVE_TOOL_NAME, solve_page
from .page_observation import _record_composition_page_observation as _record_composition_page_observation
from .page_observation import _resolve_url_title as _resolve_url_title
from .run_execution import RUN_BLOCKS_STAGNATION_WINDOW_SECONDS as RUN_BLOCKS_STAGNATION_WINDOW_SECONDS
+from .run_execution import (
+ RUN_RESULTS_MAX_ROW_KEYS,
+)
from .run_execution import WatchdogExitReason as WatchdogExitReason
from .run_execution import _any_quiet_block_requested as _any_quiet_block_requested
from .run_execution import _attach_action_traces as _attach_action_traces
@@ -213,6 +216,7 @@ from .run_execution import _block_end_urls_by_label as _block_end_urls_by_label
from .run_execution import _cancel_run_task_if_not_final as _cancel_run_task_if_not_final
from .run_execution import (
_carry_unresolved_failure_into_result,
+ _chronological_run_block_rows,
)
from .run_execution import _composition_anti_bot_reason as _composition_anti_bot_reason
from .run_execution import _detect_non_retriable_nav_error as _detect_non_retriable_nav_error
@@ -232,6 +236,9 @@ from .run_execution import (
from .run_execution import _watchdog_error_message as _watchdog_error_message
from .run_execution import (
finalize_build_test_result,
+ parse_run_results_cursor,
+ project_run_results_page,
+ run_block_loop_facts,
run_workflow_end_to_end,
)
from .scouting import _MAX_SCOUTED_INTERACTIONS as _MAX_SCOUTED_INTERACTIONS
@@ -1217,9 +1224,22 @@ async def test_workflow_from_blank_browser_tool(
async def get_run_results_tool(
ctx: RunContextWrapper,
workflow_run_id: str | None = None,
+ block_cursor: str | None = None,
+ row_keys: list[str] | None = None,
) -> str:
"""Fetch results from a previous workflow run.
Returns block statuses, failure reasons, and output data.
+ blocks is an index with one row per block execution, oldest first, up to 20
+ rows per page. Each row has a row_key (its workflow_run_block_id, or registered:<label>
+ for an output with no block row), its loop position
+ (parent_workflow_run_block_id, current_index, current_value_preview) and the size
+ and a short preview of its output and extracted_data. total_block_rows counts every
+ row; next_block_cursor is present while rows remain, and passing it as block_cursor
+ returns the next page. Run-level fields come with the first page only.
+ row_keys (at most 25) returns block_details instead of the index: each named row's
+ complete output and extracted_data plus its action observations. Rows that do not
+ fit one call are listed in deferred_row_keys; a row too large for any call returns
+ its size, a preview and child_count, and its iterations are readable as their own rows.
If workflow_run_id is omitted, fetches the run this chat carries: its last
successful test run, else the last run it tested or was opened about. When
it carries none, fetches the most recently created finished run
@@ -1239,11 +1259,37 @@ async def get_run_results_tool(
authority_error = _authority_tool_error(copilot_ctx, "get_run_results")
if authority_error:
return json.dumps({"ok": False, "error": authority_error})
- result = await _get_run_results(params, copilot_ctx)
+ offset = 0
+ if block_cursor is not None:
+ parsed_cursor = parse_run_results_cursor(block_cursor)
+ if parsed_cursor is None:
+ return json.dumps(
+ {"ok": False, "error": f"block_cursor {block_cursor!r} is not a cursor this tool returned."}
+ )
+ cursor_run_id, offset = parsed_cursor
+ if workflow_run_id and workflow_run_id != cursor_run_id:
+ return json.dumps(
+ {"ok": False, "error": f"block_cursor pages {cursor_run_id}, not workflow_run_id {workflow_run_id}."}
+ )
+ params["workflow_run_id"] = cursor_run_id
+ if row_keys is not None:
+ row_keys = list(dict.fromkeys(row_keys))
+ if len(row_keys) > RUN_RESULTS_MAX_ROW_KEYS:
+ return json.dumps(
+ {"ok": False, "error": f"row_keys holds {len(row_keys)} keys; pass at most {RUN_RESULTS_MAX_ROW_KEYS}."}
+ )
+ first_page = block_cursor is None and row_keys is None
+ result = await _get_run_results(params, copilot_ctx, read_live_page=first_page, skip_page_evidence=not first_page)
record_tool_step_result_for_ctx(copilot_ctx, "get_run_results", params, result)
+ if result.get("ok") is not False:
+ # Exact-match scrubbing has to see whole strings, before any preview cuts or re-serializes them.
+ result = scrub_secrets_from_structure(copilot_ctx, result)
+ run_rows = await _chronological_run_block_rows(result["data"]["workflow_run_id"], copilot_ctx.organization_id)
+ loop_facts = scrub_secrets_from_structure(copilot_ctx, run_block_loop_facts(run_rows))
+ result = project_run_results_page(result, loop_facts, offset=offset, row_keys=row_keys)
sanitized = sanitize_tool_result_for_llm("get_run_results", result)
- return json.dumps(sanitized)
+ return json.dumps(scrub_secrets_from_structure(copilot_ctx, sanitized))
def _promote_executed_sources(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 93a015de8..637717260 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -133,6 +133,7 @@ from skyvern.forge.sdk.copilot.nav_attribution import (
)
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import (
+ _BASE64_IMAGE_OMITTED_MESSAGE,
_INTERNAL_GOAL_PATH_OMISSIONS_KEY,
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
@@ -4508,6 +4509,7 @@ async def _get_run_results(
*,
read_live_page: bool = True,
admit_sensitive_origin_artifact: bool = True,
+ skip_page_evidence: bool = False,
) -> dict[str, Any]:
workflow_run_id = params.get("workflow_run_id")
selected_by: RunSelectedBy = "explicit"
@@ -4637,7 +4639,11 @@ async def _get_run_results(
workflow_permanent_id=ctx.workflow_permanent_id,
)
locator_observations: list[AuthoredLocatorObservationRow] | None = None
- if not sensitive_origin_run and not _run_browser_carries_a_sign_in(ctx, run.browser_session_id):
+ if (
+ not skip_page_evidence
+ and not sensitive_origin_run
+ and not _run_browser_carries_a_sign_in(ctx, run.browser_session_id)
+ ):
failed_block_code = _failed_block_code(run_workflow, newest_failed) if run_workflow is not None else None
locator_observations = await _observe_authored_locators(
ctx,
@@ -4703,6 +4709,8 @@ async def _get_run_results(
artifact_has_redaction_context = (
artifact_redaction_registry is not None and artifact_redaction_registry.contains_all_sensitive_values
)
+ if skip_page_evidence:
+ return {"ok": True, "data": result_data}
terminal_page_evidence = (
None
if (cold_artifact_requires_redaction_context or sensitive_origin_run) and not artifact_has_redaction_context
@@ -4735,6 +4743,240 @@ async def _get_run_results(
}
+RUN_RESULTS_PAGE_ROWS = 20
+_RUN_RESULTS_PAGE_CHAR_BUDGET = 15_000
+RUN_RESULTS_MAX_ROW_KEYS = 25
+_RUN_RESULTS_ROW_PREVIEW_CHARS = 160
+_RUN_RESULTS_DETAIL_CHAR_BUDGET = 30_000
+_RUN_RESULTS_DETAIL_ROW_MAX_CHARS = 45_000
+_RUN_RESULTS_DETAIL_PREVIEW_CHARS = 2_000
+_RUN_RESULTS_REGISTERED_VALUES_MAX = 25
+_RUN_RESULTS_DETAIL_PAYLOAD_KEYS = ("output", "extracted_data")
+
+
+class RunBlockLoopFacts(TypedDict):
+ created_at: datetime
+ parent_workflow_run_block_id: str | None
+ current_index: int | None
+ current_value: str | None
+
+
+def run_results_cursor(workflow_run_id: str, offset: int) -> str:
+ return f"{workflow_run_id}:{offset}"
+
+
+def parse_run_results_cursor(cursor: str) -> tuple[str, int] | None:
+ workflow_run_id, separator, offset = cursor.rpartition(":")
+ if not separator or not workflow_run_id or not (offset.isascii() and offset.isdigit()):
+ return None
+ return workflow_run_id, int(offset)
+
+
+def run_block_loop_facts(rows: Sequence[WorkflowRunBlock]) -> dict[str, RunBlockLoopFacts]:
+ return {
+ row.workflow_run_block_id: RunBlockLoopFacts(
+ created_at=row.created_at,
+ parent_workflow_run_block_id=row.parent_workflow_run_block_id,
+ current_index=row.current_index,
+ current_value=redact_totp_runtime_values(row.current_value),
+ )
+ for row in rows
+ }
+
+
+def _serialized_run_value(value: Any) -> str:
+ return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
+
+
+def _sized_run_value(value: Any, max_chars: int) -> dict[str, Any]:
+ text = _serialized_run_value(value)
+ return {"chars": len(text), "preview": text[:max_chars]}
+
+
+def _keyed_run_result_rows(
+ blocks: Sequence[Any], loop_facts: Mapping[str, RunBlockLoopFacts]
+) -> list[tuple[str, dict[str, Any]]]:
+ """Every row under a stable key, oldest first. The repository orders by created_at alone, so ties are
+ broken by id; rows with no persisted block row (registered-output rows) follow in their given order."""
+ dated: list[tuple[datetime, str, dict[str, Any]]] = []
+ undated: list[tuple[str, dict[str, Any]]] = []
+ for row in blocks:
+ if not isinstance(row, dict):
+ continue
+ block_id = row.get("workflow_run_block_id")
+ if isinstance(block_id, str) and block_id in loop_facts:
+ dated.append((loop_facts[block_id]["created_at"], block_id, row))
+ elif isinstance(block_id, str) and block_id:
+ undated.append((block_id, row))
+ else:
+ undated.append((f"registered:{row.get('label')}", row))
+ dated.sort(key=lambda entry: (entry[0], entry[1]))
+ return [(block_id, row) for _, block_id, row in dated] + undated
+
+
+def _run_results_row_fields(row_key: str, row: Mapping[str, Any], facts: RunBlockLoopFacts | None) -> dict[str, Any]:
+ entry: dict[str, Any] = {
+ "row_key": row_key,
+ "label": row.get("label"),
+ "block_type": row.get("block_type"),
+ "status": row.get("status"),
+ }
+ if facts is not None:
+ if facts["parent_workflow_run_block_id"]:
+ entry["parent_workflow_run_block_id"] = facts["parent_workflow_run_block_id"]
+ if facts["current_index"] is not None:
+ entry["current_index"] = facts["current_index"]
+ if facts["current_value"] is not None:
+ entry["current_value_preview"] = _serialized_run_value(facts["current_value"])[
+ :_RUN_RESULTS_ROW_PREVIEW_CHARS
+ ]
+ for key in ("failure_reason", "error_codes", "final_url", "at_failure_evidence"):
+ value = row.get(key)
+ if isinstance(value, str) and len(value) > _RUN_RESULTS_DETAIL_PREVIEW_CHARS:
+ # Unbounded text ahead of the payload would push the payload itself past the recent-output cut.
+ entry[key] = value[:_RUN_RESULTS_DETAIL_PREVIEW_CHARS]
+ entry[f"{key}_chars"] = len(value)
+ elif value:
+ entry[key] = value
+ if row.get("screenshot_b64"):
+ entry["screenshot_b64"] = _BASE64_IMAGE_OMITTED_MESSAGE
+ return entry
+
+
+def _run_results_row_payload(row: Mapping[str, Any]) -> dict[str, Any]:
+ payload = {key: row[key] for key in _RUN_RESULTS_DETAIL_PAYLOAD_KEYS if row.get(key) is not None}
+ # A block's registered output value is merged in as {"<label>_output": <its output>}, a second copy. A string
+ # output is stored as {"value": s} while its registered copy is the bare string.
+ output = payload.get("output")
+ copies = [output, output["value"]] if isinstance(output, dict) and list(output) == ["value"] else [output]
+ extracted = payload.get("extracted_data")
+ if "output" in payload and isinstance(extracted, dict) and len(extracted) == 1:
+ if next(iter(extracted.values())) in copies:
+ del payload["extracted_data"]
+ return payload
+
+
+def _run_results_index_row(row_key: str, row: Mapping[str, Any], facts: RunBlockLoopFacts | None) -> dict[str, Any]:
+ entry = _run_results_row_fields(row_key, row, facts)
+ for key, value in _run_results_row_payload(row).items():
+ sized = _sized_run_value(value, _RUN_RESULTS_ROW_PREVIEW_CHARS)
+ entry[f"{key}_chars"] = sized["chars"]
+ entry[f"{key}_preview"] = sized["preview"]
+ return entry
+
+
+def _bound_registered_values(data: dict[str, Any]) -> None:
+ registered = data.get("registered_output_parameter_values")
+ if not isinstance(registered, list):
+ return
+ bounded: list[dict[str, Any]] = []
+ for item in registered[:_RUN_RESULTS_REGISTERED_VALUES_MAX]:
+ if not isinstance(item, Mapping):
+ continue
+ sized = _sized_run_value(item.get("value"), _RUN_RESULTS_ROW_PREVIEW_CHARS)
+ entry = {
+ "output_parameter_key": item.get("output_parameter_key"),
+ "block_label": item.get("block_label"),
+ "value_chars": sized["chars"],
+ }
+ # A value with a label and key is merged into that row's extracted_data and read there; any other is not.
+ if not (item.get("block_label") and item.get("output_parameter_key")):
+ entry["value_preview"] = sized["preview"]
+ bounded.append(entry)
+ data["registered_output_parameter_values"] = bounded
+ if len(registered) > _RUN_RESULTS_REGISTERED_VALUES_MAX:
+ data["registered_output_parameter_values_omitted"] = len(registered) - _RUN_RESULTS_REGISTERED_VALUES_MAX
+
+
+def project_run_results_page(
+ result: Mapping[str, Any],
+ loop_facts: Mapping[str, RunBlockLoopFacts],
+ *,
+ offset: int = 0,
+ row_keys: Sequence[str] | None = None,
+) -> dict[str, Any]:
+ """The model-facing page of a run's results: an index row per block execution, or the recorded output
+ of the rows ``row_keys`` names. Run-level facts ride on the first page only; every row left out is either
+ reachable by cursor or reported by key."""
+ data = result["data"]
+ keyed_rows = _keyed_run_result_rows(data.get("blocks") or [], loop_facts)
+ total = len(keyed_rows)
+ if offset and offset >= total:
+ return {"ok": False, "error": f"block_cursor offset {offset} is past the run's {total} block rows."}
+
+ # Paging keys lead the result, ahead of run-level fields and rows, so a head cut cannot remove the cursor.
+ page_data: dict[str, Any] = {
+ "workflow_run_id": data["workflow_run_id"],
+ "overall_status": data["overall_status"],
+ "total_block_rows": total,
+ }
+ if not WorkflowRunStatus(data["overall_status"]).is_final():
+ page_data["run_final"] = False
+
+ if row_keys is None:
+ # failure_reason is unbounded, so the page stops at a size budget as well as a row count.
+ page: list[dict[str, Any]] = []
+ page_chars = 0
+ for row_key, block_row in keyed_rows[offset : offset + RUN_RESULTS_PAGE_ROWS]:
+ entry = _run_results_index_row(row_key, block_row, loop_facts.get(row_key))
+ entry_chars = len(json.dumps(entry, default=str))
+ if page and page_chars + entry_chars > _RUN_RESULTS_PAGE_CHAR_BUDGET:
+ break
+ page.append(entry)
+ page_chars += entry_chars
+ page_data["returned_block_rows"] = len(page)
+ if offset + len(page) < total:
+ page_data["next_block_cursor"] = run_results_cursor(data["workflow_run_id"], offset + len(page))
+ if offset == 0:
+ run_fields = {key: value for key, value in data.items() if key != "blocks" and key not in page_data}
+ for key in ("requested_block_labels", "executed_block_labels"):
+ if isinstance(run_fields.get(key), list):
+ run_fields[key] = list(dict.fromkeys(run_fields[key]))
+ _bound_registered_values(run_fields)
+ page_data.update(run_fields)
+ page_data["blocks"] = page
+ return {**result, "data": page_data}
+
+ rows_by_key = dict(keyed_rows)
+ details: list[dict[str, Any]] = []
+ unknown: list[str] = []
+ deferred: list[dict[str, Any]] = []
+ remaining = _RUN_RESULTS_DETAIL_CHAR_BUDGET
+ for row_key in row_keys:
+ row = rows_by_key.get(row_key)
+ if row is None:
+ unknown.append(row_key)
+ continue
+ entry = _run_results_row_fields(row_key, row, loop_facts.get(row_key))
+ entry["action_observations"] = _retained_action_observations([row])
+ payload = _run_results_row_payload(row)
+ # Sized as the tool serializes it (ensure_ascii), so escaped text cannot slip past the budget.
+ chars = len(json.dumps({**entry, **payload}, default=str))
+ if chars > _RUN_RESULTS_DETAIL_ROW_MAX_CHARS:
+ # ponytail: a non-loop row over the budget is only readable as a preview; add content paging if one matters.
+ entry.update(
+ {key: _sized_run_value(value, _RUN_RESULTS_DETAIL_PREVIEW_CHARS) for key, value in payload.items()}
+ )
+ entry["child_count"] = sum(
+ 1 for facts in loop_facts.values() if facts["parent_workflow_run_block_id"] == row_key
+ )
+ chars = len(json.dumps(entry, default=str))
+ else:
+ entry.update(payload)
+ if details and chars > remaining:
+ deferred.append({"row_key": row_key, "chars": chars})
+ continue
+ remaining -= chars
+ details.append(entry)
+
+ if unknown:
+ page_data["unknown_row_keys"] = unknown
+ if deferred:
+ page_data["deferred_row_keys"] = deferred
+ page_data["block_details"] = details
+ return {**result, "data": page_data}
+
+
def _composition_anti_bot_reason(copilot_ctx: object) -> str | None:
evidence = getattr(copilot_ctx, "composition_page_evidence", None)
if not isinstance(evidence, dict):
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 84488f261..847f49dc7 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -643,6 +643,10 @@ def _fc(call_id: str) -> dict[str, str]:
return {"type": "function_call", "call_id": call_id, "name": "evaluate", "arguments": "{}"}
+def _reasoning() -> dict[str, Any]:
+ return {"type": "reasoning", "summary": []}
+
+
def _history_item(fields: dict[str, Any], *, attr_style: bool) -> dict[str, Any] | SimpleNamespace:
return SimpleNamespace(**fields) if attr_style else fields
@@ -887,7 +891,9 @@ def test_recent_code_sized_output_survives_untruncated() -> None:
def test_old_code_output_synopsis_names_elided_code_size() -> None:
code = "await page.click()\n" * 300
old_output = json.dumps({"ok": True, "data": {"code": code}})
- items = [_fco("c_old", old_output)] + [_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)]
+ items = [_fco("c_old", old_output), _reasoning()] + [
+ _fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)
+ ]
pruned = _prune_input_list(items)
synopsis = json.loads(pruned[0]["output"])
@@ -897,7 +903,7 @@ def test_old_code_output_synopsis_names_elided_code_size() -> None:
def _prune_twice(old_output: str) -> tuple[dict[str, Any], dict[str, Any]]:
"""Session continuation re-compacts already-summarized outputs; both passes must agree."""
filler = [_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)]
- first = _prune_input_list([_fco("c_old", old_output)] + filler)
+ first = _prune_input_list([_fco("c_old", old_output), _reasoning()] + filler)
second = _prune_input_list(first)
return json.loads(first[0]["output"]), json.loads(second[0]["output"])
@@ -972,7 +978,9 @@ def test_an_evicted_build_test_packet_keeps_its_challenge_facts_and_levers() ->
},
}
)
- items = [_fco("c_old", old_output)] + [_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)]
+ items = [_fco("c_old", old_output), _reasoning()] + [
+ _fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)
+ ]
pruned = _prune_input_list(items)
@@ -999,7 +1007,9 @@ def test_an_evicted_frame_only_packet_gains_no_solver_facts_on_the_way_through()
},
}
)
- items = [_fco("c_old", old_output)] + [_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)]
+ items = [_fco("c_old", old_output), _reasoning()] + [
+ _fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)
+ ]
pruned = _prune_input_list(items)
@@ -1029,7 +1039,9 @@ def test_an_evicted_page_evidence_summary_keeps_the_author_time_levers() -> None
}
)
assert len(old_output) > _TOOL_OUTPUT_SUMMARIZE_THRESHOLD, "a short payload is returned byte-identical"
- items = [_fco("c_old", old_output)] + [_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)]
+ items = [_fco("c_old", old_output), _reasoning()] + [
+ _fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)
+ ]
pruned = _prune_input_list(items)
@@ -1068,7 +1080,7 @@ def test_old_large_output_is_summarized() -> None:
heavy_output = json.dumps(heavy_payload)
assert len(heavy_output) > 4000
- items = [_fco("c_old", heavy_output)]
+ items = [_fco("c_old", heavy_output), _reasoning()]
# Add enough recent outputs to push the first one out of the recent window.
for i in range(KEEP_RECENT_TOOL_OUTPUTS):
items.append(_fco(f"c_new_{i}", '{"ok":true,"data":{"overall_status":"completed"}}'))
@@ -1116,6 +1128,74 @@ def test_recent_large_output_is_head_truncated_not_summarized() -> None:
assert any(entry["event"] == "copilot_recent_tool_output_truncated" for entry in logs)
+def _run_results_call(call_id: str) -> dict[str, str]:
+ return {"type": "function_call", "call_id": call_id, "name": "get_run_results", "arguments": "{}"}
+
+
+def _run_results_output(call_id: str) -> dict[str, str]:
+ blocks = [{"label": f"row_{index}", "status": "completed", "output_preview": "v" * 80} for index in range(20)]
+ return _fco(call_id, json.dumps({"ok": True, "data": {"workflow_run_id": call_id, "blocks": blocks}}))
+
+
+def _batch(order: str) -> list[dict[str, Any]]:
+ call_ids = [f"call_{index}" for index in range(5)]
+ if order == "calls_then_outputs":
+ return [_run_results_call(cid) for cid in call_ids] + [_run_results_output(cid) for cid in call_ids]
+ assert order == "one_call_per_response"
+ return [item for cid in call_ids for item in (_run_results_call(cid), _run_results_output(cid))]
+
+
[email protected]("trailing", [[], [{"role": "user", "content": "[copilot:nudge] keep going"}]])
+def test_a_parallel_batch_the_model_has_not_read_is_never_summarized(trailing: list[dict[str, Any]]) -> None:
+ items = [{"role": "user", "content": "goal"}, _reasoning(), *_batch("calls_then_outputs"), *trailing]
+ outputs = [item["output"] for item in items if item.get("type") == "function_call_output"]
+ assert len(outputs) > KEEP_RECENT_TOOL_OUTPUTS
+ assert all(len(output) > _TOOL_OUTPUT_SUMMARIZE_THRESHOLD for output in outputs)
+
+ pruned = _prune_input_list(items)
+
+ assert [item["output"] for item in pruned if item.get("type") == "function_call_output"] == outputs
+
+
+ "items",
+ [
+ [_reasoning(), *_batch("calls_then_outputs"), _reasoning()],
+ # One call per response with no reasoning between, as in a page-by-page walk: each later call is a read.
+ [_reasoning(), *_batch("one_call_per_response")],
+ ],
+)
+def test_outputs_the_model_has_read_go_back_through_the_recent_window(items: list[dict[str, Any]]) -> None:
+ pruned = _prune_input_list([{"role": "user", "content": "goal"}, *items])
+
+ outputs = [item["output"] for item in pruned if item.get("type") == "function_call_output"]
+ assert ["_summarized" in output for output in outputs] == [True, True, False, False, False]
+
+
+def test_a_summarized_run_results_page_keeps_each_rows_key_and_iteration() -> None:
+ row = {"row_key": "wrb_7", "label": "extract_item", "status": "completed", "current_index": 13}
+ page = {"ok": True, "data": {"workflow_run_id": "wr_1", "blocks": [{**row, "output_preview": "v" * 400}]}}
+ detail = {"ok": True, "data": {"workflow_run_id": "wr_1", "block_details": [{**row, "output": {"v": "v" * 400}}]}}
+
+ for payload, key in ((page, "blocks"), (detail, "block_details")):
+ synopsis = json.loads(_summarize_tool_output(json.dumps(payload)))
+ assert synopsis[key] == [
+ {"label": "extract_item", "status": "completed", "row_key": "wrb_7", "current_index": 13}
+ ]
+
+
+def test_an_unread_output_over_the_cap_is_still_head_cut_and_logged_with_its_tool() -> None:
+ large = '{"ok":true,"data":{"value":"' + ("y" * (_RECENT_TOOL_OUTPUT_CHAR_CAP + 1000)) + '"}}'
+ items = [_reasoning(), _run_results_call("c_big"), _fco("c_big", large)]
+
+ with capture_logs() as logs:
+ pruned = _prune_input_list(items)
+
+ assert pruned[2]["output"].endswith("\n... [truncated]")
+ event = next(entry for entry in logs if entry["event"] == "copilot_recent_tool_output_truncated")
+ assert event["tool_name"] == ["get_run_results"]
+
+
LISTING_DETAIL_URL = "http://localhost:8901/record/1457803926"
# Generic multi-field detail DOM: exercises the contract's label/header binding vs the
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index 192c83410..dd21df8e0 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -1,4 +1,4 @@
-"""Tests for truncate_output and sanitize_tool_result_for_llm."""
+"""Tests for sanitize_tool_result_for_llm and tool-result summaries."""
from __future__ import annotations
@@ -37,42 +37,10 @@ from skyvern.forge.sdk.copilot.output_utils import (
screened_recorded_url,
summarize_tool_result,
summarize_tool_result_detail,
- truncate_output,
user_facing_success,
)
-def test_truncate_output_none() -> None:
- assert truncate_output(None) is None
-
-
-def test_truncate_output_short_string() -> None:
- assert truncate_output("ok") == "ok"
-
-
-def test_truncate_output_long_string_truncates() -> None:
- text = "x" * 2100
- result = truncate_output(text, max_chars=2000)
-
- assert result is not None
- assert result.startswith("x" * 2000)
- assert result.endswith("\n... [truncated]")
-
-
-def test_truncate_output_serializes_dict() -> None:
- result = truncate_output({"a": 1, "b": True})
- assert result == '{"a": 1, "b": true}'
-
-
-def test_truncate_output_falls_back_to_str_on_json_error() -> None:
- circular: dict[str, object] = {}
- circular["self"] = circular
-
- result = truncate_output(circular)
- assert result is not None
- assert "self" in result
-
-
def test_sanitize_get_run_results_scrubs_nested_block_screenshots() -> None:
result = {
"ok": True,
@@ -104,22 +72,6 @@ def test_sanitize_get_run_results_scrubs_nested_block_screenshots() -> None:
assert blocks[0]["status"] == "completed"
-def test_sanitize_get_run_results_bounds_recorded_block_output_without_packet() -> None:
- recorded_output = {"service_name": "a" * 2200}
- result = {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_123",
- "blocks": [{"label": "collect_service", "output": recorded_output}],
- },
- }
-
- sanitized = sanitize_tool_result_for_llm("get_run_results", result)
-
- assert sanitized["data"]["blocks"][0]["output"].endswith("\n... [truncated]")
- assert result["data"]["blocks"][0]["output"] is recorded_output
-
-
def test_sanitize_does_not_mutate_original_blocks() -> None:
original_screenshot = "iVBORw0KGgo" + "B" * 500
result = {
diff --git a/tests/unit/test_copilot_repair_origin_run.py b/tests/unit/test_copilot_repair_origin_run.py
index 9ea8ff87e..286ca1aa2 100644
--- a/tests/unit/test_copilot_repair_origin_run.py
+++ b/tests/unit/test_copilot_repair_origin_run.py
@@ -26,7 +26,7 @@ from skyvern.forge.sdk.copilot.repair_origin_run import (
resolve_repair_origin_binding,
seed_repair_origin_run,
)
-from skyvern.forge.sdk.copilot.tools import _record_run_blocks_result, run_execution
+from skyvern.forge.sdk.copilot.tools import _record_run_blocks_result, get_run_results_tool, run_execution
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.db.models import WorkflowModel, WorkflowRunModel
@@ -38,6 +38,7 @@ from tests.unit.copilot_test_helpers import (
harness_run,
install_get_run_results_harness,
origin_run_input,
+ run_result_block_row,
stub_copilot_agent_loop,
)
@@ -605,3 +606,230 @@ async def test_the_history_query_lists_only_newer_finished_runs_of_this_workflow
)
assert [entry["workflow_run_id"] for entry in facts["newer_finished_runs"]] == ["wr_newest", "wr_scheduled"]
+
+
+PAGED_SECRET = "tok-9f8e7d6c5b4a"
+LOOP_ITERATIONS = 29
+
+
+def _paged_run_rows() -> list[list[MagicMock]]:
+ """A for-loop row and two child rows per iteration whose created_at ties, in the two orders the
+ repository may return tied rows in; each list is newest first, as the repository returns it."""
+
+ def row(block_id: str, label: str, block_type: str, created_at: datetime, output: object) -> MagicMock:
+ block = run_result_block_row(label, "completed")
+ block.workflow_run_block_id = block_id
+ block.block_type = SimpleNamespace(name=block_type)
+ block.created_at = created_at
+ block.output = output
+ block.parent_workflow_run_block_id = None
+ block.current_index = None
+ block.current_value = None
+ return block
+
+ loop = row("wrb_loop", "each_item", "FOR_LOOP", HARNESS_RUN_CREATED_AT, [{"item": "x" * 1000}] * 60)
+ pairs: list[tuple[MagicMock, MagicMock]] = []
+ for index in range(LOOP_ITERATIONS):
+ at = HARNESS_RUN_CREATED_AT + timedelta(seconds=index + 1)
+ opened = row(f"wrb_open_{index:02d}", "open_item", "NAVIGATION", at, "p" * 150 + PAGED_SECRET + " tail")
+ opened.failure_reason = f"retried item {index}: " + "r" * 6000
+ extracted = row(
+ f"wrb_extract_{index:02d}",
+ "extract_item",
+ "EXTRACTION",
+ at,
+ {"extracted_information": {"name": f"item {index}", "detail": "d" * 5000}},
+ )
+ for child in (opened, extracted):
+ child.parent_workflow_run_block_id = "wrb_loop"
+ child.current_index = index
+ child.current_value = json.dumps({"name": f"item {index}"})
+ pairs.append((opened, extracted))
+ oldest_first = [loop, *(child for pair in pairs for child in pair)]
+ swapped = [loop, *(child for pair in pairs for child in reversed(pair))]
+ return [list(reversed(oldest_first)), list(reversed(swapped))]
+
+
+def _install_paged_run(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
+ orders = _paged_run_rows()
+ ctx = install_get_run_results_harness(monkeypatch, blocks=orders[0], run_status="completed")
+ reads = iter(range(1_000))
+ run_execution.app.DATABASE.observer.get_workflow_run_blocks.side_effect = lambda **_: orders[next(reads) // 2 % 2]
+
+ async def attach_registered(*, workflow_run_id: str, data: dict[str, Any], **_: object) -> dict[str, Any]:
+ data["registered_output_parameter_values"] = [
+ {"output_parameter_key": "report_output", "block_label": "report", "value": {"note": PAGED_SECRET}}
+ ]
+ run_execution._merge_registered_output_parameter_values_into_blocks(data)
+ return {}
+
+ monkeypatch.setattr(run_execution, "_attach_registered_output_parameter_values", attach_registered)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._authority_tool_error", lambda *_args: None)
+ ctx.secret_scrub_values = [PAGED_SECRET]
+ return ctx
+
+
+async def _get_run_results_page(ctx: SimpleNamespace, **arguments: object) -> tuple[str, dict[str, Any]]:
+ raw = await get_run_results_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps(arguments)
+ )
+ return raw, json.loads(raw)
+
+
+async def test_following_the_cursor_returns_every_row_once_in_a_stable_order(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _install_paged_run(monkeypatch)
+
+ first_raw, first = await _get_run_results_page(ctx, workflow_run_id="wr-1")
+ page = first
+ raws = [first_raw]
+ keys = [row["row_key"] for row in first["data"]["blocks"]]
+ while cursor := page["data"].get("next_block_cursor"):
+ raw, page = await _get_run_results_page(ctx, block_cursor=cursor)
+ raws.append(raw)
+ keys.extend(row["row_key"] for row in page["data"]["blocks"])
+
+ expected = ["wrb_loop"]
+ for index in range(LOOP_ITERATIONS):
+ expected += [f"wrb_extract_{index:02d}", f"wrb_open_{index:02d}"]
+ assert keys == [*expected, "registered:report"]
+ assert first["data"]["total_block_rows"] == len(keys) == 60
+ assert len(first_raw) < 50_000
+ child = next(row for row in first["data"]["blocks"] if row["row_key"] == "wrb_open_00")
+ assert child["parent_workflow_run_block_id"] == "wrb_loop"
+ assert child["current_index"] == 0
+ assert child["current_value_preview"] == json.dumps({"name": "item 0"})
+ assert child["output_chars"] > len(child["output_preview"])
+ assert all(PAGED_SECRET[:8] not in raw for raw in raws)
+
+
+async def test_row_keys_return_full_output_and_account_for_every_row_they_cannot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _install_paged_run(monkeypatch)
+ requested = [
+ "wrb_loop",
+ *(f"wrb_extract_{index:02d}" for index in range(8)),
+ *(f"wrb_open_{index:02d}" for index in range(12)),
+ "wrb_missing",
+ ]
+
+ raw, page = await _get_run_results_page(ctx, workflow_run_id="wr-1", row_keys=requested)
+
+ assert len(raw) < 50_000
+ data = page["data"]
+ details = {row["row_key"]: row for row in data["block_details"]}
+ deferred = [row["row_key"] for row in data["deferred_row_keys"]]
+ assert data["unknown_row_keys"] == ["wrb_missing"]
+ assert sorted([*details, *deferred]) == sorted(requested[:-1])
+ assert deferred
+ loop = details["wrb_loop"]
+ assert loop["child_count"] == 2 * LOOP_ITERATIONS
+ assert set(loop["output"]) == {"chars", "preview"}
+ assert details["wrb_extract_00"]["output"] == {"extracted_information": {"name": "item 0", "detail": "d" * 5000}}
+ assert all("action_observations" in row for row in details.values())
+
+
+ "arguments",
+ [
+ {"workflow_run_id": "wr-1", "block_cursor": "wr-other:20"},
+ {"block_cursor": "not-a-cursor"},
+ {"block_cursor": "wr-1:500"},
+ {"workflow_run_id": "wr-1", "row_keys": [f"wrb_{index}" for index in range(26)]},
+ ],
+)
+async def test_a_foreign_or_unusable_page_request_is_refused(
+ monkeypatch: pytest.MonkeyPatch, arguments: dict[str, object]
+) -> None:
+ ctx = _install_paged_run(monkeypatch)
+
+ _, page = await _get_run_results_page(ctx, **arguments)
+
+ assert page["ok"] is False
+
+
+ "big_output,registered_value",
+ [({"text": "x" * 35_000}, {"text": "x" * 35_000}), ({"value": "x" * 35_000}, "x" * 35_000)],
+)
+async def test_a_large_single_output_read_by_key_comes_back_whole(
+ monkeypatch: pytest.MonkeyPatch, big_output: dict[str, str], registered_value: object
+) -> None:
+ block = run_result_block_row("big", "completed")
+ block.created_at = HARNESS_RUN_CREATED_AT
+ block.output = big_output
+ block.parent_workflow_run_block_id = None
+ block.current_index = None
+ block.current_value = None
+ ctx = install_get_run_results_harness(monkeypatch, blocks=[block], run_status="completed")
+
+ async def attach_registered(*, workflow_run_id: str, data: dict[str, Any], **_: object) -> dict[str, Any]:
+ data["registered_output_parameter_values"] = [
+ {"output_parameter_key": "big_output", "block_label": "big", "value": registered_value}
+ ]
+ run_execution._merge_registered_output_parameter_values_into_blocks(data)
+ return {}
+
+ monkeypatch.setattr(run_execution, "_attach_registered_output_parameter_values", attach_registered)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._authority_tool_error", lambda *_args: None)
+ ctx.secret_scrub_values = []
+
+ raw, page = await _get_run_results_page(ctx, workflow_run_id="wr-1", row_keys=["wrb_big"])
+
+ assert len(raw) < 50_000
+ assert page["data"]["block_details"][0]["output"] == big_output
+
+
+def test_a_row_with_a_huge_failure_reason_still_returns_its_output_inside_the_cut() -> None:
+ block = {
+ "workflow_run_block_id": "wrb_big",
+ "label": "step",
+ "status": "failed",
+ "failure_reason": "f" * 60_000,
+ "output": {"answer": "the recorded output"},
+ }
+ facts: dict[str, Any] = {
+ "wrb_big": {
+ "created_at": HARNESS_RUN_CREATED_AT,
+ "parent_workflow_run_block_id": None,
+ "current_index": None,
+ "current_value": None,
+ }
+ }
+ result = {"ok": True, "data": {"workflow_run_id": "wr-1", "overall_status": "failed", "blocks": [block]}}
+
+ page = run_execution.project_run_results_page(result, facts, row_keys=["wrb_big"])
+
+ row = page["data"]["block_details"][0]
+ assert row["failure_reason_chars"] == 60_000
+ assert '"the recorded output"' in json.dumps(page)[:50_000]
+
+
+def test_the_first_page_cursor_survives_a_head_cut_behind_a_large_run_field() -> None:
+ blocks = [{"workflow_run_block_id": f"wrb_{i:02d}", "label": "step", "status": "completed"} for i in range(30)]
+ facts: dict[str, Any] = {
+ block["workflow_run_block_id"]: {
+ "created_at": HARNESS_RUN_CREATED_AT + timedelta(seconds=i),
+ "parent_workflow_run_block_id": None,
+ "current_index": None,
+ "current_value": None,
+ }
+ for i, block in enumerate(blocks)
+ }
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr-1",
+ "overall_status": "failed",
+ "failure_reason": "f" * 55_000,
+ "blocks": blocks,
+ },
+ }
+
+ page = run_execution.project_run_results_page(result, facts)
+
+ assert '"next_block_cursor": "wr-1:20"' in json.dumps(page)[:50_000]
diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.py
index 55c6f3504..f4875c8a1 100644
--- a/tests/unit/test_copilot_screenshot_handling.py
+++ b/tests/unit/test_copilot_screenshot_handling.py
@@ -4,6 +4,8 @@ from __future__ import annotations
import base64
import io
+import json
+from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -1137,6 +1139,37 @@ class TestAttachFailedBlockScreenshots:
)
assert _resolve_run_screenshot_b64(live_capture=None, results=result["data"]["blocks"], run_ok=False) is None
+ @pytest.mark.asyncio
+ async def test_a_failed_row_read_by_key_keeps_its_at_failure_facts_without_the_image_bytes(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import skyvern.forge.sdk.copilot.tools.run_execution as run_execution_module
+ from skyvern.forge.sdk.copilot.tools import get_run_results_tool
+
+ block = run_result_block_row("open_item", "failed", "https://example.com/item/7")
+ block.created_at = datetime(2026, 9, 1, tzinfo=timezone.utc)
+ block.parent_workflow_run_block_id = None
+ block.current_index = None
+ block.current_value = None
+ ctx = install_get_run_results_harness(
+ monkeypatch,
+ blocks=[block],
+ attach_failed_block_screenshots=run_execution_module._attach_failed_block_screenshots,
+ )
+ ctx.secret_scrub_values = []
+ self._add_artifact_store(run_block_artifact=MagicMock())
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._authority_tool_error", lambda *_args: None)
+
+ raw = await get_run_results_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="get_run_results"),
+ json.dumps({"workflow_run_id": "wr-1", "row_keys": ["wrb_open_item"]}),
+ )
+
+ row = json.loads(raw)["data"]["block_details"][0]
+ assert row["final_url"] == "https://example.com/item/7"
+ assert row["screenshot_b64"] == "[base64 image omitted — screenshot was taken successfully]"
+ assert base64.b64encode(self.PNG_BYTES).decode("utf-8") not in raw
+
@pytest.mark.asyncio
async def test_get_run_results_wires_the_persisted_end_url_into_the_packet(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index c42b90fe7..8ab03774c 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.ask_user import (
QuestionResponse,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.enforcement import estimate_tokens
from skyvern.forge.sdk.copilot.screenshot_utils import (
ScreenshotActionRelation,
ScreenshotEntry,
@@ -26,12 +27,29 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
)
from skyvern.forge.sdk.copilot.session_factory import (
copilot_call_model_input_filter,
+ copilot_session_input_callback,
make_copilot_call_model_input_filter,
)
from tests.unit.copilot_test_helpers import make_copilot_ctx
from tests.unit.copilot_test_helpers import make_model_input_data as _mk_input_data
+def _unread_run_results_batch() -> list[dict[str, Any]]:
+ call_ids = [f"call_{index}" for index in range(5)]
+ calls = [
+ {"type": "function_call", "call_id": cid, "name": "get_run_results", "arguments": "{}"} for cid in call_ids
+ ]
+ outputs = [
+ {
+ "type": "function_call_output",
+ "call_id": cid,
+ "output": json.dumps({"ok": True, "data": {"workflow_run_id": cid, "blob": "z" * 3000}}),
+ }
+ for cid in call_ids
+ ]
+ return [{"type": "reasoning", "summary": []}, *calls, *outputs]
+
+
def _image_parts(item: Any) -> list[Any]:
content = item.get("content") if isinstance(item, dict) else None
if not isinstance(content, list):
@@ -77,12 +95,16 @@ class TestFirstTurnCompaction:
{"role": "user", "content": "please build me a workflow"},
# six function_call_output items; the last 3 stay raw, older 3 compact.
*(
- {
- "type": "function_call_output",
- "call_id": f"call-{i}",
- "output": json.dumps({"ok": True, "data": {"blob": large_output}}),
- }
+ item
for i in range(6)
+ for item in (
+ {
+ "type": "function_call_output",
+ "call_id": f"call-{i}",
+ "output": json.dumps({"ok": True, "data": {"blob": large_output}}),
+ },
+ {"type": "reasoning", "summary": []},
+ )
),
]
result = copilot_call_model_input_filter(_mk_input_data(items))
@@ -95,6 +117,36 @@ class TestFirstTurnCompaction:
for recent in recent_three:
assert small_summary_marker not in recent["output"]
+ def test_filter_keeps_every_unread_output_of_a_parallel_batch_whole(self) -> None:
+ batch = _unread_run_results_batch()
+ items = [{"role": "user", "content": "please build me a workflow"}, *batch]
+
+ with capture_logs() as logs:
+ result = copilot_call_model_input_filter(_mk_input_data(items))
+
+ sent = {it["call_id"]: it["output"] for it in result.input if it.get("type") == "function_call_output"}
+ assert sent == {it["call_id"]: it["output"] for it in batch if it.get("type") == "function_call_output"}
+ assert not [entry for entry in logs if "truncat" in entry["event"]]
+
+ def test_an_unread_batch_over_budget_is_split_not_summarized(self) -> None:
+ from skyvern.forge.sdk.copilot.session_factory import make_copilot_call_model_input_filter
+
+ batch = _unread_run_results_batch()
+ items = [{"role": "user", "content": "please build me a workflow"}, *batch]
+ outputs = {it["call_id"]: it["output"] for it in batch if it.get("type") == "function_call_output"}
+ budget = estimate_tokens(items) - 100
+
+ result = make_copilot_call_model_input_filter(token_budget=budget)(_mk_input_data(items))
+
+ sent = {it["call_id"]: it["output"] for it in result.input if it.get("type") == "function_call_output"}
+ whole = [cid for cid in outputs if sent[cid] == outputs[cid]]
+ deferred = [cid for cid in outputs if cid not in whole]
+ assert whole[0] == "call_0" and deferred
+ for cid in deferred:
+ notice = json.loads(sent[cid])
+ assert notice["tool_name"] == "get_run_results" and notice["arguments"] == "{}"
+ assert notice["already_ran"] is True
+
def test_recent_code_sized_output_survives_session_compaction(self) -> None:
from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
@@ -118,13 +170,15 @@ class TestFirstTurnCompaction:
oversized = "x" * (_RECENT_TOOL_OUTPUT_CHAR_CAP + 1000)
items: list[dict[str, Any]] = [
{"role": "user", "content": "please build me a workflow"},
+ {"type": "function_call", "call_id": "call-big", "name": "get_run_results", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call-big", "output": oversized},
]
with structlog.testing.capture_logs() as logs:
result = copilot_call_model_input_filter(_mk_input_data(items))
outputs = [it for it in result.input if it.get("type") == "function_call_output"]
assert outputs[0]["output"].endswith("... [truncated]")
- assert any(entry["event"] == "copilot_recent_tool_output_truncated" for entry in logs)
+ truncated = [entry for entry in logs if entry["event"] == "copilot_recent_tool_output_truncated"]
+ assert [entry["tool_name"] for entry in truncated] == [["get_run_results"]]
def test_emergency_truncation_logs_distinct_event_with_count(self) -> None:
import structlog.testing
@@ -135,6 +189,7 @@ class TestFirstTurnCompaction:
{"role": "user", "content": "please build me a workflow"},
*({"type": "function_call_output", "call_id": f"call-{i}", "output": "x" * 5000} for i in range(3)),
{"type": "function_call_output", "call_id": "call-small", "output": "ok"},
+ {"type": "reasoning", "summary": []},
]
tight_filter = make_copilot_call_model_input_filter(token_budget=200)
with structlog.testing.capture_logs() as logs:
@@ -289,12 +344,16 @@ class TestSessionInputCallback:
goal = {"role": "user", "content": "please build me a workflow"}
tool_items = [
- {
- "type": "function_call_output",
- "call_id": f"c-{i}",
- "output": json.dumps({"ok": True, "data": {"blob": "y" * 4000}}),
- }
+ item
for i in range(5)
+ for item in (
+ {
+ "type": "function_call_output",
+ "call_id": f"c-{i}",
+ "output": json.dumps({"ok": True, "data": {"blob": "y" * 4000}}),
+ },
+ {"type": "reasoning", "summary": []},
+ )
]
new = [{"role": "user", "content": "[copilot:nudge] please finish"}]
@@ -308,6 +367,16 @@ class TestSessionInputCallback:
for recent in tool_outputs_in_combined[2:]:
assert "_summarized" not in recent["output"]
+ def test_unread_batch_at_the_end_of_the_middle_is_not_summarized(self) -> None:
+ goal = {"role": "user", "content": "please build me a workflow"}
+ batch = _unread_run_results_batch()
+ nudge = [{"role": "user", "content": "[copilot:nudge] please finish"}]
+
+ combined = copilot_session_input_callback([goal, *batch], nudge)
+
+ outputs = [it["output"] for it in combined if it.get("type") == "function_call_output"]
+ assert outputs == [it["output"] for it in batch if it.get("type") == "function_call_output"]
+
def test_no_duplication_when_boundary_equals_one(self) -> None:
"""Regression guard: when ``_find_real_user_boundary`` returns 1, the
earlier partitioning logic emitted ``history_items[1:]`` in both the