Skip to content

Automate browser based workflows with AI

Copilot: keep the current screenshot when the token budget's last-resort prune runs (SKY-18319) (#8956)

Andrew Neilsonedafdaa145c1
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.py
index 23866135f..614be9dfd 100644
--- a/skyvern/forge/sdk/copilot/session_factory.py
+++ b/skyvern/forge/sdk/copilot/session_factory.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.agents.context import (
replace_agent_message_field,
)
from skyvern.forge.sdk.copilot.browser_ablation import prompt_sha256
+from skyvern.forge.sdk.copilot.config import SCREENSHOT_DROPPED_NUDGE
from skyvern.forge.sdk.copilot.enforcement import (
_RECENT_TOOL_OUTPUT_CHAR_CAP,
_TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX,
@@ -368,7 +369,8 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
then replace the latest unread outputs that do not fit with a notice that the call ran.
3. If still over budget: truncate ALL tool outputs — first to 2000 chars,
then to 300 only if the softer pass was not enough.
- 4. If still over budget: aggressive prune as last resort.
+ 4. If still over budget: aggressive prune as last resort, keeping the most recent screenshot unless
+ it predates every step the prune keeps.
"""
if not items:
return ModelInputData(input=items, instructions=instructions)
@@ -441,7 +443,27 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
and call_id.startswith("call_demo_")
and get_agent_message_field(item, "type") == "function_call"
}
- items = aggressive_prune(items)
+ frame = next((item for item in reversed(items[1:]) if _screenshot_with_image(item)), None)
+ pruned = aggressive_prune(items)
+ # The prune drops every screenshot and appends a re-take note, or keeps an earlier note instead of adding one.
+ # A frame newer than the oldest kept step goes back in its input position without any note older than it; an
+ # older frame stays dropped because the steps taken after it were cut.
+ position = {id(item): index for index, item in enumerate(items)}
+ oldest_kept = next((position[id(item)] for item in pruned[1:] if id(item) in position), None)
+ if frame is not None and (oldest_kept is None or position[id(frame)] > oldest_kept):
+ retake_note = NUDGE_SENTINEL + SCREENSHOT_DROPPED_NUDGE
+ pruned = sorted(
+ (
+ item
+ for item in [*pruned, frame]
+ if id(item) in position
+ and (
+ position[id(item)] > position[id(frame)] or get_agent_message_field(item, "content") != retake_note
+ )
+ ),
+ key=lambda item: position[id(item)],
+ )
+ items = pruned
retained_demonstration_call_ids = {
call_id
for item in items
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index ea21707a7..c505fef12 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -19,8 +19,8 @@ from skyvern.forge.sdk.copilot.ask_user import (
QuestionPart,
QuestionResponse,
)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.enforcement import estimate_tokens, pending_screenshot_message
+from skyvern.forge.sdk.copilot.config import SCREENSHOT_DROPPED_NUDGE, BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, estimate_tokens, pending_screenshot_message
from skyvern.forge.sdk.copilot.screenshot_utils import (
ScreenshotActionRelation,
ScreenshotEntry,
@@ -775,16 +775,33 @@ class TestStagedScreenshotBinding:
assert not any(_image_parts(item) for item in result.input)
assert not any(log["event"] == "Injecting screenshot user message" for log in logs)
- def test_aggressive_prune_drops_the_bound_frame_like_any_other_screenshot(self) -> None:
- ctx = _ctx_with_staged_frame()
- items: list[dict[str, Any]] = [{"role": "user", "content": "build me a workflow"}]
+ def test_last_resort_prune_keeps_the_bound_frame_and_asks_for_a_retake_only_without_one(self) -> None:
+ retake_note = NUDGE_SENTINEL + SCREENSHOT_DROPPED_NUDGE
+ frame_older_than_the_kept_steps = pending_screenshot_message(
+ SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)
+ )
+ items: list[dict[str, Any]] = [
+ {"role": "user", "content": "build me a workflow"},
+ frame_older_than_the_kept_steps,
+ ]
for i in range(12):
items.append({"type": "function_call", "call_id": f"call-{i}", "name": "observe", "arguments": "{}"})
items.append({"type": "function_call_output", "call_id": f"call-{i}", "output": "y" * 4000})
+ retake_note_from_an_earlier_recovery = [*items[:-2], {"role": "user", "content": retake_note}, *items[-2:]]
+ over_budget_after_every_trim = make_copilot_call_model_input_filter(1)
- result = make_copilot_call_model_input_filter(1)(_mk_input_data(items, context=ctx))
+ with_frame = over_budget_after_every_trim(
+ _mk_input_data(retake_note_from_an_earlier_recovery, context=_ctx_with_staged_frame())
+ )
+ without_frame = over_budget_after_every_trim(
+ _mk_input_data(items, context=SimpleNamespace(pending_screenshots=[]))
+ )
- assert not any(_image_parts(item) for item in result.input)
+ assert [len(_image_parts(item)) for item in with_frame.input if _image_parts(item)] == [1]
+ assert len(_image_parts(with_frame.input[-1])) == 1
+ assert not any(item.get("content") == retake_note for item in with_frame.input)
+ assert not any(_image_parts(item) for item in without_frame.input)
+ assert [item.get("content") for item in without_frame.input].count(retake_note) == 1
def test_aggressive_prune_reports_omitted_demonstration_fact_packets(self) -> None:
items: list[dict[str, Any]] = [{"role": "user", "content": "build the demonstrated workflow"}]