Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: keep the current screenshot when the token budget's last-resort prune runs (SKY-18319) (#8956)
Andrew Neilsonedafdaa145c1
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.pyindex 23866135f..614be9dfd 100644--- a/skyvern/forge/sdk/copilot/session_factory.py+++ b/skyvern/forge/sdk/copilot/session_factory.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.agents.context import (replace_agent_message_field,)from skyvern.forge.sdk.copilot.browser_ablation import prompt_sha256+from skyvern.forge.sdk.copilot.config import SCREENSHOT_DROPPED_NUDGEfrom skyvern.forge.sdk.copilot.enforcement import (_RECENT_TOOL_OUTPUT_CHAR_CAP,_TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX,@@ -368,7 +369,8 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budgethen replace the latest unread outputs that do not fit with a notice that the call ran.3. If still over budget: truncate ALL tool outputs — first to 2000 chars,then to 300 only if the softer pass was not enough.- 4. If still over budget: aggressive prune as last resort.+ 4. If still over budget: aggressive prune as last resort, keeping the most recent screenshot unless+ it predates every step the prune keeps."""if not items:return ModelInputData(input=items, instructions=instructions)@@ -441,7 +443,27 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budgeand call_id.startswith("call_demo_")and get_agent_message_field(item, "type") == "function_call"}- items = aggressive_prune(items)+ frame = next((item for item in reversed(items[1:]) if _screenshot_with_image(item)), None)+ pruned = aggressive_prune(items)+ # The prune drops every screenshot and appends a re-take note, or keeps an earlier note instead of adding one.+ # A frame newer than the oldest kept step goes back in its input position without any note older than it; an+ # older frame stays dropped because the steps taken after it were cut.+ position = {id(item): index for index, item in enumerate(items)}+ oldest_kept = next((position[id(item)] for item in pruned[1:] if id(item) in position), None)+ if frame is not None and (oldest_kept is None or position[id(frame)] > oldest_kept):+ retake_note = NUDGE_SENTINEL + SCREENSHOT_DROPPED_NUDGE+ pruned = sorted(+ (+ item+ for item in [*pruned, frame]+ if id(item) in position+ and (+ position[id(item)] > position[id(frame)] or get_agent_message_field(item, "content") != retake_note+ )+ ),+ key=lambda item: position[id(item)],+ )+ items = prunedretained_demonstration_call_ids = {call_idfor item in itemsdiff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.pyindex ea21707a7..c505fef12 100644--- a/tests/unit/test_copilot_session_factory.py+++ b/tests/unit/test_copilot_session_factory.py@@ -19,8 +19,8 @@ from skyvern.forge.sdk.copilot.ask_user import (QuestionPart,QuestionResponse,)-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.enforcement import estimate_tokens, pending_screenshot_message+from skyvern.forge.sdk.copilot.config import SCREENSHOT_DROPPED_NUDGE, BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, estimate_tokens, pending_screenshot_messagefrom skyvern.forge.sdk.copilot.screenshot_utils import (ScreenshotActionRelation,ScreenshotEntry,@@ -775,16 +775,33 @@ class TestStagedScreenshotBinding:assert not any(_image_parts(item) for item in result.input)assert not any(log["event"] == "Injecting screenshot user message" for log in logs)- def test_aggressive_prune_drops_the_bound_frame_like_any_other_screenshot(self) -> None:- ctx = _ctx_with_staged_frame()- items: list[dict[str, Any]] = [{"role": "user", "content": "build me a workflow"}]+ def test_last_resort_prune_keeps_the_bound_frame_and_asks_for_a_retake_only_without_one(self) -> None:+ retake_note = NUDGE_SENTINEL + SCREENSHOT_DROPPED_NUDGE+ frame_older_than_the_kept_steps = pending_screenshot_message(+ SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)+ )+ items: list[dict[str, Any]] = [+ {"role": "user", "content": "build me a workflow"},+ frame_older_than_the_kept_steps,+ ]for i in range(12):items.append({"type": "function_call", "call_id": f"call-{i}", "name": "observe", "arguments": "{}"})items.append({"type": "function_call_output", "call_id": f"call-{i}", "output": "y" * 4000})+ retake_note_from_an_earlier_recovery = [*items[:-2], {"role": "user", "content": retake_note}, *items[-2:]]+ over_budget_after_every_trim = make_copilot_call_model_input_filter(1)- result = make_copilot_call_model_input_filter(1)(_mk_input_data(items, context=ctx))+ with_frame = over_budget_after_every_trim(+ _mk_input_data(retake_note_from_an_earlier_recovery, context=_ctx_with_staged_frame())+ )+ without_frame = over_budget_after_every_trim(+ _mk_input_data(items, context=SimpleNamespace(pending_screenshots=[]))+ )- assert not any(_image_parts(item) for item in result.input)+ assert [len(_image_parts(item)) for item in with_frame.input if _image_parts(item)] == [1]+ assert len(_image_parts(with_frame.input[-1])) == 1+ assert not any(item.get("content") == retake_note for item in with_frame.input)+ assert not any(_image_parts(item) for item in without_frame.input)+ assert [item.get("content") for item in without_frame.input].count(retake_note) == 1def test_aggressive_prune_reports_omitted_demonstration_fact_packets(self) -> None:items: list[dict[str, Any]] = [{"role": "user", "content": "build the demonstrated workflow"}]