Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
revert(SKY-10083): revert screenshot downscaling — no measurable token savings (#6501)
Trevor Cheungea9907cf0c66
diff --git a/skyvern/config.py b/skyvern/config.pyindex dfce87eb9..b0bf2a83e 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -120,13 +120,6 @@ class Settings(BaseSettings):# Emit per-call image_tokens/image_cost/image_count on the LLM duration log so# screenshot spend can be monitored independently of the provider's blended tokens.LLM_IMAGE_COST_TRACKING_ENABLED: bool = True- # Global on-switch: when true, downscales screenshots for ALL orgs/runs in the environment- # (cuts vision-token + storage cost; viewport stays full-res). For a single-org / per-run A/B,- # leave false and use the screenshot_downscale flag.- SCREENSHOT_DOWNSCALE_ENABLED: bool = False- # Capped at a vision-token tile boundary: providers bill screenshots in fixed-size tiles, so- # raising this past the boundary can sharply increase per-image cost for little fidelity gain.- SCREENSHOT_DOWNSCALE_MAX_HEIGHT: int = 768# Ratio should be between 0 and 1.# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex a42b42364..1a2f8329b 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -98,7 +98,6 @@ from skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler-from skyvern.forge.sdk.experimentation.screenshot_downscale import resolve_screenshot_downscale_for_contextfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_valuefrom skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logsfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus@@ -1364,13 +1363,6 @@ class ForgeAgent:task_url=task.url,log_context={"task_id": task.task_id},)- await resolve_screenshot_downscale_for_context(- context,- task.task_id,- task.organization_id,- task_url=task.url,- log_context={"task_id": task.task_id},- )step = await self.update_step(step=step, status=StepStatus.running)injected_actions = await app.AGENT_FUNCTION.prepare_step_execution(diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex bf5feb9b9..8690fb278 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -46,7 +46,6 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContextfrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerTypefrom skyvern.forge.sdk.experimentation.prompt_families import effective_prompt_schema_variant-from skyvern.forge.sdk.experimentation.screenshot_downscale import effective_downscale_heightfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Stepfrom skyvern.forge.sdk.schemas.ai_suggestions import AISuggestionfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thought@@ -278,15 +277,6 @@ def _slim_log_fields(context: SkyvernContext | None, prompt_name: str | None) ->}-def _screenshot_downscale_log_fields(context: SkyvernContext | None) -> dict[str, Any]:- # variant is the raw A/B arm ("control"/"treatment"/None); max_height is the effective applied- # height (also non-null when the static kill-switch downscales outside the flag).- return {- "screenshot_downscale_variant": context.screenshot_downscale_variant if context else None,- "screenshot_downscale_max_height": effective_downscale_height(context),- }--@runtime_checkableclass RouterWithModelList(Protocol):model_list: list[dict[str, Any]]@@ -1565,7 +1555,6 @@ class LLMAPIHandlerFactory:llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),- **_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),)@@ -2116,7 +2105,6 @@ class LLMAPIHandlerFactory:llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),- **_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),)@@ -2605,7 +2593,6 @@ class LLMCaller:llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),- **_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),)diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 238b3c586..185b67015 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -117,10 +117,6 @@ class SkyvernContext:# and read sync from prompt-build sites.enable_lean_element_tree: bool = Falseenrich_tree_mode: EnrichTreeMode = EnrichTreeMode.CONTROL- # Per-run screenshot_downscale A/B state, set at run boot. variant is the raw flag value- # ("control"/"treatment"/None); max_height (None = no downscale) drives the actual resize.- screenshot_downscale_variant: str | None = None- screenshot_downscale_max_height: int | None = Nonestep_retry_index: int = 0# Run-level SLIM_LLM_OUTPUT_PROMPTS assignment, resolved once by slim_llm_output.diff --git a/skyvern/forge/sdk/experimentation/screenshot_downscale.py b/skyvern/forge/sdk/experimentation/screenshot_downscale.pydeleted file mode 100644index 2e147294e..000000000--- a/skyvern/forge/sdk/experimentation/screenshot_downscale.py+++ /dev/null@@ -1,81 +0,0 @@-from typing import Any--import structlog--from skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.settings_manager import SettingsManager--LOG = structlog.get_logger()--SCREENSHOT_DOWNSCALE_FLAG = "screenshot_downscale"--_DISABLED_FLAG_VALUES = {"control", "off", "false", "0", "disabled", "no"}-_ENABLED_FLAG_VALUES = {"treatment", "on", "true", "enabled", "yes"}---def _setting_default_height() -> int | None:- settings = SettingsManager.get_settings()- return settings.SCREENSHOT_DOWNSCALE_MAX_HEIGHT if settings.SCREENSHOT_DOWNSCALE_ENABLED else None---def _height_from_flag(flag_value: str | None, default_height: int | None) -> int | None:- if not isinstance(flag_value, str):- return default_height- value = flag_value.strip().lower()- if value in _DISABLED_FLAG_VALUES:- return None- if value in _ENABLED_FLAG_VALUES:- return SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT- if value.isdigit():- height = int(value)- return height if height > 0 else None- return default_height---def effective_downscale_height(context: SkyvernContext | None) -> int | None:- """Resolved downscale height (None = skip): the per-run flag wins, else the static setting."""- if context is not None and context.screenshot_downscale_max_height is not None:- return context.screenshot_downscale_max_height- return _setting_default_height()---async def resolve_screenshot_downscale_for_context(- context: SkyvernContext,- distinct_id: str,- organization_id: str | None,- *,- workflow_permanent_id: str | None = None,- task_url: str | None = None,- log_context: dict[str, Any] | None = None,-) -> None:- # Lazy import breaks an import cycle: this module is imported by webeye.utils.page, which- # sits below the forge app singleton.- from skyvern.forge import app-- default_height = _setting_default_height()- properties: dict[str, str] = {}- if organization_id:- properties["organization_id"] = organization_id- if workflow_permanent_id:- properties["workflow_permanent_id"] = workflow_permanent_id- if task_url:- properties["task_url"] = task_url-- try:- flag_value = await app.EXPERIMENTATION_PROVIDER.get_value_cached(- SCREENSHOT_DOWNSCALE_FLAG,- distinct_id,- properties=properties,- )- except Exception:- LOG.warning(- "Failed to check screenshot_downscale feature flag",- exc_info=True,- distinct_id=distinct_id,- **(log_context or {}),- )- context.screenshot_downscale_max_height = default_height- return-- context.screenshot_downscale_variant = flag_value if isinstance(flag_value, str) else None- context.screenshot_downscale_max_height = _height_from_flag(flag_value, default_height)diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 1d821ba05..4cab85787 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -64,7 +64,6 @@ from skyvern.forge.sdk.db._sentinels import _UNSETfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerTypefrom skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_featuresfrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context-from skyvern.forge.sdk.experimentation.screenshot_downscale import resolve_screenshot_downscale_for_contextfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.organizations import Organization@@ -1184,13 +1183,6 @@ class WorkflowService:workflow_permanent_id=workflow_run.workflow_permanent_id,log_context={"workflow_run_id": workflow_run.workflow_run_id},)- await resolve_screenshot_downscale_for_context(- new_context,- workflow_run.workflow_run_id,- organization.organization_id,- workflow_permanent_id=workflow_run.workflow_permanent_id,- log_context={"workflow_run_id": workflow_run.workflow_run_id},- )# Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)diff --git a/skyvern/utils/image_resizer.py b/skyvern/utils/image_resizer.pyindex 5b1295966..5f2848ee9 100644--- a/skyvern/utils/image_resizer.py+++ b/skyvern/utils/image_resizer.py@@ -1,11 +1,8 @@import iofrom typing import TypedDict-import structlogfrom PIL import Image-LOG = structlog.get_logger()-class Resolution(TypedDict):width: int@@ -53,33 +50,6 @@ def resize_screenshots(screenshots: list[bytes], target_dimension: Resolution) -return new_screenshots-def downscale_screenshots_to_height(screenshots: list[bytes], max_height: int) -> list[bytes]:- """Downscale each screenshot to a max height (aspect-preserving).-- Images already at or below ``max_height`` pass through unchanged.- """- if max_height <= 0:- return screenshots- new_screenshots: list[bytes] = []- for screenshot in screenshots:- try:- img = Image.open(io.BytesIO(screenshot))- if img.height <= max_height:- new_screenshots.append(screenshot)- continue- target_width = max(1, round(img.width * max_height / img.height))- resized_img = img.resize((target_width, max_height), Image.Resampling.LANCZOS)- img_byte_arr = io.BytesIO()- resized_img.save(img_byte_arr, format="PNG")- new_screenshots.append(img_byte_arr.getvalue())- except Exception:- # Keep the original so one unreadable screenshot can't abort the batch or leave a- # treatment-run LLM call screenshot-less.- LOG.warning("Failed to downscale screenshot; using original", exc_info=True)- new_screenshots.append(screenshot)- return new_screenshots--def scale_coordinates(current_coordinates: tuple[int, int],current_dimension: Resolution,diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex ab7900085..17e675956 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -16,10 +16,8 @@ from playwright.async_api import ElementHandle, Frame, Pagefrom skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIRfrom skyvern.exceptions import FailedToTakeScreenshotfrom skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.experimentation.screenshot_downscale import effective_downscale_heightfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import apply_context_attrs, traced-from skyvern.utils.image_resizer import downscale_screenshots_to_heightLOG = structlog.get_logger()@@ -513,19 +511,15 @@ class SkyvernFrame:scroll: bool = True,) -> list[bytes]:if not scroll:- screenshots = [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]- else:- screenshots, _ = await _scrolling_screenshots_helper(- page=page,- url=url,- max_number=max_number,- draw_boxes=draw_boxes,- mode=ScreenshotMode.DETAILED,- )+ return [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]- max_height = effective_downscale_height(skyvern_context.current())- if max_height:- screenshots = downscale_screenshots_to_height(screenshots, max_height)+ screenshots, _ = await _scrolling_screenshots_helper(+ page=page,+ url=url,+ max_number=max_number,+ draw_boxes=draw_boxes,+ mode=ScreenshotMode.DETAILED,+ )return screenshots@classmethoddiff --git a/tests/unit/test_image_resizer_downscale.py b/tests/unit/test_image_resizer_downscale.pydeleted file mode 100644index 042ae45e2..000000000--- a/tests/unit/test_image_resizer_downscale.py+++ /dev/null@@ -1,45 +0,0 @@-from io import BytesIO--from PIL import Image--from skyvern.utils.image_resizer import downscale_screenshots_to_height---def _png(width: int, height: int) -> bytes:- buf = BytesIO()- Image.new("RGB", (width, height), "white").save(buf, format="PNG")- return buf.getvalue()---def _dims(screenshot: bytes) -> tuple[int, int]:- return Image.open(BytesIO(screenshot)).size---def test_downscales_1080p_to_768p_preserving_aspect() -> None:- out = downscale_screenshots_to_height([_png(1920, 1080)], 768)- assert _dims(out[0]) == (1365, 768)---def test_leaves_images_at_or_below_target_unchanged() -> None:- original = _png(1280, 720)- out = downscale_screenshots_to_height([original], 720)- assert out[0] is original # no re-encode: avoids a needless PNG round-trip + byte bloat- assert _dims(downscale_screenshots_to_height([_png(800, 600)], 720)[0]) == (800, 600)---def test_disabled_when_max_height_non_positive() -> None:- shots = [_png(1920, 1080)]- assert downscale_screenshots_to_height(shots, 0) is shots---def test_non_16_9_aspect_preserved() -> None:- # 1000x1000 -> height 500, width scales proportionally- assert _dims(downscale_screenshots_to_height([_png(1000, 1000)], 500)[0]) == (500, 500)---def test_corrupt_screenshot_kept_and_batch_continues() -> None:- good = _png(1920, 1080)- corrupt = b"not-a-png"- out = downscale_screenshots_to_height([good, corrupt], 768)- assert _dims(out[0]) == (1365, 768) # good one still downscaled- assert out[1] is corrupt # bad one kept as-is, batch not aborteddiff --git a/tests/unit/test_screenshot_downscale_flag.py b/tests/unit/test_screenshot_downscale_flag.pydeleted file mode 100644index bdfa8cf1b..000000000--- a/tests/unit/test_screenshot_downscale_flag.py+++ /dev/null@@ -1,114 +0,0 @@-import pytest--from skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.experimentation.screenshot_downscale import (- _height_from_flag,- effective_downscale_height,- resolve_screenshot_downscale_for_context,-)-from skyvern.forge.sdk.settings_manager import SettingsManager---def test_height_from_flag_absent_falls_back_to_default() -> None:- assert _height_from_flag(None, 768) == 768- assert _height_from_flag(None, None) is None--[email protected]("value", ["control", "off", "false", "0", "no", "CONTROL"])-def test_height_from_flag_disabled_values_return_none(value: str) -> None:- assert _height_from_flag(value, 768) is None--[email protected]("value", ["treatment", "on", "true", "yes", "ENABLED"])-def test_height_from_flag_enabled_uses_setting_height(value: str) -> None:- assert _height_from_flag(value, None) == SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT---def test_height_from_flag_explicit_numeric_height() -> None:- assert _height_from_flag("720", None) == 720- assert _height_from_flag("1080", 768) == 1080---def test_height_from_flag_unknown_value_falls_back_to_default() -> None:- assert _height_from_flag("garbage", 768) == 768---def test_effective_height_per_run_context_overrides_disabled_setting(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)- assert effective_downscale_height(SkyvernContext(screenshot_downscale_max_height=720)) == 720---def test_effective_height_falls_back_to_setting_when_context_unset(monkeypatch: pytest.MonkeyPatch) -> None:- settings = SettingsManager.get_settings()- monkeypatch.setattr(settings, "SCREENSHOT_DOWNSCALE_ENABLED", True)- monkeypatch.setattr(settings, "SCREENSHOT_DOWNSCALE_MAX_HEIGHT", 768)- assert effective_downscale_height(None) == 768- assert effective_downscale_height(SkyvernContext()) == 768---def test_effective_height_disabled_with_no_context(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)- assert effective_downscale_height(None) is None---class _FakeProvider:- def __init__(self, value: str | None) -> None:- self._value = value-- async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:- return self._value---async def test_resolver_treatment_records_variant_and_height(monkeypatch: pytest.MonkeyPatch) -> None:- from skyvern.forge import app as forge_app-- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider("treatment"))- ctx = SkyvernContext()- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")- assert ctx.screenshot_downscale_variant == "treatment"- assert ctx.screenshot_downscale_max_height == SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT---async def test_resolver_control_records_variant_no_downscale(monkeypatch: pytest.MonkeyPatch) -> None:- from skyvern.forge import app as forge_app-- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider("control"))- ctx = SkyvernContext()- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")- assert ctx.screenshot_downscale_variant == "control"- assert ctx.screenshot_downscale_max_height is None---async def test_resolver_untargeted_run_has_null_variant(monkeypatch: pytest.MonkeyPatch) -> None:- from skyvern.forge import app as forge_app-- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider(None))- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)- ctx = SkyvernContext()- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")- assert ctx.screenshot_downscale_variant is None- assert ctx.screenshot_downscale_max_height is None---def test_height_from_flag_non_string_is_defensive() -> None:- # A provider returning a non-str (e.g. an unconfigured test mock) must not crash.- assert _height_from_flag(object(), 768) == 768 # type: ignore[arg-type]---async def test_resolver_tolerates_non_string_flag_value(monkeypatch: pytest.MonkeyPatch) -> None:- from skyvern.forge import app as forge_app-- class _NonStringProvider:- async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> object:- return object()-- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _NonStringProvider())- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)- ctx = SkyvernContext()- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")- assert ctx.screenshot_downscale_variant is None- assert ctx.screenshot_downscale_max_height is None