Skip to content

Automate browser based workflows with AI

revert(SKY-10083): revert screenshot downscaling — no measurable token savings (#6501)

Trevor Cheungea9907cf0c66
diff --git a/skyvern/config.py b/skyvern/config.py
index dfce87eb9..b0bf2a83e 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -120,13 +120,6 @@ class Settings(BaseSettings):
# Emit per-call image_tokens/image_cost/image_count on the LLM duration log so
# screenshot spend can be monitored independently of the provider's blended tokens.
LLM_IMAGE_COST_TRACKING_ENABLED: bool = True
- # Global on-switch: when true, downscales screenshots for ALL orgs/runs in the environment
- # (cuts vision-token + storage cost; viewport stays full-res). For a single-org / per-run A/B,
- # leave false and use the screenshot_downscale flag.
- SCREENSHOT_DOWNSCALE_ENABLED: bool = False
- # Capped at a vision-token tile boundary: providers bill screenshots in fixed-size tiles, so
- # raising this past the boundary can sharply increase per-image cost for little fidelity gain.
- SCREENSHOT_DOWNSCALE_MAX_HEIGHT: int = 768
# Ratio should be between 0 and 1.
# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.
LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index a42b42364..1a2f8329b 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -98,7 +98,6 @@ from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
-from skyvern.forge.sdk.experimentation.screenshot_downscale import resolve_screenshot_downscale_for_context
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus
@@ -1364,13 +1363,6 @@ class ForgeAgent:
task_url=task.url,
log_context={"task_id": task.task_id},
)
- await resolve_screenshot_downscale_for_context(
- context,
- task.task_id,
- task.organization_id,
- task_url=task.url,
- log_context={"task_id": task.task_id},
- )
step = await self.update_step(step=step, status=StepStatus.running)
injected_actions = await app.AGENT_FUNCTION.prepare_step_execution(
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index bf5feb9b9..8690fb278 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -46,7 +46,6 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.experimentation.prompt_families import effective_prompt_schema_variant
-from skyvern.forge.sdk.experimentation.screenshot_downscale import effective_downscale_height
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thought
@@ -278,15 +277,6 @@ def _slim_log_fields(context: SkyvernContext | None, prompt_name: str | None) ->
}
-def _screenshot_downscale_log_fields(context: SkyvernContext | None) -> dict[str, Any]:
- # variant is the raw A/B arm ("control"/"treatment"/None); max_height is the effective applied
- # height (also non-null when the static kill-switch downscales outside the flag).
- return {
- "screenshot_downscale_variant": context.screenshot_downscale_variant if context else None,
- "screenshot_downscale_max_height": effective_downscale_height(context),
- }
-
-
@runtime_checkable
class RouterWithModelList(Protocol):
model_list: list[dict[str, Any]]
@@ -1565,7 +1555,6 @@ class LLMAPIHandlerFactory:
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
- **_screenshot_downscale_log_fields(context),
**_consume_prompt_breakdown(context),
)
@@ -2116,7 +2105,6 @@ class LLMAPIHandlerFactory:
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
- **_screenshot_downscale_log_fields(context),
**_consume_prompt_breakdown(context),
)
@@ -2605,7 +2593,6 @@ class LLMCaller:
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
- **_screenshot_downscale_log_fields(context),
**_consume_prompt_breakdown(context),
)
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 238b3c586..185b67015 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -117,10 +117,6 @@ class SkyvernContext:
# and read sync from prompt-build sites.
enable_lean_element_tree: bool = False
enrich_tree_mode: EnrichTreeMode = EnrichTreeMode.CONTROL
- # Per-run screenshot_downscale A/B state, set at run boot. variant is the raw flag value
- # ("control"/"treatment"/None); max_height (None = no downscale) drives the actual resize.
- screenshot_downscale_variant: str | None = None
- screenshot_downscale_max_height: int | None = None
step_retry_index: int = 0
# Run-level SLIM_LLM_OUTPUT_PROMPTS assignment, resolved once by slim_llm_output.
diff --git a/skyvern/forge/sdk/experimentation/screenshot_downscale.py b/skyvern/forge/sdk/experimentation/screenshot_downscale.py
deleted file mode 100644
index 2e147294e..000000000
--- a/skyvern/forge/sdk/experimentation/screenshot_downscale.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from typing import Any
-
-import structlog
-
-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.settings_manager import SettingsManager
-
-LOG = structlog.get_logger()
-
-SCREENSHOT_DOWNSCALE_FLAG = "screenshot_downscale"
-
-_DISABLED_FLAG_VALUES = {"control", "off", "false", "0", "disabled", "no"}
-_ENABLED_FLAG_VALUES = {"treatment", "on", "true", "enabled", "yes"}
-
-
-def _setting_default_height() -> int | None:
- settings = SettingsManager.get_settings()
- return settings.SCREENSHOT_DOWNSCALE_MAX_HEIGHT if settings.SCREENSHOT_DOWNSCALE_ENABLED else None
-
-
-def _height_from_flag(flag_value: str | None, default_height: int | None) -> int | None:
- if not isinstance(flag_value, str):
- return default_height
- value = flag_value.strip().lower()
- if value in _DISABLED_FLAG_VALUES:
- return None
- if value in _ENABLED_FLAG_VALUES:
- return SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT
- if value.isdigit():
- height = int(value)
- return height if height > 0 else None
- return default_height
-
-
-def effective_downscale_height(context: SkyvernContext | None) -> int | None:
- """Resolved downscale height (None = skip): the per-run flag wins, else the static setting."""
- if context is not None and context.screenshot_downscale_max_height is not None:
- return context.screenshot_downscale_max_height
- return _setting_default_height()
-
-
-async def resolve_screenshot_downscale_for_context(
- context: SkyvernContext,
- distinct_id: str,
- organization_id: str | None,
- *,
- workflow_permanent_id: str | None = None,
- task_url: str | None = None,
- log_context: dict[str, Any] | None = None,
-) -> None:
- # Lazy import breaks an import cycle: this module is imported by webeye.utils.page, which
- # sits below the forge app singleton.
- from skyvern.forge import app
-
- default_height = _setting_default_height()
- properties: dict[str, str] = {}
- if organization_id:
- properties["organization_id"] = organization_id
- if workflow_permanent_id:
- properties["workflow_permanent_id"] = workflow_permanent_id
- if task_url:
- properties["task_url"] = task_url
-
- try:
- flag_value = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
- SCREENSHOT_DOWNSCALE_FLAG,
- distinct_id,
- properties=properties,
- )
- except Exception:
- LOG.warning(
- "Failed to check screenshot_downscale feature flag",
- exc_info=True,
- distinct_id=distinct_id,
- **(log_context or {}),
- )
- context.screenshot_downscale_max_height = default_height
- return
-
- context.screenshot_downscale_variant = flag_value if isinstance(flag_value, str) else None
- context.screenshot_downscale_max_height = _height_from_flag(flag_value, default_height)
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 1d821ba05..4cab85787 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -64,7 +64,6 @@ from skyvern.forge.sdk.db._sentinels import _UNSET
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType
from skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_features
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
-from skyvern.forge.sdk.experimentation.screenshot_downscale import resolve_screenshot_downscale_for_context
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
@@ -1184,13 +1183,6 @@ class WorkflowService:
workflow_permanent_id=workflow_run.workflow_permanent_id,
log_context={"workflow_run_id": workflow_run.workflow_run_id},
)
- await resolve_screenshot_downscale_for_context(
- new_context,
- workflow_run.workflow_run_id,
- organization.organization_id,
- workflow_permanent_id=workflow_run.workflow_permanent_id,
- log_context={"workflow_run_id": workflow_run.workflow_run_id},
- )
# Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.
all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)
diff --git a/skyvern/utils/image_resizer.py b/skyvern/utils/image_resizer.py
index 5b1295966..5f2848ee9 100644
--- a/skyvern/utils/image_resizer.py
+++ b/skyvern/utils/image_resizer.py
@@ -1,11 +1,8 @@
import io
from typing import TypedDict
-import structlog
from PIL import Image
-LOG = structlog.get_logger()
-
class Resolution(TypedDict):
width: int
@@ -53,33 +50,6 @@ def resize_screenshots(screenshots: list[bytes], target_dimension: Resolution) -
return new_screenshots
-def downscale_screenshots_to_height(screenshots: list[bytes], max_height: int) -> list[bytes]:
- """Downscale each screenshot to a max height (aspect-preserving).
-
- Images already at or below ``max_height`` pass through unchanged.
- """
- if max_height <= 0:
- return screenshots
- new_screenshots: list[bytes] = []
- for screenshot in screenshots:
- try:
- img = Image.open(io.BytesIO(screenshot))
- if img.height <= max_height:
- new_screenshots.append(screenshot)
- continue
- target_width = max(1, round(img.width * max_height / img.height))
- resized_img = img.resize((target_width, max_height), Image.Resampling.LANCZOS)
- img_byte_arr = io.BytesIO()
- resized_img.save(img_byte_arr, format="PNG")
- new_screenshots.append(img_byte_arr.getvalue())
- except Exception:
- # Keep the original so one unreadable screenshot can't abort the batch or leave a
- # treatment-run LLM call screenshot-less.
- LOG.warning("Failed to downscale screenshot; using original", exc_info=True)
- new_screenshots.append(screenshot)
- return new_screenshots
-
-
def scale_coordinates(
current_coordinates: tuple[int, int],
current_dimension: Resolution,
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index ab7900085..17e675956 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -16,10 +16,8 @@ from playwright.async_api import ElementHandle, Frame, Page
from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIR
from skyvern.exceptions import FailedToTakeScreenshot
from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.experimentation.screenshot_downscale import effective_downscale_height
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced
-from skyvern.utils.image_resizer import downscale_screenshots_to_height
LOG = structlog.get_logger()
@@ -513,19 +511,15 @@ class SkyvernFrame:
scroll: bool = True,
) -> list[bytes]:
if not scroll:
- screenshots = [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]
- else:
- screenshots, _ = await _scrolling_screenshots_helper(
- page=page,
- url=url,
- max_number=max_number,
- draw_boxes=draw_boxes,
- mode=ScreenshotMode.DETAILED,
- )
+ return [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]
- max_height = effective_downscale_height(skyvern_context.current())
- if max_height:
- screenshots = downscale_screenshots_to_height(screenshots, max_height)
+ screenshots, _ = await _scrolling_screenshots_helper(
+ page=page,
+ url=url,
+ max_number=max_number,
+ draw_boxes=draw_boxes,
+ mode=ScreenshotMode.DETAILED,
+ )
return screenshots
@classmethod
diff --git a/tests/unit/test_image_resizer_downscale.py b/tests/unit/test_image_resizer_downscale.py
deleted file mode 100644
index 042ae45e2..000000000
--- a/tests/unit/test_image_resizer_downscale.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from io import BytesIO
-
-from PIL import Image
-
-from skyvern.utils.image_resizer import downscale_screenshots_to_height
-
-
-def _png(width: int, height: int) -> bytes:
- buf = BytesIO()
- Image.new("RGB", (width, height), "white").save(buf, format="PNG")
- return buf.getvalue()
-
-
-def _dims(screenshot: bytes) -> tuple[int, int]:
- return Image.open(BytesIO(screenshot)).size
-
-
-def test_downscales_1080p_to_768p_preserving_aspect() -> None:
- out = downscale_screenshots_to_height([_png(1920, 1080)], 768)
- assert _dims(out[0]) == (1365, 768)
-
-
-def test_leaves_images_at_or_below_target_unchanged() -> None:
- original = _png(1280, 720)
- out = downscale_screenshots_to_height([original], 720)
- assert out[0] is original # no re-encode: avoids a needless PNG round-trip + byte bloat
- assert _dims(downscale_screenshots_to_height([_png(800, 600)], 720)[0]) == (800, 600)
-
-
-def test_disabled_when_max_height_non_positive() -> None:
- shots = [_png(1920, 1080)]
- assert downscale_screenshots_to_height(shots, 0) is shots
-
-
-def test_non_16_9_aspect_preserved() -> None:
- # 1000x1000 -> height 500, width scales proportionally
- assert _dims(downscale_screenshots_to_height([_png(1000, 1000)], 500)[0]) == (500, 500)
-
-
-def test_corrupt_screenshot_kept_and_batch_continues() -> None:
- good = _png(1920, 1080)
- corrupt = b"not-a-png"
- out = downscale_screenshots_to_height([good, corrupt], 768)
- assert _dims(out[0]) == (1365, 768) # good one still downscaled
- assert out[1] is corrupt # bad one kept as-is, batch not aborted
diff --git a/tests/unit/test_screenshot_downscale_flag.py b/tests/unit/test_screenshot_downscale_flag.py
deleted file mode 100644
index bdfa8cf1b..000000000
--- a/tests/unit/test_screenshot_downscale_flag.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import pytest
-
-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.experimentation.screenshot_downscale import (
- _height_from_flag,
- effective_downscale_height,
- resolve_screenshot_downscale_for_context,
-)
-from skyvern.forge.sdk.settings_manager import SettingsManager
-
-
-def test_height_from_flag_absent_falls_back_to_default() -> None:
- assert _height_from_flag(None, 768) == 768
- assert _height_from_flag(None, None) is None
-
-
[email protected]("value", ["control", "off", "false", "0", "no", "CONTROL"])
-def test_height_from_flag_disabled_values_return_none(value: str) -> None:
- assert _height_from_flag(value, 768) is None
-
-
[email protected]("value", ["treatment", "on", "true", "yes", "ENABLED"])
-def test_height_from_flag_enabled_uses_setting_height(value: str) -> None:
- assert _height_from_flag(value, None) == SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT
-
-
-def test_height_from_flag_explicit_numeric_height() -> None:
- assert _height_from_flag("720", None) == 720
- assert _height_from_flag("1080", 768) == 1080
-
-
-def test_height_from_flag_unknown_value_falls_back_to_default() -> None:
- assert _height_from_flag("garbage", 768) == 768
-
-
-def test_effective_height_per_run_context_overrides_disabled_setting(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)
- assert effective_downscale_height(SkyvernContext(screenshot_downscale_max_height=720)) == 720
-
-
-def test_effective_height_falls_back_to_setting_when_context_unset(monkeypatch: pytest.MonkeyPatch) -> None:
- settings = SettingsManager.get_settings()
- monkeypatch.setattr(settings, "SCREENSHOT_DOWNSCALE_ENABLED", True)
- monkeypatch.setattr(settings, "SCREENSHOT_DOWNSCALE_MAX_HEIGHT", 768)
- assert effective_downscale_height(None) == 768
- assert effective_downscale_height(SkyvernContext()) == 768
-
-
-def test_effective_height_disabled_with_no_context(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)
- assert effective_downscale_height(None) is None
-
-
-class _FakeProvider:
- def __init__(self, value: str | None) -> None:
- self._value = value
-
- async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:
- return self._value
-
-
-async def test_resolver_treatment_records_variant_and_height(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge import app as forge_app
-
- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider("treatment"))
- ctx = SkyvernContext()
- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")
- assert ctx.screenshot_downscale_variant == "treatment"
- assert ctx.screenshot_downscale_max_height == SettingsManager.get_settings().SCREENSHOT_DOWNSCALE_MAX_HEIGHT
-
-
-async def test_resolver_control_records_variant_no_downscale(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge import app as forge_app
-
- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider("control"))
- ctx = SkyvernContext()
- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")
- assert ctx.screenshot_downscale_variant == "control"
- assert ctx.screenshot_downscale_max_height is None
-
-
-async def test_resolver_untargeted_run_has_null_variant(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge import app as forge_app
-
- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _FakeProvider(None))
- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)
- ctx = SkyvernContext()
- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")
- assert ctx.screenshot_downscale_variant is None
- assert ctx.screenshot_downscale_max_height is None
-
-
-def test_height_from_flag_non_string_is_defensive() -> None:
- # A provider returning a non-str (e.g. an unconfigured test mock) must not crash.
- assert _height_from_flag(object(), 768) == 768 # type: ignore[arg-type]
-
-
-async def test_resolver_tolerates_non_string_flag_value(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge import app as forge_app
-
- class _NonStringProvider:
- async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> object:
- return object()
-
- monkeypatch.setattr(forge_app, "EXPERIMENTATION_PROVIDER", _NonStringProvider())
- monkeypatch.setattr(SettingsManager.get_settings(), "SCREENSHOT_DOWNSCALE_ENABLED", False)
- ctx = SkyvernContext()
- await resolve_screenshot_downscale_for_context(ctx, "wr_test", "o_test")
- assert ctx.screenshot_downscale_variant is None
- assert ctx.screenshot_downscale_max_height is None