Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-9718] Layer 1 — deterministic HTML element-tree compression (#5970)
Shuchang Zhengcd137154597b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex d6b2f6a0b..f3d823f4c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2505,6 +2505,12 @@ class ForgeAgent:template_name = "check-user-goal-with-termination" if use_termination_prompt else "check-user-goal"prompt_name = "check-user-goal-with-termination" if use_termination_prompt else "check-user-goal"+ # SKY-9718 Layer 1: gate the lean recipe on the PostHog flag at the+ # call site. complete_verify only needs the visual page state to+ # decide is_complete / is_terminate / continue — no element-id refs,+ # no new_elements_ids threading — so we also drop Skyvern IDs.+ _ctx = skyvern_context.current()+ lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)verification_prompt = load_prompt_with_elements(element_tree_builder=scraped_page_refreshed,prompt_engine=prompt_engine,@@ -2515,6 +2521,10 @@ class ForgeAgent:terminate_criterion=task.terminate_criterion,action_history=actions_and_results_str,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ html_need_skyvern_attrs=False,+ lean_compress_long_href=lean_enabled,+ lean_compress_image_src=lean_enabled,+ lean_strip_url_query_strings=lean_enabled,)# This prompt is critical for our agent, we probably should use the primary LLM handler@@ -2936,6 +2946,32 @@ class ForgeAgent:)context.enable_speed_optimizations = False+ # SKY-9718 Layer 1: local-override env var for bench / debugging.+ # `FORCE_ENABLE_LEAN_ELEMENT_TREE=true` bypasses the PostHog gate+ # and forces lean ON for every run in this process. Never set in+ # production — the PostHog flag is the only prod control.+ if os.getenv("FORCE_ENABLE_LEAN_ELEMENT_TREE", "").lower() in ("true", "1", "yes"):+ context.enable_lean_element_tree = True+ LOG.info(+ "ENABLE_LEAN_ELEMENT_TREE forced ON via env var (bypasses PostHog gate)",+ task_id=task.task_id,+ )+ else:+ try:+ distinct_id = task.workflow_run_id if task.workflow_run_id else task.task_id+ context.enable_lean_element_tree = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "ENABLE_LEAN_ELEMENT_TREE",+ distinct_id,+ properties={"organization_id": task.organization_id},+ )+ except Exception:+ LOG.warning(+ "Failed to check ENABLE_LEAN_ELEMENT_TREE feature flag",+ exc_info=True,+ task_id=task.task_id,+ )+ context.enable_lean_element_tree = False+try:distinct_id = task.workflow_run_id if task.workflow_run_id else task.task_idcontext.use_artifact_bundling = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(@@ -3405,11 +3441,26 @@ class ForgeAgent:)enable_speed_optimizations = context.enable_speed_optimizationselement_tree_format = ElementTreeFormat.HTML+ # SKY-9718 Layer 1: extract-action is a planner template — keep Skyvern+ # internal IDs (default `html_need_skyvern_attrs=True`) and apply the+ # 3 lean transforms. Speed-optimization path uses economy tree which+ # drops lean for the same firefighting reason as prompt_engine's+ # overflow path.+ use_lean_tree = context.enable_lean_element_treeif enable_speed_optimizations:if step.retry_index == 0:elements_for_prompt = scraped_page.build_economy_elements_tree(element_tree_format)else:elements_for_prompt = scraped_page.build_element_tree(element_tree_format)+ elif use_lean_tree:+ elements_for_prompt = scraped_page.build_lean_elements_tree(+ element_tree_format,+ # compress_long_href stays OFF — the planner reads the href+ # to decide whether to click a link (destination signal).+ compress_long_href=False,+ compress_image_src=True,+ strip_url_query_strings=True,+ )else:elements_for_prompt = scraped_page.build_element_tree(element_tree_format)@@ -3531,6 +3582,14 @@ class ForgeAgent:parse_select_feature_enabled=context.enable_parse_select_in_extract,has_magic_link_page=context.has_magic_link_page(task.task_id),recent_dialog_messages_str=recent_dialog_messages_str,+ # SKY-9718 Layer 1: planner non-cached fallback. Keep Skyvern IDs+ # (default html_need_skyvern_attrs=True) — the planner emits+ # `click(id=...)` references. Gate lean on the PostHog flag.+ # `lean_compress_long_href` intentionally stays OFF — the planner+ # reads the href to decide whether to click a link.+ lean_compress_long_href=False,+ lean_compress_image_src=context.enable_lean_element_tree,+ lean_strip_url_query_strings=context.enable_lean_element_tree,)# Map template to prompt_name for logging/caching guardsdiff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex fc89771c5..e447009c9 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -89,6 +89,10 @@ class SkyvernContext:prompt_caching_settings: dict[str, bool] | None = Noneenable_speed_optimizations: bool = Falseuse_artifact_bundling: bool = False+ # SKY-9718 Layer 1 — gates apply_lean_recipe in prompt_engine + agent.+ # PostHog flag ENABLE_LEAN_ELEMENT_TREE, evaluated once per run at scrape time+ # and read sync from prompt-build sites.+ enable_lean_element_tree: bool = False# Trigger type of the enclosing workflow run (manual/api/scheduled/webhook).# Routed through SkyvernContext so non-API entry points (workers, scripts) can populate itdiff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex a5539f5bb..24e2d17de 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -28,6 +28,34 @@ from skyvern.services.browser_recording.types import (OutputBlock,)++def summarize_exfiltrated_recording_events(events: list[ExfiltratedEvent]) -> dict[str, t.Any]:+ cdp_by_event_name: dict[str, int] = {}+ console_by_dom_type: dict[str, int] = {}+ console_by_exfil_event_name: dict[str, int] = {}+ cdp_total = 0+ console_total = 0++ for ev in events:+ if isinstance(ev, ExfiltratedCdpEvent):+ cdp_total += 1+ cdp_by_event_name[ev.event_name] = cdp_by_event_name.get(ev.event_name, 0) + 1+ elif isinstance(ev, ExfiltratedConsoleEvent):+ console_total += 1+ dom_type = ev.params.type+ console_by_dom_type[dom_type] = console_by_dom_type.get(dom_type, 0) + 1+ console_by_exfil_event_name[ev.event_name] = console_by_exfil_event_name.get(ev.event_name, 0) + 1++ return {+ "recording_exfil_total_events": len(events),+ "recording_exfil_cdp_event_count": cdp_total,+ "recording_exfil_console_event_count": console_total,+ "recording_exfil_cdp_event_name_counts": cdp_by_event_name,+ "recording_exfil_console_dom_type_counts": console_by_dom_type,+ "recording_exfil_console_exfil_event_name_counts": console_by_exfil_event_name,+ }++LOG = structlog.get_logger(__name__)# avoid decompression bombs@@ -403,6 +431,12 @@ class Processor:"""events = self.compressed_chunks_to_events(compressed_chunks)+ LOG.info(+ "record_browser.process_recording_payload",+ recording_compressed_chunk_count=len(compressed_chunks),+ **summarize_exfiltrated_recording_events(events),+ **self.identity,+ )actions = self.events_to_actions(events)blocks = await self.actions_to_blocks(actions)parameters = self.blocks_to_parameters(blocks)diff --git a/skyvern/utils/lean_html.py b/skyvern/utils/lean_html.pynew file mode 100644index 000000000..f1dfe357b--- /dev/null+++ b/skyvern/utils/lean_html.py@@ -0,0 +1,125 @@+"""Deterministic element-tree compression for Skyvern prompts (SKY-9718, Layer 1).++The verifier (`check-user-goal`) and other element-rendering prompts ship a lot+of bytes that aren't needed for the LLM's decision. This module exposes three+independent transforms that callers of `load_prompt_with_elements` can compose+per template:++ transform saving (offline analysis, 729 prompts)+ ------------------------------------- --------------------------------------+ 1. compress_long_href 7.2% — long hrefs → `"#templated"`+ 2. compress_image_src 4.3% — drops `<img src="…">`+ 3. strip_url_query_strings 0.9% — strips `?…` from href/src++Combined potential ~12% planner savings. The largest single source of bloat+in offline analysis was Skyvern internal IDs (~24% of the verifier prompt),+but that is already addressable through `html_need_skyvern_attrs=False` in+`build_*_elements_tree` — no separate lean flag needed. Callers that don't+want Skyvern internal IDs in the rendered HTML pass `html_need_skyvern_attrs=False`+alongside the lean flags.++The transforms operate on the JSON element tree (a list of element dicts) BEFORE+it gets rendered to HTML by `json_to_html`. This mirrors `_process_element_for_economy_tree`+in `scraped_page.py` — same shape, different per-node logic.++`apply_lean_to_tree` deep-copies the input and is safe to share-by-reference.+"""++from __future__ import annotations++import copy+from typing import Any++# Hashed-href tokens are written by json_to_html when href > 150 chars. We+# replace long hrefs with a short marker BEFORE rendering so json_to_html's+# hash substitution never runs on lean trees.+_HASHED_HREF_PLACEHOLDER = "#templated"+_HASHED_HREF_LEN_THRESHOLD = 150+++def _transform_node(+ node: dict,+ *,+ compress_long_href: bool,+ compress_image_src: bool,+ strip_url_query_strings: bool,+) -> dict | None:+ """Apply the selected lean transforms to a single node and recurse into children.++ Mutates `node` in place (caller passes a deep-copied tree). Returns the+ transformed node, or None if the node should be dropped entirely (none of+ the current transforms drop nodes, but the return shape mirrors+ `_process_element_for_economy_tree`).+ """+ tag = (node.get("tagName") or "").lower()+ attributes: dict[str, Any] = node.get("attributes") or {}++ # Ordering matters when both `compress_long_href` and `strip_url_query_strings`+ # are on: strip first so a `https://x.co/foo?<200 chars of utm>` URL collapses+ # to a short `https://x.co/foo` and skips the length check, instead of being+ # blanket-replaced with `#templated` and losing the destination signal.++ # 1. Strip URL query strings from href / src.+ if strip_url_query_strings:+ for key in ("href", "src"):+ val = attributes.get(key)+ if isinstance(val, str) and "?" in val:+ attributes[key] = val.split("?", 1)[0]++ # 2. Compress image src — drop entirely; keep alt + id + everything else.+ if compress_image_src and tag == "img":+ attributes.pop("src", None)++ # 3. Compress long hrefs (would otherwise become `{{_<sha256>}}` in+ # json_to_html). Runs after query-strip so URLs that were only "long" due+ # to tracking junk keep their meaningful path; URLs that are long even+ # without the query (signed CDN, encoded payloads) still get hashed.+ if compress_long_href:+ href = attributes.get("href")+ if isinstance(href, str) and len(href) > _HASHED_HREF_LEN_THRESHOLD:+ attributes["href"] = _HASHED_HREF_PLACEHOLDER++ node["attributes"] = attributes++ children = node.get("children")+ if children:+ new_children: list[dict] = []+ for child in children:+ transformed = _transform_node(+ child,+ compress_long_href=compress_long_href,+ compress_image_src=compress_image_src,+ strip_url_query_strings=strip_url_query_strings,+ )+ if transformed is not None:+ new_children.append(transformed)+ node["children"] = new_children++ return node+++def apply_lean_to_tree(+ elements: list[dict],+ *,+ compress_long_href: bool = False,+ compress_image_src: bool = False,+ strip_url_query_strings: bool = False,+) -> list[dict]:+ """Apply the deterministic lean-tree recipe to a list of element dicts.++ Each transform is independently gated. Deep-copies the input; returns a+ new list of transformed element dicts. With every flag False the result is+ a deep copy of the input (no transforms applied).+ """+ out: list[dict] = []+ for element in elements:+ copied = copy.deepcopy(element)+ transformed = _transform_node(+ copied,+ compress_long_href=compress_long_href,+ compress_image_src=compress_image_src,+ strip_url_query_strings=strip_url_query_strings,+ )+ if transformed is not None:+ out.append(transformed)+ return outdiff --git a/skyvern/utils/prompt_engine.py b/skyvern/utils/prompt_engine.pyindex 46ade72da..00a6ab259 100644--- a/skyvern/utils/prompt_engine.py+++ b/skyvern/utils/prompt_engine.py@@ -62,6 +62,16 @@ def load_prompt_with_elements_tracked(prompt_engine: PromptEngine,template_name: str,html_need_skyvern_attrs: bool = True,+ *,+ # SKY-9718 Layer 1 — deterministic lean-tree transforms. Each flag toggles+ # one transform independently. Callers decide which to enable (typically by+ # AND-ing with `skyvern_context.current().enable_lean_element_tree` if they+ # want the experiment gate). To drop Skyvern internal IDs from the rendered+ # HTML, callers pass `html_need_skyvern_attrs=False` — that's the existing+ # `json_to_html` mechanism and stacks on top of any lean flags chosen here.+ lean_compress_long_href: bool = False,+ lean_compress_image_src: bool = False,+ lean_strip_url_query_strings: bool = False,**kwargs: Any,) -> tuple[str, dict[str, Any]]:"""Same as load_prompt_with_elements but also returns post-ceiling kwargs.@@ -71,7 +81,18 @@ def load_prompt_with_elements_tracked(inputs for caching should use these values instead of the pre-drop kwargsso two requests that render to the same final prompt share a cache key."""- elements = element_tree_builder.build_element_tree(html_need_skyvern_attrs=html_need_skyvern_attrs)+ lean_any = lean_compress_long_href or lean_compress_image_src or lean_strip_url_query_strings+ if lean_any and element_tree_builder.support_lean_elements_tree():+ elements = element_tree_builder.build_lean_elements_tree(+ html_need_skyvern_attrs=html_need_skyvern_attrs,+ compress_long_href=lean_compress_long_href,+ compress_image_src=lean_compress_image_src,+ strip_url_query_strings=lean_strip_url_query_strings,+ )+ else:+ # Builder doesn't implement lean (e.g. IncrementalScrapePage) or caller+ # asked for no transforms — fall back to the plain element tree.+ elements = element_tree_builder.build_element_tree(html_need_skyvern_attrs=html_need_skyvern_attrs)prompt = prompt_engine.load_prompt(template_name,elements=elements,@@ -80,6 +101,8 @@ def load_prompt_with_elements_tracked(token_count = count_tokens(prompt)if token_count > DEFAULT_MAX_TOKENS and element_tree_builder.support_economy_elements_tree():# get rid of all the secondary elements like SVG, etc+ # NOTE: economy fallback drops the lean recipe — context-overflow firefighting+ # path; we accept the lean savings loss in exchange for fitting under the cap.elements = element_tree_builder.build_economy_elements_tree(html_need_skyvern_attrs=html_need_skyvern_attrs)prompt = prompt_engine.load_prompt(template_name, elements=elements, **kwargs)economy_token_count = count_tokens(prompt)@@ -139,6 +162,10 @@ def load_prompt_with_elements(prompt_engine: PromptEngine,template_name: str,html_need_skyvern_attrs: bool = True,+ *,+ lean_compress_long_href: bool = False,+ lean_compress_image_src: bool = False,+ lean_strip_url_query_strings: bool = False,**kwargs: Any,) -> str:prompt, _ = load_prompt_with_elements_tracked(@@ -146,6 +173,9 @@ def load_prompt_with_elements(prompt_engine=prompt_engine,template_name=template_name,html_need_skyvern_attrs=html_need_skyvern_attrs,+ lean_compress_long_href=lean_compress_long_href,+ lean_compress_image_src=lean_compress_image_src,+ lean_strip_url_query_strings=lean_strip_url_query_strings,**kwargs,)return promptdiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 0b3a05ea5..dc6e9dab8 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -976,6 +976,12 @@ async def handle_sequential_click_for_dropdown(}action_history_str = json.dumps(action_result)+ # SKY-9718 Layer 1: sequential-click after-dropdown verifier path. Keep+ # Skyvern IDs (default html_need_skyvern_attrs=True) because+ # `new_elements_ids` is threaded and the LLM compares those IDs to what's+ # rendered. Gate lean on the PostHog flag.+ _ctx = skyvern_context.current()+ lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)prompt = load_prompt_with_elements(element_tree_builder=scraped_page_after_open,prompt_engine=prompt_engine,@@ -986,6 +992,9 @@ async def handle_sequential_click_for_dropdown(without_screenshots=True,action_history=action_history_str,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ lean_compress_long_href=lean_enabled,+ lean_compress_image_src=lean_enabled,+ lean_strip_url_query_strings=lean_enabled,)response = await app.CHECK_USER_GOAL_LLM_API_HANDLER(prompt=prompt,@@ -3692,6 +3701,12 @@ async def select_from_emerging_elements(if len(new_interactable_element_ids) == 0:raise NoIncrementalElementFoundForCustomSelection(element_id=current_element_id)+ # SKY-9718 Layer 1: custom-select threads `new_elements_ids` so Skyvern IDs+ # stay (default html_need_skyvern_attrs=True). Gate lean on the PostHog flag.+ # `lean_compress_long_href` stays OFF — the LLM reads option hrefs to pick+ # the right select target.+ _ctx = skyvern_context.current()+ lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)prompt = load_prompt_with_elements(element_tree_builder=scraped_page_after_open,prompt_engine=prompt_engine,@@ -3704,6 +3719,9 @@ async def select_from_emerging_elements(new_elements_ids=new_interactable_element_ids,navigation_payload_str=json.dumps(task.navigation_payload),local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ lean_compress_long_href=False,+ lean_compress_image_src=lean_enabled,+ lean_strip_url_query_strings=lean_enabled,)LOG.info("Calling LLM to find the match element")diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex e33b1897a..7443c0f1a 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -115,6 +115,15 @@ class ElementTreeBuilder(ABC):def support_economy_elements_tree(self) -> bool:pass+ @abstractmethod+ def support_lean_elements_tree(self) -> bool:+ """SKY-9718 Layer 1 — whether this builder implements build_lean_elements_tree.++ Mirrors `support_economy_elements_tree`. Callers of `load_prompt_with_elements`+ check this before passing lean flags so builders that only implement the+ plain `build_element_tree` (e.g. `IncrementalScrapePage`) don't crash.+ """+@abstractmethoddef build_element_tree(self, fmt: ElementTreeFormat = ElementTreeFormat.HTML, html_need_skyvern_attrs: bool = True@@ -130,6 +139,18 @@ class ElementTreeBuilder(ABC):) -> str:pass+ @abstractmethod+ def build_lean_elements_tree(+ self,+ fmt: ElementTreeFormat = ElementTreeFormat.HTML,+ html_need_skyvern_attrs: bool = True,+ *,+ compress_long_href: bool = False,+ compress_image_src: bool = False,+ strip_url_query_strings: bool = False,+ ) -> str:+ pass+class ScrapedPage(BaseModel, ElementTreeBuilder):"""@@ -152,6 +173,10 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):element_tree: list[dict]element_tree_trimmed: list[dict]economy_element_tree: list[dict] | None = None+ # SKY-9718 Layer 1: lazy cache for lean trees, keyed by the 3-flag combo+ # the caller asked for. Two call sites in one prompt build asking for+ # different combos each pay the walk cost once.+ lean_element_tree_cache: dict[tuple[bool, bool, bool], list[dict]] = {}last_used_element_tree: list[dict] | None = None# Last HTML variant built for the LLM (captures economy / truncation).# None when the last build used fmt=JSON or no build has run yet. The@@ -229,6 +254,9 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):def support_economy_elements_tree(self) -> bool:return True+ def support_lean_elements_tree(self) -> bool:+ return True+def build_element_tree(self, fmt: ElementTreeFormat = ElementTreeFormat.HTML, html_need_skyvern_attrs: bool = True) -> str:@@ -309,6 +337,58 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):element["children"] = new_childrenreturn element+ def build_lean_elements_tree(+ self,+ fmt: ElementTreeFormat = ElementTreeFormat.HTML,+ html_need_skyvern_attrs: bool = True,+ *,+ compress_long_href: bool = False,+ compress_image_src: bool = False,+ strip_url_query_strings: bool = False,+ ) -> str:+ """SKY-9718 Layer 1 — deterministic lean element tree.++ Same shape as `build_economy_elements_tree`: deep-copy the trimmed+ tree, walk it applying the lean recipe, cache by flag combo, then+ render.++ Each of the 3 transforms is independently gated by its kwarg. Callers+ of `load_prompt_with_elements` pick the right combo per template.++ Skyvern internal IDs are *not* a lean flag — drop them by passing+ `html_need_skyvern_attrs=False` (the existing mechanism). `json_to_html`+ only copies `element["id"]` into the rendered HTML when that flag is True.+ """+ from skyvern.utils.lean_html import apply_lean_to_tree++ cache_key = (+ compress_long_href,+ compress_image_src,+ strip_url_query_strings,+ )+ cached = self.lean_element_tree_cache.get(cache_key)+ if cached is None:+ cached = apply_lean_to_tree(+ self.element_tree_trimmed,+ compress_long_href=compress_long_href,+ compress_image_src=compress_image_src,+ strip_url_query_strings=strip_url_query_strings,+ )+ self.lean_element_tree_cache[cache_key] = cached++ self.last_used_element_tree = cached++ if fmt == ElementTreeFormat.JSON:+ self.last_used_element_tree_html = None+ return json.dumps(cached)++ if fmt == ElementTreeFormat.HTML:+ result = "".join(json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs) for element in cached)+ self.last_used_element_tree_html = result+ return result++ raise UnknownElementTreeFormat(fmt=fmt)+async def refresh(self, draw_boxes: bool = True, scroll: bool = True, max_retries: int = 0) -> Self:refreshed_page = await self._browser_state.scrape_website(url=self.url,@@ -333,6 +413,16 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):# Defensive: callers today rebuild before reading, but future direct# reads of this field post-refresh would otherwise see stale HTML.self.last_used_element_tree_html = None+ # SKY-9718 Layer 1 + pre-existing bug for economy: derived element-tree+ # caches must be invalidated when the underlying `element_tree_trimmed`+ # is replaced. Without these resets, the next build_*_elements_tree+ # call returns a tree from the pre-refresh page state — particularly+ # bad on the `complete_verify` hot path, where the verifier would+ # reason about the pre-action page when checking if a post-action+ # goal was achieved.+ self.economy_element_tree = None+ self.lean_element_tree_cache = {}+ self.last_used_element_tree = Nonereturn selfasync def generate_scraped_page(diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 777e04f98..f68ef9900 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -701,6 +701,9 @@ class IncrementalScrapePage(ElementTreeBuilder):def support_economy_elements_tree(self) -> bool:return False+ def support_lean_elements_tree(self) -> bool:+ return False+def build_element_tree(self, fmt: ElementTreeFormat = ElementTreeFormat.HTML, html_need_skyvern_attrs: bool = True) -> str:@@ -721,6 +724,17 @@ class IncrementalScrapePage(ElementTreeBuilder):) -> str:raise NotImplementedError("Not implemented")+ def build_lean_elements_tree(+ self,+ fmt: ElementTreeFormat = ElementTreeFormat.HTML,+ html_need_skyvern_attrs: bool = True,+ *,+ compress_long_href: bool = False,+ compress_image_src: bool = False,+ strip_url_query_strings: bool = False,+ ) -> str:+ raise NotImplementedError("Not implemented")+def _should_keep_unique_id(element: dict) -> bool:# case where we shouldn't keep unique_iddiff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex 23ca00f6a..ab9c4e362 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -4,9 +4,11 @@ Just an example unit test for now. Will expand later.import typing as t-from skyvern.services.browser_recording.service import Processor+from skyvern.services.browser_recording.service import Processor, summarize_exfiltrated_recording_eventsfrom skyvern.services.browser_recording.types import (+ ExfiltratedCdpEvent,ExfiltratedConsoleEvent,+ ExfiltratedEventCdpParams,)ORG_ID = "org_123"@@ -37,7 +39,7 @@ def make_console_event(return ExfiltratedConsoleEvent(kind="exfiltrated-event",source="console",- event_name="user-interaction",+ event_name="user_interaction",params=params,timestamp=timestamp,)@@ -75,6 +77,38 @@ def make_mouseleave_event()+def make_click_event(+ target: dict[str, t.Any],+ timestamp: float,+) -> ExfiltratedConsoleEvent:+ params: dict[str, t.Any] = {+ "type": "click",+ "target": target,+ "timestamp": timestamp,+ }++ return make_console_event(+ params=params,+ timestamp=timestamp,+ )+++def test_click() -> None:+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])++ event = make_click_event(+ target=target,+ timestamp=1000.0,+ )++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ actions = processor.events_to_actions([event])++ assert len(actions) == 1+ assert actions[0].kind == "click"+ assert actions[0].target.sky_id == "sky-123"++def test_hover() -> None:target = dict(id="button-1", skyId="sky-123", text=["Click me"])@@ -92,3 +126,39 @@ def test_hover() -> None:actions = processor.events_to_actions([event1, event2])assert len(actions) == 1+++def test_summarize_exfiltrated_recording_events_mixed() -> None:+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])+ click = make_click_event(target=target, timestamp=1000.0)+ keypress = make_console_event(+ params={+ "type": "keypress",+ "target": target,+ "timestamp": 1001.0,+ },+ timestamp=1001.0,+ )+ cdp_nav = ExfiltratedCdpEvent(+ kind="exfiltrated-event",+ event_name="nav:frame_navigated",+ params=ExfiltratedEventCdpParams(),+ source="cdp",+ timestamp=999.0,+ )+ cdp_nav_2 = ExfiltratedCdpEvent(+ kind="exfiltrated-event",+ event_name="nav:frame_navigated",+ params=ExfiltratedEventCdpParams(),+ source="cdp",+ timestamp=1002.0,+ )++ summary = summarize_exfiltrated_recording_events([cdp_nav, click, keypress, cdp_nav_2])++ assert summary["recording_exfil_total_events"] == 4+ assert summary["recording_exfil_cdp_event_count"] == 2+ assert summary["recording_exfil_console_event_count"] == 2+ assert summary["recording_exfil_cdp_event_name_counts"] == {"nav:frame_navigated": 2}+ assert summary["recording_exfil_console_dom_type_counts"] == {"click": 1, "keypress": 1}+ assert summary["recording_exfil_console_exfil_event_name_counts"] == {"user_interaction": 2}diff --git a/tests/unit/test_build_lean_elements_tree.py b/tests/unit/test_build_lean_elements_tree.pynew file mode 100644index 000000000..ef476582f--- /dev/null+++ b/tests/unit/test_build_lean_elements_tree.py@@ -0,0 +1,215 @@+"""SKY-9718 Layer 1 — coverage for ScrapedPage.build_lean_elements_tree.++The pure walker (`apply_lean_to_tree`) is covered by test_lean_html.py. This+file covers the builder method on `ScrapedPage`: cache hit/miss, cache+isolation across flag combos, JSON vs HTML dispatch, `html_need_skyvern_attrs`+interaction, and `last_used_element_tree[_html]` side effects.+"""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage+++def _make_scraped_page(element_tree_trimmed: list[dict]) -> ScrapedPage:+ return ScrapedPage(+ elements=[],+ element_tree=[],+ element_tree_trimmed=element_tree_trimmed,+ _browser_state=MagicMock(),+ _clean_up_func=AsyncMock(return_value=[]),+ _scrape_exclude=None,+ )++[email protected](autouse=True)+def _scoped_context():+ """`json_to_html` calls `skyvern_context.ensure_context()`, so we need one."""+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):+ yield+++def _node(tag: str, *, id: str | None = None, attributes: dict | None = None, children: list | None = None) -> dict:+ n: dict = {"tagName": tag, "attributes": dict(attributes or {}), "children": list(children or [])}+ if id is not None:+ n["id"] = id+ return n+++# --- cache hit / miss ----------------------------------------------------+++def test_cache_hit_same_flag_combo_returns_same_tree_object() -> None:+ """Two calls with identical flags should reuse the cached transformed tree."""+ page = _make_scraped_page([_node("a", id="AAAB", attributes={"href": "/x?utm=1"})])++ # First call populates the cache.+ page.build_lean_elements_tree(strip_url_query_strings=True)+ cache_key = (False, False, True)+ assert cache_key in page.lean_element_tree_cache+ cached_tree = page.lean_element_tree_cache[cache_key]++ # Second call with the same flags must reuse the cached list (identity, not equality).+ page.build_lean_elements_tree(strip_url_query_strings=True)+ assert page.lean_element_tree_cache[cache_key] is cached_tree+++def test_cache_isolation_across_flag_combos() -> None:+ """Different flag combos populate different cache slots and don't clobber each other."""+ page = _make_scraped_page(+ [+ _node(+ "a",+ attributes={"href": "/x?utm=1"},+ children=[_node("img", attributes={"src": "/cat.jpg?cb=1", "alt": "cat"})],+ )+ ]+ )++ page.build_lean_elements_tree(compress_image_src=True)+ page.build_lean_elements_tree(strip_url_query_strings=True)+ page.build_lean_elements_tree(compress_image_src=True, strip_url_query_strings=True)++ keys = set(page.lean_element_tree_cache.keys())+ assert keys == {(False, True, False), (False, False, True), (False, True, True)}++ # Different combos produce different rendered output.+ only_img = page.build_lean_elements_tree(compress_image_src=True)+ only_qs = page.build_lean_elements_tree(strip_url_query_strings=True)+ both = page.build_lean_elements_tree(compress_image_src=True, strip_url_query_strings=True)+ assert only_img != only_qs+ assert both != only_img+ assert both != only_qs+++def test_cache_isolation_default_flags_off_is_its_own_slot() -> None:+ """No-op call (all flags False) is a valid combo with its own cache key."""+ page = _make_scraped_page([_node("a", attributes={"href": "/x?utm=1"})])+ page.build_lean_elements_tree()+ assert (False, False, False) in page.lean_element_tree_cache+++# --- JSON vs HTML dispatch -----------------------------------------------+++def test_fmt_html_returns_rendered_html_string() -> None:+ page = _make_scraped_page([_node("a", attributes={"href": "/foo"}, children=[])])+ out = page.build_lean_elements_tree(ElementTreeFormat.HTML)+ assert isinstance(out, str)+ assert out.startswith("<a")+ assert 'href="/foo"' in out+++def test_fmt_json_returns_json_string() -> None:+ page = _make_scraped_page([_node("a", attributes={"href": "/foo"}, children=[])])+ out = page.build_lean_elements_tree(ElementTreeFormat.JSON)+ assert isinstance(out, str)+ # Valid JSON serialization of the cached tree.+ import json as _json++ parsed = _json.loads(out)+ assert isinstance(parsed, list)+ assert parsed[0]["tagName"] == "a"+++# --- html_need_skyvern_attrs interaction ---------------------------------+++def test_html_need_skyvern_attrs_false_drops_top_level_id_from_rendered_html() -> None:+ """Skyvern internal IDs (element['id']) are agent-scaffolding — drop at render+ when html_need_skyvern_attrs=False."""+ page = _make_scraped_page([_node("a", id="AAAB", attributes={"href": "/x"})])+ with_ids = page.build_lean_elements_tree(html_need_skyvern_attrs=True)+ without_ids = page.build_lean_elements_tree(html_need_skyvern_attrs=False)+ assert 'id="AAAB"' in with_ids+ assert 'id="AAAB"' not in without_ids+ # Same href in both — lean recipe is independent of the ID-rendering toggle.+ assert 'href="/x"' in with_ids+ assert 'href="/x"' in without_ids+++# --- last_used_element_tree[_html] side effects --------------------------+++def test_html_render_writes_last_used_element_tree_html() -> None:+ page = _make_scraped_page([_node("a", attributes={"href": "/foo"})])+ out = page.build_lean_elements_tree(ElementTreeFormat.HTML)+ assert page.last_used_element_tree_html == out+ assert page.last_used_element_tree is not None+++def test_json_render_clears_last_used_element_tree_html() -> None:+ """Matches the contract on build_element_tree / build_economy_elements_tree."""+ page = _make_scraped_page([_node("a", attributes={"href": "/foo"})])+ page.build_lean_elements_tree(ElementTreeFormat.HTML)+ assert page.last_used_element_tree_html is not None+ page.build_lean_elements_tree(ElementTreeFormat.JSON)+ assert page.last_used_element_tree_html is None+++# --- refresh() invalidates derived caches --------------------------------+++async def test_refresh_clears_lean_and_economy_caches() -> None:+ """Regression: refresh() replaces element_tree_trimmed, so derived caches+ (lean, economy) must be cleared or subsequent build_*_elements_tree calls+ return pre-refresh data. Particularly bad on the complete_verify hot path+ where the verifier reasons post-action."""+ from unittest.mock import patch++ page = _make_scraped_page([_node("a", attributes={"href": "/before"})])++ # Populate both caches from the pre-refresh tree.+ page.build_lean_elements_tree(strip_url_query_strings=True)+ page.build_economy_elements_tree()+ page.last_used_element_tree = [_node("marker", attributes={})]+ assert len(page.lean_element_tree_cache) == 1+ assert page.economy_element_tree is not None+ assert page.last_used_element_tree is not None++ # Simulate refresh: the browser-state scrape returns a new tree.+ refreshed_tree = [_node("a", attributes={"href": "/after"})]+ fake_refreshed = _make_scraped_page(refreshed_tree)+ page._browser_state.scrape_website = AsyncMock(return_value=fake_refreshed)++ with patch.object(type(page), "model_config", page.model_config):+ await page.refresh()++ # All derived caches reset.+ assert page.lean_element_tree_cache == {}+ assert page.economy_element_tree is None+ assert page.last_used_element_tree is None+ assert page.last_used_element_tree_html is None+++# --- end-to-end: flag combo actually transforms the output --------------+++def test_flag_combo_actually_compresses_in_rendered_output() -> None:+ """Sanity check: a combo of flags reaches the rendered HTML."""+ long_href = "https://example.com/" + "x" * 200+ page = _make_scraped_page(+ [+ _node(+ "a",+ attributes={"href": long_href + "?utm=1"},+ children=[_node("img", attributes={"src": "/cat.jpg?cb=1", "alt": "cat"})],+ )+ ]+ )+ out = page.build_lean_elements_tree(+ compress_long_href=True,+ compress_image_src=True,+ strip_url_query_strings=True,+ )+ assert "#templated" in out # long href compressed (after query strip)+ assert "/cat.jpg" not in out # img src dropped+ assert "utm=1" not in out # query strings gone+ assert "cb=1" not in outdiff --git a/tests/unit/test_lean_html.py b/tests/unit/test_lean_html.pynew file mode 100644index 000000000..a4be6b836--- /dev/null+++ b/tests/unit/test_lean_html.py@@ -0,0 +1,236 @@+"""SKY-9718 Layer 1 — deterministic element-tree compression unit tests.++Operates on the JSON element-tree shape (mirror of `_process_element_for_economy_tree`+in scraped_page.py). Each of the 3 transforms is independently gated; tests+exercise each in isolation and a realistic end-to-end combo.+"""++import copy++from skyvern.utils.lean_html import apply_lean_to_tree+++def _node(+ tag: str, *, id: str | None = None, attributes: dict | None = None, children: list | None = None, **extra+) -> dict:+ """Build an element-tree node in the shape ScrapedPage produces."""+ n: dict = {"tagName": tag, "attributes": dict(attributes or {}), "children": list(children or [])}+ if id is not None:+ n["id"] = id+ n.update(extra)+ return n+++# Convenience constant for tests that want to enable every transform.+ALL_LEAN_FLAGS = dict(+ compress_long_href=True,+ compress_image_src=True,+ strip_url_query_strings=True,+)+++# --- flag #1: compress_long_href -----------------------------------------+++def test_compress_long_href_replaces_with_templated() -> None:+ """Hrefs > 150 chars get replaced with '#templated' (short-circuits json_to_html's sha256 substitution)."""+ long_href = "https://example.com/path/with/very/long/" + "x" * 200+ assert len(long_href) > 150+ tree = [_node("a", attributes={"href": long_href}, children=[])]+ out = apply_lean_to_tree(tree, compress_long_href=True)+ assert out[0]["attributes"]["href"] == "#templated"+++def test_compress_long_href_short_url_left_alone() -> None:+ tree = [_node("a", attributes={"href": "/login"}, children=[])]+ out = apply_lean_to_tree(tree, compress_long_href=True)+ assert out[0]["attributes"]["href"] == "/login"+++def test_compress_long_href_off_keeps_long_url() -> None:+ long_href = "https://example.com/" + "x" * 200+ tree = [_node("a", attributes={"href": long_href}, children=[])]+ out = apply_lean_to_tree(tree, compress_long_href=False)+ assert out[0]["attributes"]["href"] == long_href+++def test_strip_query_runs_before_long_href_so_short_paths_survive() -> None:+ """Regression: a short path with a giant query string should keep its path,+ not collapse to '#templated'. Strip-QS runs first, then long-href checks length."""+ short_path = "https://x.co/foo"+ long_query = "?" + "utm=" + "x" * 250+ assert len(short_path + long_query) > 150+ tree = [_node("a", attributes={"href": short_path + long_query}, children=[])]+ out = apply_lean_to_tree(tree, compress_long_href=True, strip_url_query_strings=True)+ # Path preserved; long-href no-ops because the post-strip URL is short.+ assert out[0]["attributes"]["href"] == short_path+++def test_compress_long_href_still_fires_when_path_alone_exceeds_threshold() -> None:+ """A URL that's long even after the query is stripped still gets hashed."""+ long_path = "https://example.com/" + "x" * 200+ tree = [_node("a", attributes={"href": long_path + "?ignored=1"}, children=[])]+ out = apply_lean_to_tree(tree, compress_long_href=True, strip_url_query_strings=True)+ assert out[0]["attributes"]["href"] == "#templated"+++# --- flag #2: compress_image_src -----------------------------------------+++def test_compress_image_src_drops_src_keeps_alt_and_id() -> None:+ tree = [+ _node(+ "img",+ id="AEE0",+ attributes={"src": "https://cdn.example.com/very/long/path/image.jpg", "alt": "A pretty cat"},+ children=[],+ )+ ]+ out = apply_lean_to_tree(tree, compress_image_src=True)+ assert "src" not in out[0]["attributes"]+ assert out[0]["attributes"]["alt"] == "A pretty cat"+ assert out[0]["id"] == "AEE0"+++def test_compress_image_src_does_not_touch_non_img_src() -> None:+ """Only `<img>` src is dropped; <script src>, <iframe src> etc are left alone."""+ tree = [+ _node("script", attributes={"src": "https://cdn.example.com/foo.js"}, children=[]),+ _node("iframe", attributes={"src": "https://example.com/frame"}, children=[]),+ ]+ out = apply_lean_to_tree(tree, compress_image_src=True)+ assert out[0]["attributes"]["src"] == "https://cdn.example.com/foo.js"+ assert out[1]["attributes"]["src"] == "https://example.com/frame"+++def test_compress_image_src_off_keeps_src() -> None:+ tree = [_node("img", attributes={"src": "/cat.jpg", "alt": "cat"}, children=[])]+ out = apply_lean_to_tree(tree, compress_image_src=False)+ assert out[0]["attributes"]["src"] == "/cat.jpg"+++# --- flag #3: strip_url_query_strings ------------------------------------+++def test_strip_url_query_strings_href() -> None:+ tree = [_node("a", attributes={"href": "https://example.com/foo?utm_source=x&utm_medium=y"}, children=[])]+ out = apply_lean_to_tree(tree, strip_url_query_strings=True)+ assert out[0]["attributes"]["href"] == "https://example.com/foo"+++def test_strip_url_query_strings_src() -> None:+ """Strips ?... from `src` too. Combined with compress_image_src=False so we can observe."""+ tree = [_node("script", attributes={"src": "/path/foo.js?v=12345"}, children=[])]+ out = apply_lean_to_tree(tree, strip_url_query_strings=True, compress_image_src=False)+ assert out[0]["attributes"]["src"] == "/path/foo.js"+++def test_strip_url_query_strings_leaves_clean_url_alone() -> None:+ tree = [_node("a", attributes={"href": "/clean/path"}, children=[])]+ out = apply_lean_to_tree(tree, strip_url_query_strings=True)+ assert out[0]["attributes"]["href"] == "/clean/path"+++def test_strip_url_query_strings_off_keeps_query() -> None:+ tree = [_node("a", attributes={"href": "/x?utm=1"}, children=[])]+ out = apply_lean_to_tree(tree, strip_url_query_strings=False)+ assert out[0]["attributes"]["href"] == "/x?utm=1"+++# --- defaults / no-ops ---------------------------------------------------+++def test_default_flags_off_is_a_deep_copy() -> None:+ """With every flag off, the recipe is a deep copy (no mutations)."""+ tree = [_node("a", id="AAAB", attributes={"href": "/x?utm=1"}, children=[])]+ out = apply_lean_to_tree(tree)+ assert out == tree+ # And it really is a copy:+ out[0]["id"] = "MUTATED"+ assert tree[0]["id"] == "AAAB"+++def test_apply_lean_to_tree_does_not_mutate_input() -> None:+ tree = [_node("a", id="AAAB", attributes={"href": "/x?utm=1"}, children=[])]+ snapshot = copy.deepcopy(tree)+ apply_lean_to_tree(tree, **ALL_LEAN_FLAGS)+ assert tree == snapshot, "apply_lean_to_tree must deep-copy and leave the input untouched"+++def test_apply_lean_to_tree_is_idempotent_all_flags() -> None:+ tree = [+ _node(+ "div",+ id="AAA",+ attributes={},+ children=[+ _node(+ "a",+ id="AAB",+ attributes={"href": "/x?utm=1"},+ children=[_node("img", attributes={"src": "//cdn/i.png?w=4", "alt": "x"}, children=[])],+ )+ ],+ )+ ]+ once = apply_lean_to_tree(tree, **ALL_LEAN_FLAGS)+ twice = apply_lean_to_tree(once, **ALL_LEAN_FLAGS)+ assert once == twice+++def test_apply_lean_to_tree_preserves_skyvern_internal_id() -> None:+ """Skyvern internal IDs (`element["id"]`) are NOT touched by this module.+ Callers drop them via `html_need_skyvern_attrs=False` at the render layer.+ """+ tree = [_node("a", id="AAAB", attributes={"href": "/x"}, children=[])]+ out = apply_lean_to_tree(tree, **ALL_LEAN_FLAGS)+ assert out[0]["id"] == "AAAB"+++# --- end-to-end combo ----------------------------------------------------+++def test_realistic_snippet_all_flags_on() -> None:+ """Every transform fires on a realistic snippet. Skyvern IDs untouched at this layer."""+ tree = [+ _node(+ "div",+ id="AAAP",+ attributes={},+ children=[+ _node(+ "a",+ id="AAAF",+ attributes={+ "href": "https://www.example.com/business?utm_content=x&utm_campaign=y&utm_source=z",+ },+ children=[_node("span", id="AAAG", text="Businesses", children=[])],+ ),+ _node(+ "img",+ id="AEE0",+ attributes={+ "src": "https://cdn.example.com/api/utilities/v1/imageproxy/x.jpg?w=388",+ "alt": "logo",+ },+ children=[],+ ),+ _node(+ "a",+ id="AAAL",+ attributes={"href": "https://www.example.com/very/long/path/" + "x" * 200},+ children=[],+ ),+ ],+ )+ ]+ out = apply_lean_to_tree(tree, **ALL_LEAN_FLAGS)+ # Skyvern IDs survive — that's `html_need_skyvern_attrs`'s job, not ours.+ for node in [out[0], out[0]["children"][0], out[0]["children"][1], out[0]["children"][2]]:+ assert node.get("id") is not None+ # URL query stripped on the first <a>+ assert out[0]["children"][0]["attributes"]["href"] == "https://www.example.com/business"+ # img src dropped+ assert "src" not in out[0]["children"][1]["attributes"]+ # hashed-href compressed+ assert out[0]["children"][2]["attributes"]["href"] == "#templated"