Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12848): ignore unmatched VNC touch releases (#7663)
Aaron Perez8ef5ec674da8
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 01bd39f4a..cac5a587d 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -34,6 +34,7 @@ import { useSettingsStore } from "@/store/SettingsStore";import { wssBaseUrl, newWssBaseUrl, getCredentialParam } from "@/util/env";import { copyText } from "@/util/copyText";import { formatRecordingClock } from "@/util/recordingClock";+import { installNoVncGestureCrashGuard } from "@/util/novncGestureCrashGuard";import { cn } from "@/util/utils";import {StreamStatusPanel,@@ -43,6 +44,8 @@ import { handleVncClipboardPasteShortcut } from "@/components/browserStreamClipbimport "./browser-stream.css";+installNoVncGestureCrashGuard();+const MESSAGE_RECONNECT_DELAY_MS = 1000;const MESSAGE_MAX_RECONNECT_ATTEMPTS = 20;const STREAM_GAVE_UP_DIAGNOSTIC: StreamDiagnostic = {diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.tsindex 0e4d1ade0..ebbbd40aa 100644--- a/skyvern-frontend/src/novnc.d.ts+++ b/skyvern-frontend/src/novnc.d.ts@@ -43,3 +43,18 @@ declare module "@novnc/novnc/lib/rfb.js" {viewportChange(): void;}}++declare module "@novnc/novnc/lib/input/gesturehandler.js" {+ interface TrackedTouch {+ id: number;+ }++ export default class GestureHandler {+ _ignored: number[];+ _tracked: TrackedTouch[];++ attach(target: HTMLElement): void;+ _touchStart(id: number, x: number, y: number): void;+ _touchEnd(id: number, x: number, y: number): void;+ }+}diff --git a/skyvern-frontend/src/util/novncGestureCrashGuard.test.ts b/skyvern-frontend/src/util/novncGestureCrashGuard.test.tsnew file mode 100644index 000000000..2f8739208--- /dev/null+++ b/skyvern-frontend/src/util/novncGestureCrashGuard.test.ts@@ -0,0 +1,49 @@+import { beforeAll, describe, expect, it } from "vitest";+import _GestureHandler from "@novnc/novnc/lib/input/gesturehandler.js";++import { installNoVncGestureCrashGuard } from "./novncGestureCrashGuard";++const GestureHandler =+ (+ _GestureHandler as typeof _GestureHandler & {+ default?: typeof _GestureHandler;+ }+ ).default ?? _GestureHandler;++describe("installNoVncGestureCrashGuard", () => {+ beforeAll(() => {+ installNoVncGestureCrashGuard();+ });++ it("ignores a touchend without a matching tracked touch", () => {+ const handler = new GestureHandler();++ expect(() => handler._touchEnd(1, 0, 0)).not.toThrow();+ });++ it("preserves normal tracked touch handling", () => {+ const handler = new GestureHandler();+ const target = document.createElement("div");++ handler.attach(target);+ handler._touchStart(1, 0, 0);+ expect(() => handler._touchEnd(1, 0, 0)).not.toThrow();+ });++ it("preserves ignored-touch cleanup", () => {+ const handler = new GestureHandler();+ handler._ignored.push(1);++ handler._touchEnd(1, 0, 0);++ expect(handler._ignored).not.toContain(1);+ });++ it("does not wrap the handler more than once", () => {+ const guardedTouchEnd = GestureHandler.prototype._touchEnd;++ installNoVncGestureCrashGuard();++ expect(GestureHandler.prototype._touchEnd).toBe(guardedTouchEnd);+ });+});diff --git a/skyvern-frontend/src/util/novncGestureCrashGuard.ts b/skyvern-frontend/src/util/novncGestureCrashGuard.tsnew file mode 100644index 000000000..d1673690d--- /dev/null+++ b/skyvern-frontend/src/util/novncGestureCrashGuard.ts@@ -0,0 +1,29 @@+import _GestureHandler from "@novnc/novnc/lib/input/gesturehandler.js";++const GestureHandler =+ (+ _GestureHandler as typeof _GestureHandler & {+ default?: typeof _GestureHandler;+ }+ ).default ?? _GestureHandler;++let installed = false;++export function installNoVncGestureCrashGuard(): void {+ if (installed) return;+ installed = true;++ const originalTouchEnd = GestureHandler.prototype._touchEnd;+ GestureHandler.prototype._touchEnd = function (+ id: number,+ x: number,+ y: number,+ ): void {+ const isTracked = this._tracked.some((touch) => touch.id === id);+ const isIgnored = this._ignored.includes(id);+ // noVNC 1.5.0 dereferences the lookup even when a touch ends after the handler attached mid-gesture.+ if (!isTracked && !isIgnored) return;++ originalTouchEnd.call(this, id, x, y);+ };+}diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 57b7709c1..6ed907ecc 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -101,11 +101,13 @@ from skyvern.schemas.workflows import (WorkflowCreateYAMLRequest,WorkflowDefinitionYAML,)+from skyvern.utils.prompt_truncation import truncate_page_html_for_summaryfrom skyvern.utils.strings import escape_code_fencesfrom skyvern.utils.yaml_loader import safe_load_no_datesWORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")CHAT_HISTORY_CONTEXT_MESSAGES = 10+WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS = 3_000_000ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {"audio/mp4","audio/mpeg",@@ -1004,6 +1006,21 @@ async def _get_debug_artifact(organization_id: str, workflow_run_id: str) -> Artreturn artifacts[0] if isinstance(artifacts, list) and artifacts else None+async def _get_debug_html(organization_id: str, workflow_run_id: str) -> str | None:+ artifact = await _get_debug_artifact(organization_id, workflow_run_id)+ if not artifact:+ return None+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)+ if not artifact_bytes:+ return None+ # The extreme-size cap can drop middle-tree evidence; move to structure-aware+ # selection if that trade-off causes copilot regressions.+ return truncate_page_html_for_summary(+ artifact_bytes.decode("utf-8"),+ max_chars=WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS,+ )++async def _get_debug_run_info(organization_id: str, workflow_run_id: str | None) -> RunInfo | None:if not workflow_run_id:return None@@ -1016,12 +1033,7 @@ async def _get_debug_run_info(organization_id: str, workflow_run_id: str | None)block = blocks[0]- artifact = await _get_debug_artifact(organization_id, workflow_run_id)- if artifact:- artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)- html = artifact_bytes.decode("utf-8") if artifact_bytes else None- else:- html = None+ html = await _get_debug_html(organization_id, workflow_run_id)return RunInfo(block_label=block.label,@@ -1063,11 +1075,7 @@ async def _get_new_copilot_block_infos())- artifact = await _get_debug_artifact(organization_id, workflow_run_id)- html: str | None = None- if artifact:- artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)- html = artifact_bytes.decode("utf-8") if artifact_bytes else None+ html = await _get_debug_html(organization_id, workflow_run_id)return block_infos, htmldiff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 7e8534ec0..fa290fd80 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -77,6 +77,44 @@ def _make_chat_request()+async def test_get_debug_run_info_bounds_visible_elements_html(monkeypatch: pytest.MonkeyPatch) -> None:+ max_chars = workflow_copilot_route.WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS+ html = "HEAD_MARKER" + ("x" * (max_chars * 2)) + "TAIL_MARKER"+ artifact = SimpleNamespace()+ block = SimpleNamespace(+ label="block-1",+ block_type=SimpleNamespace(name="task"),+ status="failed",+ failure_reason="timed out",+ )+ monkeypatch.setattr(+ workflow_copilot_route,+ "_get_debug_artifact",+ AsyncMock(return_value=artifact),+ )+ monkeypatch.setattr(+ app.DATABASE,+ "observer",+ SimpleNamespace(get_workflow_run_blocks=AsyncMock(return_value=[block])),+ )+ monkeypatch.setattr(+ app,+ "ARTIFACT_MANAGER",+ SimpleNamespace(retrieve_artifact=AsyncMock(return_value=html.encode("utf-8"))),+ )++ run_info = await workflow_copilot_route._get_debug_run_info("org-1", "wr-1")++ assert run_info is not None+ assert run_info.html is not None+ assert len(run_info.html) < len(html)+ assert len(run_info.html) <= max_chars + 100+ assert "HEAD_MARKER" in run_info.html+ assert "TAIL_MARKER" in run_info.html+ assert "truncated by Skyvern" in run_info.html++@pytest.mark.asyncio@pytest.mark.parametrize("finalizer", ["normal", "cancel"], ids=["normal-turn", "cancel-turn"])@pytest.mark.parametrize("destination", ["persistence", "sse", "history"])