Skip to content

Automate browser based workflows with AI

fix(SKY-12848): ignore unmatched VNC touch releases (#7663)

Aaron Perez8ef5ec674da8
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 01bd39f4a..cac5a587d 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -34,6 +34,7 @@ import { useSettingsStore } from "@/store/SettingsStore";
import { wssBaseUrl, newWssBaseUrl, getCredentialParam } from "@/util/env";
import { copyText } from "@/util/copyText";
import { formatRecordingClock } from "@/util/recordingClock";
+import { installNoVncGestureCrashGuard } from "@/util/novncGestureCrashGuard";
import { cn } from "@/util/utils";
import {
StreamStatusPanel,
@@ -43,6 +44,8 @@ import { handleVncClipboardPasteShortcut } from "@/components/browserStreamClipb
import "./browser-stream.css";
+installNoVncGestureCrashGuard();
+
const MESSAGE_RECONNECT_DELAY_MS = 1000;
const MESSAGE_MAX_RECONNECT_ATTEMPTS = 20;
const STREAM_GAVE_UP_DIAGNOSTIC: StreamDiagnostic = {
diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.ts
index 0e4d1ade0..ebbbd40aa 100644
--- a/skyvern-frontend/src/novnc.d.ts
+++ b/skyvern-frontend/src/novnc.d.ts
@@ -43,3 +43,18 @@ declare module "@novnc/novnc/lib/rfb.js" {
viewportChange(): void;
}
}
+
+declare module "@novnc/novnc/lib/input/gesturehandler.js" {
+ interface TrackedTouch {
+ id: number;
+ }
+
+ export default class GestureHandler {
+ _ignored: number[];
+ _tracked: TrackedTouch[];
+
+ attach(target: HTMLElement): void;
+ _touchStart(id: number, x: number, y: number): void;
+ _touchEnd(id: number, x: number, y: number): void;
+ }
+}
diff --git a/skyvern-frontend/src/util/novncGestureCrashGuard.test.ts b/skyvern-frontend/src/util/novncGestureCrashGuard.test.ts
new file mode 100644
index 000000000..2f8739208
--- /dev/null
+++ b/skyvern-frontend/src/util/novncGestureCrashGuard.test.ts
@@ -0,0 +1,49 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import _GestureHandler from "@novnc/novnc/lib/input/gesturehandler.js";
+
+import { installNoVncGestureCrashGuard } from "./novncGestureCrashGuard";
+
+const GestureHandler =
+ (
+ _GestureHandler as typeof _GestureHandler & {
+ default?: typeof _GestureHandler;
+ }
+ ).default ?? _GestureHandler;
+
+describe("installNoVncGestureCrashGuard", () => {
+ beforeAll(() => {
+ installNoVncGestureCrashGuard();
+ });
+
+ it("ignores a touchend without a matching tracked touch", () => {
+ const handler = new GestureHandler();
+
+ expect(() => handler._touchEnd(1, 0, 0)).not.toThrow();
+ });
+
+ it("preserves normal tracked touch handling", () => {
+ const handler = new GestureHandler();
+ const target = document.createElement("div");
+
+ handler.attach(target);
+ handler._touchStart(1, 0, 0);
+ expect(() => handler._touchEnd(1, 0, 0)).not.toThrow();
+ });
+
+ it("preserves ignored-touch cleanup", () => {
+ const handler = new GestureHandler();
+ handler._ignored.push(1);
+
+ handler._touchEnd(1, 0, 0);
+
+ expect(handler._ignored).not.toContain(1);
+ });
+
+ it("does not wrap the handler more than once", () => {
+ const guardedTouchEnd = GestureHandler.prototype._touchEnd;
+
+ installNoVncGestureCrashGuard();
+
+ expect(GestureHandler.prototype._touchEnd).toBe(guardedTouchEnd);
+ });
+});
diff --git a/skyvern-frontend/src/util/novncGestureCrashGuard.ts b/skyvern-frontend/src/util/novncGestureCrashGuard.ts
new file mode 100644
index 000000000..d1673690d
--- /dev/null
+++ b/skyvern-frontend/src/util/novncGestureCrashGuard.ts
@@ -0,0 +1,29 @@
+import _GestureHandler from "@novnc/novnc/lib/input/gesturehandler.js";
+
+const GestureHandler =
+ (
+ _GestureHandler as typeof _GestureHandler & {
+ default?: typeof _GestureHandler;
+ }
+ ).default ?? _GestureHandler;
+
+let installed = false;
+
+export function installNoVncGestureCrashGuard(): void {
+ if (installed) return;
+ installed = true;
+
+ const originalTouchEnd = GestureHandler.prototype._touchEnd;
+ GestureHandler.prototype._touchEnd = function (
+ id: number,
+ x: number,
+ y: number,
+ ): void {
+ const isTracked = this._tracked.some((touch) => touch.id === id);
+ const isIgnored = this._ignored.includes(id);
+ // noVNC 1.5.0 dereferences the lookup even when a touch ends after the handler attached mid-gesture.
+ if (!isTracked && !isIgnored) return;
+
+ originalTouchEnd.call(this, id, x, y);
+ };
+}
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 57b7709c1..6ed907ecc 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -101,11 +101,13 @@ from skyvern.schemas.workflows import (
WorkflowCreateYAMLRequest,
WorkflowDefinitionYAML,
)
+from skyvern.utils.prompt_truncation import truncate_page_html_for_summary
from skyvern.utils.strings import escape_code_fences
from skyvern.utils.yaml_loader import safe_load_no_dates
WORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")
CHAT_HISTORY_CONTEXT_MESSAGES = 10
+WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS = 3_000_000
ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
"audio/mp4",
"audio/mpeg",
@@ -1004,6 +1006,21 @@ async def _get_debug_artifact(organization_id: str, workflow_run_id: str) -> Art
return artifacts[0] if isinstance(artifacts, list) and artifacts else None
+async def _get_debug_html(organization_id: str, workflow_run_id: str) -> str | None:
+ artifact = await _get_debug_artifact(organization_id, workflow_run_id)
+ if not artifact:
+ return None
+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
+ if not artifact_bytes:
+ return None
+ # The extreme-size cap can drop middle-tree evidence; move to structure-aware
+ # selection if that trade-off causes copilot regressions.
+ return truncate_page_html_for_summary(
+ artifact_bytes.decode("utf-8"),
+ max_chars=WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS,
+ )
+
+
async def _get_debug_run_info(organization_id: str, workflow_run_id: str | None) -> RunInfo | None:
if not workflow_run_id:
return None
@@ -1016,12 +1033,7 @@ async def _get_debug_run_info(organization_id: str, workflow_run_id: str | None)
block = blocks[0]
- artifact = await _get_debug_artifact(organization_id, workflow_run_id)
- if artifact:
- artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
- html = artifact_bytes.decode("utf-8") if artifact_bytes else None
- else:
- html = None
+ html = await _get_debug_html(organization_id, workflow_run_id)
return RunInfo(
block_label=block.label,
@@ -1063,11 +1075,7 @@ async def _get_new_copilot_block_infos(
)
)
- artifact = await _get_debug_artifact(organization_id, workflow_run_id)
- html: str | None = None
- if artifact:
- artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
- html = artifact_bytes.decode("utf-8") if artifact_bytes else None
+ html = await _get_debug_html(organization_id, workflow_run_id)
return block_infos, html
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 7e8534ec0..fa290fd80 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -77,6 +77,44 @@ def _make_chat_request(
)
+async def test_get_debug_run_info_bounds_visible_elements_html(monkeypatch: pytest.MonkeyPatch) -> None:
+ max_chars = workflow_copilot_route.WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS
+ html = "HEAD_MARKER" + ("x" * (max_chars * 2)) + "TAIL_MARKER"
+ artifact = SimpleNamespace()
+ block = SimpleNamespace(
+ label="block-1",
+ block_type=SimpleNamespace(name="task"),
+ status="failed",
+ failure_reason="timed out",
+ )
+ monkeypatch.setattr(
+ workflow_copilot_route,
+ "_get_debug_artifact",
+ AsyncMock(return_value=artifact),
+ )
+ monkeypatch.setattr(
+ app.DATABASE,
+ "observer",
+ SimpleNamespace(get_workflow_run_blocks=AsyncMock(return_value=[block])),
+ )
+ monkeypatch.setattr(
+ app,
+ "ARTIFACT_MANAGER",
+ SimpleNamespace(retrieve_artifact=AsyncMock(return_value=html.encode("utf-8"))),
+ )
+
+ run_info = await workflow_copilot_route._get_debug_run_info("org-1", "wr-1")
+
+ assert run_info is not None
+ assert run_info.html is not None
+ assert len(run_info.html) < len(html)
+ assert len(run_info.html) <= max_chars + 100
+ assert "HEAD_MARKER" in run_info.html
+ assert "TAIL_MARKER" in run_info.html
+ assert "truncated by Skyvern" in run_info.html
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("finalizer", ["normal", "cancel"], ids=["normal-turn", "cancel-turn"])
@pytest.mark.parametrize("destination", ["persistence", "sse", "history"])