Skip to content

Agent Zero AI framework

Harden desktop session reliability

Alessandro07e7e1e07554
diff --git a/plugins/_desktop/helpers/desktop_session.py b/plugins/_desktop/helpers/desktop_session.py
index 4acdab0cc..d538db037 100644
--- a/plugins/_desktop/helpers/desktop_session.py
+++ b/plugins/_desktop/helpers/desktop_session.py
@@ -5,7 +5,6 @@ import fcntl
import hashlib
import json
import os
-import re
import shutil
import socket
import subprocess
@@ -223,39 +222,69 @@ class DesktopSessionManager:
return {"ok": True, "refreshed": refreshed, "desktop": session.public(doc)}
def save(self, session_id: str, file_id: str = "") -> dict[str, Any]:
- session = self.require(session_id)
- doc = self._document_for_save(session, file_id)
- xdotool = shutil.which("xdotool")
- if not xdotool:
- updated = document_store.register_document(doc["path"]) if doc else None
- return {
- "ok": False,
- "error": "xdotool is not installed; use LibreOffice's Save control inside the canvas.",
- "document": _public_doc(updated) if updated else None,
- }
+ with self._lock:
+ session = self.require(session_id)
+ doc = self._document_for_save(session, file_id)
+ if not doc:
+ return {
+ "ok": True,
+ "session_id": session.session_id,
+ "document": None,
+ "changed": False,
+ }
- result = subprocess.run(
- [xdotool, "key", "--clearmodifiers", "ctrl+s"],
- check=False,
- capture_output=True,
- text=True,
- timeout=8,
- env=self._display_env(session),
- )
- time.sleep(0.8)
- updated = document_store.register_document(doc["path"]) if doc else None
- if result.returncode != 0:
- detail = (result.stderr or result.stdout or "").strip()
+ xdotool = shutil.which("xdotool")
+ if not xdotool:
+ updated = document_store.register_document(doc["path"])
+ return {
+ "ok": False,
+ "error": "xdotool is not installed; use LibreOffice's Save control inside the canvas.",
+ "document": _public_doc(updated),
+ }
+
+ window_id = self._office_window_id_locked(
+ session,
+ title=str(doc.get("basename") or Path(doc["path"]).name),
+ fallback=False,
+ )
+ if not window_id:
+ return {
+ "ok": False,
+ "error": f"LibreOffice window not found for {doc['basename']}.",
+ "document": _public_doc(doc),
+ }
+
+ result = subprocess.run(
+ [
+ xdotool,
+ "windowactivate",
+ "--sync",
+ window_id,
+ "key",
+ "--clearmodifiers",
+ "ctrl+s",
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=8,
+ env=self._display_env(session),
+ )
+ time.sleep(0.8)
+ updated = document_store.register_document(doc["path"])
+ if result.returncode != 0:
+ detail = (result.stderr or result.stdout or "").strip()
+ return {
+ "ok": False,
+ "error": detail or "LibreOffice desktop save shortcut failed.",
+ "document": _public_doc(updated),
+ }
return {
- "ok": False,
- "error": detail or "LibreOffice desktop save shortcut failed.",
- "document": _public_doc(updated) if updated else None,
+ "ok": True,
+ "session_id": session.session_id,
+ "document": _public_doc(updated),
+ "changed": updated.get("sha256") != doc.get("sha256"),
}
- return {
- "ok": True,
- "session_id": session.session_id,
- "document": _public_doc(updated) if updated else None,
- }
def sync(self, session_id: str = "", file_id: str = "") -> dict[str, Any]:
session = self.get(session_id) if session_id else self._find_by_file_id(file_id)
@@ -820,57 +849,48 @@ class DesktopSessionManager:
return ""
env = self._display_env(session)
title = str(title or "").strip()
- searches: list[list[str]] = []
+ try:
+ result = subprocess.run(
+ [xdotool, "search", "--onlyvisible", "--class", "libreoffice"],
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=2,
+ env=env,
+ )
+ window_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
+ except (OSError, subprocess.TimeoutExpired):
+ window_ids = []
if title:
- escaped_title = re.escape(title)
- searches.append([
- xdotool,
- "search",
- "--onlyvisible",
- "--name",
- escaped_title,
- ])
- for window_class in (
- "libreoffice",
- "libreoffice-writer",
- "libreoffice-calc",
- "libreoffice-impress",
- ):
- searches.append([
- xdotool,
- "search",
- "--onlyvisible",
- "--class",
- window_class,
- "--name",
- escaped_title,
- ])
+ for window_id in reversed(window_ids):
+ try:
+ result = subprocess.run(
+ [xdotool, "getwindowname", window_id],
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=2,
+ env=env,
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ continue
+ if title.casefold() in result.stdout.strip().casefold():
+ return window_id
+ if fallback and window_ids:
+ return window_ids[-1]
if fallback:
- for window_class in (
- "libreoffice",
- "libreoffice-writer",
- "libreoffice-calc",
- "libreoffice-impress",
- ):
- searches.append([xdotool, "search", "--onlyvisible", "--class", window_class])
- searches.append([xdotool, "search", "--onlyvisible", "--name", "LibreOffice"])
- for command in searches:
try:
result = subprocess.run(
- command,
+ [xdotool, "search", "--onlyvisible", "--name", "LibreOffice"],
check=False,
capture_output=True,
text=True,
timeout=2,
env=env,
)
+ window_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
except (OSError, subprocess.TimeoutExpired):
- continue
- window_ids = [
- line.strip()
- for line in result.stdout.splitlines()
- if line.strip()
- ]
+ window_ids = []
if window_ids:
return window_ids[-1]
return ""
@@ -1573,14 +1593,20 @@ fi
self._remove_manifest(session_id)
def _wait_for_display(self, session: DesktopSession) -> None:
- marker = Path(f"/tmp/.X11-unix/X{session.display}")
deadline = time.time() + DISPLAY_START_TIMEOUT_SECONDS
while time.time() < deadline:
process = session.processes.get("xvfb") or session.processes.get("xpra")
if process and process.poll() is not None:
raise RuntimeError("The LibreOffice X display exited before it was ready.")
- if marker.exists():
- return
+ try:
+ if virtual_desktop.current_display_size(
+ session.display,
+ xauthority=self._xauthority(session),
+ home=str(session.profile_dir),
+ ):
+ return
+ except (OSError, subprocess.TimeoutExpired):
+ pass
time.sleep(0.1)
raise TimeoutError("Timed out waiting for the LibreOffice X display.")
@@ -1589,7 +1615,7 @@ fi
while time.time() < deadline:
process = session.processes.get("xfce")
if process and process.poll() is not None:
- return
+ raise RuntimeError("The XFCE desktop session exited before it was ready.")
if virtual_desktop.has_window(
display=session.display,
name="xfce4-panel",
@@ -1619,7 +1645,13 @@ fi
"owner_pid": os.getpid(),
"pids": pids,
}
- (SESSION_DIR / f"{session.session_id}.json").write_text(json.dumps(payload), encoding="utf-8")
+ manifest = SESSION_DIR / f"{session.session_id}.json"
+ tmp = manifest.with_name(f".{manifest.name}.{uuid.uuid4().hex}.tmp")
+ try:
+ tmp.write_text(json.dumps(payload), encoding="utf-8")
+ os.replace(tmp, manifest)
+ finally:
+ tmp.unlink(missing_ok=True)
def _remove_manifest(self, session_id: str) -> None:
(SESSION_DIR / f"{session_id}.json").unlink(missing_ok=True)
diff --git a/plugins/_desktop/helpers/desktop_state.py b/plugins/_desktop/helpers/desktop_state.py
index cbfbb7a55..50bba6538 100644
--- a/plugins/_desktop/helpers/desktop_state.py
+++ b/plugins/_desktop/helpers/desktop_state.py
@@ -262,50 +262,43 @@ def convert_xwd_to_image(raw_path: Path, target: Path) -> dict[str, int]:
from PIL import Image
data = raw_path.read_bytes()
- header, endian = parse_xwd_header(data)
+ header, _ = parse_xwd_header(data)
width = header["pixmap_width"]
height = header["pixmap_height"]
bytes_per_line = header["bytes_per_line"]
bits_per_pixel = header["bits_per_pixel"]
- image_byte_order = "little" if header["byte_order"] == 0 else "big"
color_table_size = header["ncolors"] * 12
pixel_offset = header["header_size"] + color_table_size
- bytes_per_pixel = max((bits_per_pixel + 7) // 8, 1)
- if width > 0 and bytes_per_line % width == 0:
- bytes_per_pixel = max(bytes_per_pixel, bytes_per_line // width)
if width <= 0 or height <= 0 or bytes_per_line <= 0:
raise ValueError("invalid XWD dimensions")
- if pixel_offset + (height * bytes_per_line) > len(data):
+ pixel_size = height * bytes_per_line
+ if pixel_offset + pixel_size > len(data):
raise ValueError("truncated XWD pixel data")
- red_mask = header["red_mask"]
- green_mask = header["green_mask"]
- blue_mask = header["blue_mask"]
- red_shift, red_bits = mask_shift_and_bits(red_mask)
- green_shift, green_bits = mask_shift_and_bits(green_mask)
- blue_shift, blue_bits = mask_shift_and_bits(blue_mask)
- if min(red_bits, green_bits, blue_bits) <= 0:
+ if (header["red_mask"], header["green_mask"], header["blue_mask"]) != (
+ 0x00FF0000,
+ 0x0000FF00,
+ 0x000000FF,
+ ):
raise ValueError("unsupported XWD visual masks")
-
- pixels: list[tuple[int, int, int]] = []
- for row in range(height):
- row_start = pixel_offset + (row * bytes_per_line)
- for column in range(width):
- start = row_start + (column * bytes_per_pixel)
- pixel_bytes = data[start : start + bytes_per_pixel]
- if len(pixel_bytes) < bytes_per_pixel:
- raise ValueError("truncated XWD pixel")
- pixel = int.from_bytes(pixel_bytes, image_byte_order, signed=False)
- pixels.append(
- (
- scale_channel((pixel & red_mask) >> red_shift, red_bits),
- scale_channel((pixel & green_mask) >> green_shift, green_bits),
- scale_channel((pixel & blue_mask) >> blue_shift, blue_bits),
- ),
- )
-
- image = Image.new("RGB", (width, height))
- image.putdata(pixels)
+ raw_mode = {
+ (24, 0): "BGR",
+ (24, 1): "RGB",
+ (32, 0): "BGRX",
+ (32, 1): "XRGB",
+ }.get((bits_per_pixel, header["byte_order"]))
+ if not raw_mode:
+ raise ValueError(f"unsupported XWD pixel layout: {bits_per_pixel} bpp")
+
+ image = Image.frombytes(
+ "RGB",
+ (width, height),
+ data[pixel_offset : pixel_offset + pixel_size],
+ "raw",
+ raw_mode,
+ bytes_per_line,
+ 1,
+ )
image.save(target)
return {"width": width, "height": height}
@@ -348,28 +341,6 @@ def parse_xwd_header(data: bytes) -> tuple[dict[str, int], str]:
raise ValueError("unsupported XWD header")
-def mask_shift_and_bits(mask: int) -> tuple[int, int]:
- if mask <= 0:
- return 0, 0
- shift = 0
- value = mask
- while value and value & 1 == 0:
- shift += 1
- value >>= 1
- bits = 0
- while value & 1:
- bits += 1
- value >>= 1
- return shift, bits
-
-
-def scale_channel(value: int, bits: int) -> int:
- if bits >= 8:
- return max(0, min(255, value >> (bits - 8)))
- max_value = (1 << bits) - 1
- return 0 if max_value <= 0 else round((value / max_value) * 255)
-
-
def resolve_environment(*, errors: list[str] | None = None, session_id: str = SESSION_ID) -> dict[str, str]:
local_errors = errors if errors is not None else []
manifest = session_manifest_path(session_id)
diff --git a/tests/test_office_desktop_state.py b/tests/test_office_desktop_state.py
index 12ae906a9..506b277b9 100644
--- a/tests/test_office_desktop_state.py
+++ b/tests/test_office_desktop_state.py
@@ -6,6 +6,8 @@ import sys
import types
from pathlib import Path
+import pytest
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
@@ -324,7 +326,16 @@ def test_virtual_desktop_system_display_normalization_rejects_portrait_viewports
assert virtual_desktop.normalize_desktop_display_size(1600, 900) == (1600, 900)
-def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):
+ ("byte_order", "pixel_bytes"),
+ (
+ (0, bytes.fromhex("0000ff00") + bytes.fromhex("00ff0000")),
+ (1, bytes.fromhex("00ff0000") + bytes.fromhex("0000ff00")),
+ ),
+)
+def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, byte_order, pixel_bytes):
+ from PIL import Image
+
raw_path = tmp_path / "shot.xwd"
target = tmp_path / "shot.png"
header_values = [
@@ -335,7 +346,7 @@ def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):
2, # pixmap_width
1, # pixmap_height
0, # xoffset
- 1, # byte_order: MSBFirst for pixel bytes
+ byte_order,
32, # bitmap_unit
1, # bitmap_bit_order
32, # bitmap_pad
@@ -354,38 +365,12 @@ def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):
0, # window_y
0, # window_bdrwidth
]
- raw_path.write_bytes(
- struct.pack(">25I", *header_values)
- + bytes.fromhex("00ff0000")
- + bytes.fromhex("0000ff00")
- )
-
- captured: dict[str, object] = {}
- image_module = types.ModuleType("PIL.Image")
-
- class FakeOutputImage:
- def putdata(self, pixels):
- captured["pixels"] = list(pixels)
-
- def save(self, path):
- Path(path).write_bytes(b"fallback-png")
-
- def fake_new(mode, size):
- captured["mode"] = mode
- captured["size"] = size
- return FakeOutputImage()
-
- image_module.new = fake_new
- pil_module = types.ModuleType("PIL")
- pil_module.Image = image_module
-
- monkeypatch.setitem(sys.modules, "PIL", pil_module)
- monkeypatch.setitem(sys.modules, "PIL.Image", image_module)
+ raw_path.write_bytes(struct.pack(">25I", *header_values) + pixel_bytes)
converted = desktop_state.convert_xwd_to_image(raw_path, target)
assert converted == {"width": 2, "height": 1}
- assert captured["mode"] == "RGB"
- assert captured["size"] == (2, 1)
- assert captured["pixels"] == [(255, 0, 0), (0, 255, 0)]
- assert target.read_bytes() == b"fallback-png"
+ with Image.open(target) as image:
+ assert image.mode == "RGB"
+ assert image.size == (2, 1)
+ assert list(image.getdata()) == [(255, 0, 0), (0, 255, 0)]
diff --git a/tests/test_office_document_store.py b/tests/test_office_document_store.py
index 011e619ed..7c4e8850c 100644
--- a/tests/test_office_document_store.py
+++ b/tests/test_office_document_store.py
@@ -1037,6 +1037,123 @@ def test_official_desktop_session_manager_opens_binary_session(office_state, tmp
assert manager.close(payload["session_id"], save_first=False)["persistent"] is True
+def test_desktop_save_targets_requested_libreoffice_window(office_state, tmp_path, monkeypatch):
+ doc = document_store.create_document("spreadsheet", "Targeted Save", "ods", "Name,Value\nA,1")
+ session = desktop_session.DesktopSession(
+ session_id=desktop_session.SYSTEM_SESSION_ID,
+ file_id=doc["file_id"],
+ extension=doc["extension"],
+ path=doc["path"],
+ title=doc["basename"],
+ display=desktop_session.DISPLAY_BASE,
+ xpra_port=desktop_session.XPRA_PORT_BASE,
+ token=desktop_session.SYSTEM_SESSION_ID,
+ url="/desktop/session/agent-zero-desktop/index.html",
+ profile_dir=tmp_path / "profile",
+ processes={"xpra": types.SimpleNamespace(poll=lambda: None)},
+ )
+ manager = desktop_session.DesktopSessionManager()
+ manager._sessions[session.session_id] = session
+ commands = []
+
+ monkeypatch.setattr(desktop_session.shutil, "which", lambda name: f"/usr/bin/{name}")
+ monkeypatch.setattr(desktop_session.time, "sleep", lambda _seconds: None)
+
+ def fake_run(command, **_kwargs):
+ commands.append(command)
+ if command[1] == "search":
+ assert command == ["/usr/bin/xdotool", "search", "--onlyvisible", "--class", "libreoffice"]
+ return subprocess.CompletedProcess(command, 0, "222\n", "")
+ if command[1] == "getwindowname":
+ return subprocess.CompletedProcess(command, 0, f"{doc['basename']} — LibreOffice Calc\n", "")
+ Path(doc["path"]).write_bytes(Path(doc["path"]).read_bytes() + b"changed")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ monkeypatch.setattr(desktop_session.subprocess, "run", fake_run)
+
+ result = manager.save(session.session_id, doc["file_id"])
+
+ assert result["ok"] is True
+ assert result["changed"] is True
+ assert commands[-1] == [
+ "/usr/bin/xdotool",
+ "windowactivate",
+ "--sync",
+ "222",
+ "key",
+ "--clearmodifiers",
+ "ctrl+s",
+ ]
+
+ session.file_id = desktop_session.SYSTEM_FILE_ID
+ command_count = len(commands)
+ assert manager.save(session.session_id)["changed"] is False
+ assert len(commands) == command_count
+
+
+def test_desktop_startup_waiters_probe_display_and_reject_dead_xfce(tmp_path, monkeypatch):
+ session = desktop_session.DesktopSession(
+ session_id=desktop_session.SYSTEM_SESSION_ID,
+ file_id=desktop_session.SYSTEM_FILE_ID,
+ extension="desktop",
+ path=str(tmp_path),
+ title=desktop_session.SYSTEM_TITLE,
+ display=desktop_session.DISPLAY_BASE,
+ xpra_port=desktop_session.XPRA_PORT_BASE,
+ token=desktop_session.SYSTEM_SESSION_ID,
+ url="/desktop/session/agent-zero-desktop/index.html",
+ profile_dir=tmp_path / "profile",
+ processes={"xvfb": types.SimpleNamespace(poll=lambda: None)},
+ )
+ manager = desktop_session.DesktopSessionManager()
+ probes = iter((None, (1920, 1080)))
+ monkeypatch.setattr(
+ desktop_session.virtual_desktop,
+ "current_display_size",
+ lambda *_args, **_kwargs: next(probes),
+ )
+ monkeypatch.setattr(desktop_session.time, "sleep", lambda _seconds: None)
+
+ manager._wait_for_display(session)
+
+ session.processes = {"xfce": types.SimpleNamespace(poll=lambda: 1)}
+ with pytest.raises(RuntimeError, match="XFCE desktop session exited"):
+ manager._wait_for_xfce(session)
+
+
+def test_desktop_manifest_is_replaced_atomically(tmp_path, monkeypatch):
+ session_dir = tmp_path / "sessions"
+ session_dir.mkdir()
+ manifest = session_dir / f"{desktop_session.SYSTEM_SESSION_ID}.json"
+ manifest.write_text('{"old": true}', encoding="utf-8")
+ session = desktop_session.DesktopSession(
+ session_id=desktop_session.SYSTEM_SESSION_ID,
+ file_id=desktop_session.SYSTEM_FILE_ID,
+ extension="desktop",
+ path=str(tmp_path),
+ title=desktop_session.SYSTEM_TITLE,
+ display=desktop_session.DISPLAY_BASE,
+ xpra_port=desktop_session.XPRA_PORT_BASE,
+ token=desktop_session.SYSTEM_SESSION_ID,
+ url="/desktop/session/agent-zero-desktop/index.html",
+ profile_dir=tmp_path / "profile",
+ )
+ real_replace = os.replace
+
+ def assert_atomic_replace(source, destination):
+ assert json.loads(Path(destination).read_text(encoding="utf-8")) == {"old": True}
+ assert json.loads(Path(source).read_text(encoding="utf-8"))["display"] == session.display
+ real_replace(source, destination)
+
+ monkeypatch.setattr(desktop_session, "SESSION_DIR", session_dir)
+ monkeypatch.setattr(desktop_session.os, "replace", assert_atomic_replace)
+
+ desktop_session.DesktopSessionManager()._write_manifest(session)
+
+ assert json.loads(manifest.read_text(encoding="utf-8"))["session_id"] == session.session_id
+ assert list(session_dir.glob(".*.tmp")) == []
+
+
def test_shutdown_panel_launcher_requires_second_click(tmp_path):
profile_dir = tmp_path / "desktop" / "profiles" / desktop_session.SYSTEM_SESSION_ID
profile_dir.mkdir(parents=True)