Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: replace Qt and debug viewer wiring with native visual admission
alexshpunt-clanker[bot]7f59d153182b
diff --git a/native/window-overlay/CMakeLists.txt b/native/window-overlay/CMakeLists.txtdeleted file mode 100644index 4dfc4e8..0000000--- a/native/window-overlay/CMakeLists.txt+++ /dev/null@@ -1,13 +0,0 @@-cmake_minimum_required(VERSION 3.22)-project(PiWindowOverlay VERSION 0.1.0 LANGUAGES CXX)--if(NOT WIN32)- message(FATAL_ERROR "This helper is qualified only on Windows.")-endif()-find_package(Qt6 6.11 REQUIRED COMPONENTS Widgets)-qt_add_executable(pi-window-overlay WIN32 main.cpp)-target_compile_features(pi-window-overlay PRIVATE cxx_std_17)-target_compile_definitions(pi-window-overlay PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX)-target_link_libraries(pi-window-overlay PRIVATE Qt6::Widgets user32 dwmapi)-install(TARGETS pi-window-overlay RUNTIME DESTINATION .)-install(FILES qt.conf DESTINATION .)\ No newline at end of filediff --git a/native/window-overlay/NOTICE.md b/native/window-overlay/NOTICE.mddeleted file mode 100644index 82a422c..0000000--- a/native/window-overlay/NOTICE.md+++ /dev/null@@ -1,15 +0,0 @@-# Qt notice and replacement instructions--This helper dynamically uses Qt 6.11.3 Core, Gui, Widgets and the Windows QPA plugin. Qt is copyright The Qt Company Ltd. and other contributors. These Qt components are used under the LGPL version 3 option. Copies of LGPLv3 and GPLv3 are in `licenses/`. The original Qt DLLs are not patched.--The release directory contains `qtbase-source.tar.gz` from the exact source revision named in the installed Qt kit's SBOM: `5a1194b2d368e2d72c230205c464b04a2f23eccd`. It includes the Qt build files and third-party sources. `qt-kit-qconfig.pri` records the installed kit's build settings. `helper-source/` contains the helper's source and CMake build file. Keep these source files available alongside the runtime when supplying this bundle to someone else; a link to upstream alone is not our source-delivery method.--Qt also includes third-party code with its own terms. `licenses/qtbase-sbom.json`, `qtbase-attributions.json` and the unmodified license texts preserve a conservative set of qtbase notices. This set includes some components not linked by this helper. The corresponding source archive keeps their original copyright notices. The Qt Windows entry point has separate BSD-style terms in that source tree.--## Replace Qt--Close the helper before replacing files. Keep the DLL names and compatible interfaces, then replace `Qt6Core.dll`, `Qt6Gui.dll`, `Qt6Widgets.dll` and `platforms/qwindows.dll` with your compatible modified builds. `qt.conf` loads plugins beside the executable. The helper does not enforce DLL hashes, signatures or a private Qt loader. No restriction is imposed on replacing Qt or reverse engineering needed to debug Qt modifications.--Build Qt with its supplied CMake/configure files for Windows x64 and a compatible MSVC toolchain. Build the helper with `scripts/build-window-overlay.ps1 -QtPrefix <your-compatible-Qt-kit>`. Use the new helper through the trusted `PI_WINDOW_OVERLAY_PATH` environment variable or install its runtime directory in the documented user-local location. No service, administrator account or locked installation is required.--This is a private local Windows deployment, not a public binary release or a legal compliance opinion. Public redistribution needs a review of the helper's own licensing terms, the complete corresponding source and build information, all third-party terms, and the Microsoft runtime distribution rules. Dynamic linking by itself does not remove those duties.diff --git a/native/window-overlay/main.cpp b/native/window-overlay/main.cppdeleted file mode 100644index d3d3941..0000000--- a/native/window-overlay/main.cpp+++ /dev/null@@ -1,244 +0,0 @@-#include <windows.h>-#include <dwmapi.h>-#include <QApplication>-#include <QElapsedTimer>-#include <QJsonDocument>-#include <QJsonObject>-#include <QPainter>-#include <QRegularExpression>-#include <QTimer>-#include <QWidget>-#include <cmath>--namespace {-constexpr DWORD captureExcluded = 0x11;-constexpr int flashMilliseconds = 2000;-constexpr int maximumMessage = 16384;--void reply(const QJsonObject &value) {- const QByteArray line = QJsonDocument(value).toJson(QJsonDocument::Compact) + '\n';- DWORD written = 0;- if (!WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), line.constData(), DWORD(line.size()), &written, nullptr)- || written != DWORD(line.size())) QCoreApplication::quit();-}--bool integer(const QJsonValue &value, qint64 low, qint64 high) {- const double number = value.toDouble(std::nan(""));- return std::isfinite(number) && std::floor(number) == number && number >= low && number <= high;-}--// This window draws only. Native APIs here read identity/focus metadata or configure our own HWND.-class Frame final : public QWidget {-public:- Frame() : QWidget(nullptr, Qt::Tool | Qt::FramelessWindowHint | Qt::WindowStaysOnTopHint- | Qt::WindowTransparentForInput | Qt::WindowDoesNotAcceptFocus) {- setAttribute(Qt::WA_TranslucentBackground);- setAttribute(Qt::WA_ShowWithoutActivating);- setFocusPolicy(Qt::NoFocus);- handle = reinterpret_cast<HWND>(winId());- const LONG_PTR styles = GetWindowLongPtrW(handle, GWL_EXSTYLE);- SetWindowLongPtrW(handle, GWL_EXSTYLE, styles | WS_EX_NOACTIVATE | WS_EX_TRANSPARENT | WS_EX_TOOLWINDOW);- DWORD affinity = 0;- excluded = SetWindowDisplayAffinity(handle, captureExcluded)- && GetWindowDisplayAffinity(handle, &affinity) && affinity == captureExcluded;- connect(&watch, &QTimer::timeout, this, [this] {- if (isVisible() && (age.elapsed() >= flashMilliseconds || !sameSource())) hide();- });- watch.start(40);- }-- bool ready() const { return excluded; }- quintptr windowId() const { return reinterpret_cast<quintptr>(handle); }-- void command(const QJsonObject &message) {- const auto number = message.value("generation");- if (!integer(number, 1, 9007199254740991LL)) { hide(); reply({{"status", "refused"}, {"generation", 0}}); return; }- const qint64 next = qint64(number.toDouble());- if (next <= generation) { reply({{"status", "stale"}, {"generation", double(next)}}); return; }- generation = next;- hide();- const QString kind = message.value("kind").toString();- if (kind == "hide") { reply({{"status", "hidden"}, {"generation", double(next)}}); return; }- const QJsonObject bounds = message.value("bounds").toObject();- if (kind != "flash" || !excluded || message.value("duration_ms").toInt() != flashMilliseconds || !rectangle(bounds)) {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "invalid_frame"}}); return;- }- physical = QRect(bounds.value("x").toInt(), bounds.value("y").toInt(), bounds.value("width").toInt(), bounds.value("height").toInt());- display = message.value("display").toString() == "primary";- if (display) {- if (message.contains("target") || message.contains("identity") || !primaryBounds(&visibleSource)) {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "display_geometry"}}); return;- }- source = nullptr;- } else {- const QJsonObject target = message.value("target").toObject();- const auto marker = QRegularExpression("^(PiComputerUse\\.[a-fA-F0-9]{32}):([1-9][0-9]*):([1-9][0-9]*)$").match(message.value("identity").toString());- if (!integer(target.value("pid"), 1, MAXDWORD) || !integer(target.value("window_id"), 1, 9007199254740991LL)- || !marker.hasMatch() || marker.captured(2).toULongLong() != quint64(target.value("pid").toDouble())) {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "source_identity"}}); return;- }- source = reinterpret_cast<HWND>(quintptr(target.value("window_id").toDouble()));- pid = DWORD(target.value("pid").toDouble());- key = marker.captured(1).toStdWString();- bool markerValid = false;- markerValue = marker.captured(3).toULongLong(&markerValid);- if (!markerValid || markerValue == 0 || markerValue > 9007199254740991ULL || !GetWindowRect(source, &original)- || FAILED(DwmGetWindowAttribute(source, DWMWA_EXTENDED_FRAME_BOUNDS, &visibleSource, sizeof(visibleSource))) || !sameSource()) {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "source_lifetime"}}); return;- }- }- // Window scope is CUA's DWM visible frame; display scope is the physical primary monitor.- if (visibleSource.left != physical.x() || visibleSource.top != physical.y()- || visibleSource.right - visibleSource.left != physical.width() || visibleSource.bottom - visibleSource.top != physical.height()) {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "source_geometry"}}); return;- }- const QJsonObject mark = message.value("mark").toObject();- style = mark.isEmpty() ? "overview" : mark.value("kind").toString();- hasPoint = mark.contains("point");- drawing = physical; point = {};- if (style != "overview" && style != "detail" && style != "action") {- reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "invalid_mark"}}); return;- }- if (style == "overview") {- if (mark.contains("region") || hasPoint) { reply({{"status", "refused"}, {"generation", double(next)}}); return; }- } else if (mark.contains("region") && !hasPoint) {- const QJsonObject r = mark.value("region").toObject();- if (!rectangle(r)) { reply({{"status", "refused"}, {"generation", double(next)}}); return; }- drawing = QRect(r.value("x").toInt(), r.value("y").toInt(), r.value("width").toInt(), r.value("height").toInt());- if (!physical.contains(drawing)) { reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "mark_scope"}}); return; }- } else if (style == "action" && hasPoint && !mark.contains("region")) {- const QJsonObject p = mark.value("point").toObject();- if (!integer(p.value("x"), -2000000, 2000000) || !integer(p.value("y"), -2000000, 2000000)) {- reply({{"status", "refused"}, {"generation", double(next)}}); return;- }- point = QPoint(p.value("x").toInt(), p.value("y").toInt());- if (!physical.contains(point)) { reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "mark_scope"}}); return; }- } else { reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "missing_mark_geometry"}}); return; }- canvas = {}; // Do not paint while Qt is still negotiating the native canvas size.- const HWND foreground = GetForegroundWindow();- GUITHREADINFO focusBefore{}; focusBefore.cbSize = sizeof(focusBefore);- const bool focusRead = GetGUIThreadInfo(0, &focusBefore);- show();- // Qt's public geometry uses logical pixels. Position this DPI-aware HWND in the physical CUA domain.- const bool positioned = SetWindowPos(handle, HWND_TOPMOST, bounds.value("x").toInt(), bounds.value("y").toInt(),- bounds.value("width").toInt(), bounds.value("height").toInt(), SWP_NOACTIVATE | SWP_SHOWWINDOW);- DWORD affinity = 0; RECT actual{};- GUITHREADINFO focusAfter{}; focusAfter.cbSize = sizeof(focusAfter);- const bool focusUnchanged = focusRead && GetGUIThreadInfo(0, &focusAfter)- && focusBefore.hwndFocus == focusAfter.hwndFocus && focusBefore.hwndActive == focusAfter.hwndActive;- const bool correct = positioned && focusUnchanged && GetWindowRect(handle, &actual) && GetWindowDisplayAffinity(handle, &affinity)- && affinity == captureExcluded && GetForegroundWindow() == foreground- && actual.left == bounds.value("x").toInt() && actual.top == bounds.value("y").toInt()- && actual.right - actual.left >= physical.width() && actual.bottom - actual.top >= physical.height()- && actual.right - actual.left - physical.width() <= int(std::ceil(devicePixelRatioF())) - 1- && actual.bottom - actual.top - physical.height() <= int(std::ceil(devicePixelRatioF())) - 1;- if (!correct) {- hide(); reply({{"status", "refused"}, {"generation", double(next)}, {"reason", "native_show_check"},- {"positioned", positioned}, {"focus_unchanged", focusUnchanged}, {"foreground_unchanged", GetForegroundWindow() == foreground},- {"capture_affinity", double(affinity)}, {"actual", QJsonObject{{"x", int(actual.left)}, {"y", int(actual.top)},- {"width", int(actual.right - actual.left)}, {"height", int(actual.bottom - actual.top)}}}}); return;- }- canvas = QSize(int(actual.right - actual.left), int(actual.bottom - actual.top));- age.restart(); update();- reply({{"status", "shown"}, {"generation", double(next)}, {"overlay_hwnd", double(windowId())},- {"capture_affinity", double(affinity)}, {"bounds", bounds}, {"focus_unchanged", focusUnchanged},- {"foreground_unchanged", true}, {"duration_ms", flashMilliseconds}, {"mark", mark}, {"style", style}});- }--protected:- void paintEvent(QPaintEvent *) override {- if (canvas.isEmpty()) return;- QPainter painter(this);- // QWidget painting is logical-pixel based. Match the physical HWND positioned below,- // including windows on monitors with a different DPI; never reuse desktop coordinates as Qt geometry.- // Qt may round the helper canvas outwards by a physical pixel at fractional DPI.- // Map painting against its actual native extent and clip to the exact source, not the overscan.- painter.scale(double(width()) / canvas.width(), double(height()) / canvas.height());- painter.setClipRect(QRect(0, 0, physical.width(), physical.height()));- painter.setRenderHint(QPainter::Antialiasing);- const QColor color = style == "action" ? QColor(255, 70, 190, 245) : style == "detail" ? QColor(255, 185, 30, 245) : QColor(30, 200, 255, 245);- painter.setPen(QPen(color, 3));- painter.setBrush(Qt::NoBrush);- const QPoint offset = physical.topLeft();- if (hasPoint) {- const QPointF p = point - offset;- painter.drawEllipse(p, 12, 12);- painter.drawLine(p + QPointF(-18, 0), p + QPointF(18, 0));- painter.drawLine(p + QPointF(0, -18), p + QPointF(0, 18));- } else {- painter.drawRect(QRectF(drawing.translated(-offset)).adjusted(2, 2, -2, -2));- }- const QPoint at = hasPoint ? point - offset + QPoint(16, 16) : drawing.topLeft() - offset + QPoint(8, 8);- const QRectF label(qBound(4, at.x(), qMax(4, physical.width() - 140)), qBound(4, at.y(), qMax(4, physical.height() - 26)), 136, 22);- painter.fillRect(label, QColor(15, 15, 20, 210));- painter.drawText(label.adjusted(5, 0, -3, 0), Qt::AlignVCenter | Qt::AlignLeft, style == "action" ? "Action target" : style == "detail" ? "Zoom / crop" : "Observe");- }- bool nativeEvent(const QByteArray &, void *message, qintptr *result) override {- const MSG *event = static_cast<MSG *>(message);- if (event->message == WM_MOUSEACTIVATE) { *result = MA_NOACTIVATE; return true; }- if (event->message == WM_NCHITTEST) { *result = HTTRANSPARENT; return true; }- return false;- }--private:- static bool rectangle(const QJsonObject &r) {- return integer(r.value("x"), -2000000, 2000000) && integer(r.value("y"), -2000000, 2000000)- && integer(r.value("width"), 1, 32768) && integer(r.value("height"), 1, 32768);- }- static bool primaryBounds(RECT *bounds) {- const HMONITOR monitor = MonitorFromPoint(POINT{0, 0}, MONITOR_DEFAULTTOPRIMARY);- MONITORINFO info{}; info.cbSize = sizeof(info);- if (!monitor || !GetMonitorInfoW(monitor, &info) || !(info.dwFlags & MONITORINFOF_PRIMARY)) return false;- *bounds = info.rcMonitor; return true;- }- bool sameSource() const {- if (display) { RECT bounds{}; return primaryBounds(&bounds) && EqualRect(&visibleSource, &bounds); }- DWORD currentPid = 0; RECT current{}; RECT visible{};- return source && IsWindow(source) && !IsIconic(source) && GetWindowThreadProcessId(source, ¤tPid)- && currentPid == pid && reinterpret_cast<quintptr>(GetPropW(source, key.c_str())) == markerValue- && GetWindowRect(source, ¤t) && EqualRect(&original, ¤t)- && SUCCEEDED(DwmGetWindowAttribute(source, DWMWA_EXTENDED_FRAME_BOUNDS, &visible, sizeof(visible)))- && EqualRect(&visibleSource, &visible);- }- HWND handle{}; HWND source{}; DWORD pid{}; quintptr markerValue{};- std::wstring key; RECT original{}; RECT visibleSource{}; qint64 generation{};- bool excluded = false; bool display = false; bool hasPoint = false;- QRect physical; QRect drawing; QSize canvas; QPoint point; QString style;- QTimer watch; QElapsedTimer age;-};-}--int main(int argc, char **argv) {- SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);- QApplication application(argc, argv);- application.setQuitOnLastWindowClosed(false);- Frame frame;- const HANDLE input = GetStdHandle(STD_INPUT_HANDLE);- if (!frame.ready() || GetFileType(input) != FILE_TYPE_PIPE) {- reply({{"status", "unavailable"}, {"generation", 0}}); return 1;- }- reply({{"status", "ready"}, {"generation", 0}, {"pid", double(GetCurrentProcessId())},- {"overlay_hwnd", double(frame.windowId())}, {"qt_version", QT_VERSION_STR}, {"protocol_version", 2}, {"capture_affinity", double(captureExcluded)}});- QByteArray pending;- QTimer inputPoll;- QObject::connect(&inputPoll, &QTimer::timeout, &application, [&] {- DWORD available = 0;- if (!PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr)) { application.quit(); return; }- if (!available) return;- char buffer[4096]; DWORD count = 0;- if (!ReadFile(input, buffer, qMin<DWORD>(available, sizeof(buffer)), &count, nullptr) || !count) { application.quit(); return; }- pending.append(buffer, int(count));- int end = -1;- while ((end = pending.indexOf('\n')) >= 0) {- if (end > maximumMessage) { application.quit(); return; }- const QByteArray line = pending.left(end); pending.remove(0, end + 1);- QJsonParseError error;- const auto document = QJsonDocument::fromJson(line, &error);- if (error.error != QJsonParseError::NoError || !document.isObject()) { application.quit(); return; }- frame.command(document.object());- }- if (pending.size() > maximumMessage) application.quit();- });- inputPoll.start(20);- return application.exec();-}diff --git a/native/window-overlay/qt.conf b/native/window-overlay/qt.confdeleted file mode 100644index f783bdb..0000000--- a/native/window-overlay/qt.conf+++ /dev/null@@ -1,3 +0,0 @@-[Paths]-Prefix=.-Plugins=.diff --git a/scripts/build-window-overlay.ps1 b/scripts/build-window-overlay.ps1deleted file mode 100644index 10cdff2..0000000--- a/scripts/build-window-overlay.ps1+++ /dev/null@@ -1,35 +0,0 @@-param(- [Parameter(Mandatory=$true)][string]$QtPrefix,- [string]$BuildRoot = "$env:LOCALAPPDATA\Programs\PiComputerUsePoC\.tmp\window-overlay-build"-)-$ErrorActionPreference = 'Stop'-$ProgressPreference = 'SilentlyContinue'-# Build on a Windows-local path: MSBuild's case folding is unsafe on a case-sensitive WSL source share.-$source = Join-Path $BuildRoot 'source'-$build = Join-Path $BuildRoot 'build'-$bundle = Join-Path $BuildRoot 'bundle'-$vs = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools"-$cmake = Join-Path $vs 'Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe'-if (!(Test-Path $cmake) -or !(Test-Path "$QtPrefix\bin\windeployqt.exe")) { throw 'Existing MSVC Build Tools and the qualified Qt Widgets kit are required.' }-New-Item -ItemType Directory -Force $source,$build,$bundle | Out-Null-Copy-Item "$PSScriptRoot\..\native\window-overlay\*" $source -Force-& $cmake -S $source -B $build -G 'Visual Studio 17 2022' -A x64 "-DCMAKE_PREFIX_PATH=$QtPrefix"-if ($LASTEXITCODE -ne 0) { throw 'CMake configure failed.' }-& $cmake --build $build --config Release-if ($LASTEXITCODE -ne 0) { throw 'Native helper build failed.' }-Copy-Item "$build\Release\pi-window-overlay.exe" $bundle -Force-& "$QtPrefix\bin\windeployqt.exe" --release --no-patchqt --no-translations --no-quick-import --no-compiler-runtime --no-opengl-sw --no-system-d3d-compiler --skip-plugin-types generic,tls,networkinformation,imageformats,styles --qtpaths "$QtPrefix\bin\qtpaths.exe" --dir $bundle "$bundle\pi-window-overlay.exe"-if ($LASTEXITCODE -ne 0) { throw 'Qt deployment failed.' }-Copy-Item "$source\qt.conf" $bundle -Force-$files = @(Get-ChildItem $bundle -File -Recurse | ForEach-Object {- @{path=$_.FullName.Substring($bundle.Length+1); bytes=$_.Length; sha256=(Get-FileHash $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()}-})-$files | ConvertTo-Json -Depth 4 | Set-Content "$BuildRoot\bundle-manifest.json" -Encoding UTF8-$toolset = Get-ChildItem "$vs\VC\Tools\MSVC" -Directory | Sort-Object Name -Descending | Select-Object -First 1-$dumpbin = Join-Path $toolset.FullName 'bin\Hostx64\x64\dumpbin.exe'-Get-ChildItem $bundle -Recurse -File | Where-Object { $_.Extension -in '.dll','.exe' } | ForEach-Object {- $_.FullName- & $dumpbin /dependents $_.FullName- if ($LASTEXITCODE -ne 0) { throw 'DLL dependency inspection failed.' }-} | Set-Content "$BuildRoot\imports.txt" -Encoding UTF8-@{bundle=$bundle; bytes=($files | ForEach-Object { $_.bytes } | Measure-Object -Sum).Sum; files=$files.Count; qt=(& "$QtPrefix\bin\qtpaths.exe" --query QT_VERSION)} | ConvertTo-Json -Compressdiff --git a/scripts/package-window-overlay.mjs b/scripts/package-window-overlay.mjsdeleted file mode 100644index a143e0c..0000000--- a/scripts/package-window-overlay.mjs+++ /dev/null@@ -1,54 +0,0 @@-import assert from "node:assert/strict";-import { execFileSync } from "node:child_process";-import { cp, mkdir, readFile, readdir, writeFile, stat } from "node:fs/promises";-import path from "node:path";-import { createHash } from "node:crypto";--// Build artifact generator, not an installer. The release keeps DLLs replaceable and ships Qt source separately.-const [bundle, qtKit, qtSource, release] = process.argv.slice(2).map(value => path.resolve(value));-assert.ok(bundle && qtKit && qtSource && release, "Usage: node scripts/package-window-overlay.mjs bundle qtKit qtSource release");-const revision = "5a1194b2d368e2d72c230205c464b04a2f23eccd";-assert.equal(execFileSync("git", ["-C", qtSource, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(), revision);-const sbom = JSON.parse(await readFile(path.join(qtKit, "sbom/qtbase-6.11.3.spdx.json"), "utf8"));-for (const name of ["Core", "Gui", "Widgets", "QWindowsIntegrationPlugin"]) {- const module = sbom.packages.find(item => item.name === name);- assert.ok(module?.licenseConcluded.includes("LGPL-3.0-only"), `${name} needs an LGPL option in the exact kit SBOM`);-}-assert.ok(!(await stat(release).catch(() => undefined)), "Use a new release directory; never mix stale binaries or manifests.");-await mkdir(release, { recursive: true });-const runtime = path.join(release, "runtime");-await mkdir(runtime, { recursive: true });-await cp(bundle, runtime, { recursive: true });-const legal = path.join(runtime, "licenses"); await mkdir(legal, { recursive: true });-await cp(path.join(qtSource, "LICENSES"), legal, { recursive: true });-await cp(path.join(qtKit, "sbom/qtbase-6.11.3.spdx.json"), path.join(legal, "qtbase-sbom.json"));-await cp(path.resolve("native/window-overlay/NOTICE.md"), path.join(runtime, "NOTICE.md"));-await cp(path.resolve("native/window-overlay"), path.join(release, "helper-source"), { recursive: true });-execFileSync("git", ["-C", qtSource, "archive", "--format=tar.gz", "--prefix=qtbase/", "-o", path.join(release, "qtbase-source.tar.gz"), revision]);-// Preserve the installed kit's public build settings with the corresponding sources.-await cp(path.join(qtKit, "mkspecs/qconfig.pri"), path.join(release, "qt-kit-qconfig.pri"));-const attributions = sbom.packages.filter(item => item.name.includes("Attribution")).map(item => ({- name: item.name, license: item.licenseConcluded, copyright: item.copyrightText, source: item.downloadLocation,-}));-await writeFile(path.join(legal, "qtbase-attributions.json"), JSON.stringify(attributions, null, 2));-const files = [];-async function walk(directory) {- for (const entry of await readdir(directory, { withFileTypes: true })) {- const file = path.join(directory, entry.name);- if (entry.isDirectory()) await walk(file);- else {- const bytes = await readFile(file);- files.push({ path: path.relative(release, file), bytes: (await stat(file)).size, sha256: createHash("sha256").update(bytes).digest("hex") });- }- }-}-await walk(release);-execFileSync("tar", ["-czf", path.join(release, "runtime.tar.gz"), "-C", release, "runtime"]);-const packedRuntime = await stat(path.join(release, "runtime.tar.gz"));-await writeFile(path.join(release, "manifest.json"), JSON.stringify({ qt: "6.11.3", qt_revision: revision, files,- runtime_bytes: files.filter(item => item.path.startsWith("runtime/")).reduce((sum, item) => sum + item.bytes, 0),- runtime_archive_bytes: packedRuntime.size,- total_bytes: files.reduce((sum, item) => sum + item.bytes, 0), source_supplied: true,- notices: "Conservative qtbase attribution set includes components not linked by this helper.",-}, null, 2));-console.log(JSON.stringify({ release, files: files.length, source_supplied: true }));diff --git a/src/cua-image-settings.ts b/src/cua-image-settings.tsindex cbba142..92d1539 100644--- a/src/cua-image-settings.ts+++ b/src/cua-image-settings.ts@@ -20,7 +20,7 @@ export function nativeImageCap(settings: Pick<CuaVisionSettings, "maxLongEdge" |/** Session-local controls for the native cap. CUA owns resizing and fresh zoom. */export class CuaImageSettings {private current = defaults();- /** The trusted runtime owns the filter; the viewer may only change the cap. */+ /** The trusted runtime owns the filter; human controls may change the cap. */constructor(private readonly overviewKernel: () => CuaVisionSettings["kernel"] = () => "lanczos3") {}/** In-flight captures keep their own immutable settings copy. */settings(): CuaVisionSettings { return structuredClone({ ...this.current, kernel: this.overviewKernel() }); }diff --git a/src/driver.ts b/src/driver.tsindex 4b3d1dd..ae9b8e7 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -13,6 +13,7 @@ import { SingleConnectionHttpTransport } from "./http-transport.ts";import { SSH_OPTIONS, localRunner, remoteCommand, sshRunner } from "./runner.ts";import { verifyBundledRuntime } from "./native-runtime.ts";import type { NativePlatform } from "./observation-source.ts";+import { NativeVisuals, VisualSettings, type WindowObserver } from "./native-visuals.ts";/** Foreground input waits until the human has not used the keyboard or mouse for this long (owner choice, LPT-835). */const HUMAN_IDLE_MS = 10_000;@@ -119,6 +120,9 @@ const timelineLength = (args: Record<string, unknown>) => {};/** Owned platform-native MCP child; Windows-host control can run through WSL interop. */export class McpDriver implements Driver {+ private nativeVisualsAvailable = false;+ private readonly presentation: NativeVisuals;+ readonly visuals: WindowObserver;private client?: Client;private transport?: Transport;private stderr = "";@@ -165,7 +169,16 @@ export class McpDriver implements Driver {get windowCrop(): boolean { return this.config.runtime === "windows-desktop"; }/** Overview metadata follows the native implementation, not an agent-chosen filter. */get overviewKernel(): "bilinear" | "lanczos3" { return ["windows-fast-bilinear", "windows-direct-process", "windows-persistent-wgc", "windows-native-pointer", "windows-coordinate-scroll", "windows-window-restore", "windows-window-management", "windows-desktop"].includes(this.config.runtime ?? "") ? "bilinear" : "lanczos3"; }- constructor(private readonly config: ComputerConfig, private readonly version: "0.33.2" = "0.33.2") {+ constructor(private readonly config: ComputerConfig, private readonly version: "0.33.2" = "0.33.2",+ private readonly visualSettings = new VisualSettings()) {+ this.presentation = new NativeVisuals(async (name, args) => {+ // Presentation never starts or reconnects a backend, including after close or while startup is pending.+ if (!this.client || this.connecting) return undefined;+ if (!this.nativeVisualsAvailable) return false;+ const result = CallToolResultSchema.parse(await this.client.callTool({ name, arguments: args }, undefined, { timeout: 2000 }));+ return !result.isError;+ }, visualSettings);+ this.visuals = this.presentation.flash;if (this.windowLifetime) this.windowIdentity = new WindowIdentity(config.endpoint ? remoteIdentityPeer(config.endpoint.identityPort): hostIdentityPeer(config.ssh ? sshRunner(config.ssh) : localRunner(config.interop ?? process.env.WSL_INTEROP)));}@@ -214,15 +227,19 @@ export class McpDriver implements Driver {throw new Error(`This backend requires cua-driver ${version}. Requalify before changing the pin.`);if (this.envelopes && (client.getServerCapabilities()?.experimental?.["ai.cua.driver.envelopes"] as { version?: unknown } | undefined)?.version !== 1)throw new Error("The native fork runtime did not acknowledge its EOF-aware receiver.");+ const { tools } = await client.listTools({}, { signal, timeout: 15_000 });+ this.nativeVisualsAvailable = ["present_agent_activity", "set_agent_visuals"].every(name => tools.some(tool => tool.name === name && tool.inputSchema.type === "object"));if (this.config.bundled) {- const { tools } = await client.listTools({}, { signal, timeout: 15_000 });this.nativeTools = new Set(tools.map(tool => tool.name));if (this.config.bundled)for (const name of ["list_apps", "list_windows", "get_window_state", "get_desktop_state"])if (!tools.some(tool => tool.name === name && tool.inputSchema.type === "object"))throw new Error(`native_runtime_schema_missing: ${name}`);}- await this.jumpCursor(client, signal);+ if (this.nativeVisualsAvailable) {+ const configured = CallToolResultSchema.parse(await client.callTool({ name: "set_agent_visuals", arguments: { ...this.visualSettings.current() } }, undefined, { signal, timeout: 15_000 }));+ if (configured.isError) throw new Error("The driver refused its native visual configuration.");+ } else await this.jumpCursor(client, signal);return client;} catch (error) {await this.close();@@ -354,6 +371,8 @@ export class McpDriver implements Driver {/** EOF owns shutdown; the SDK escalates only for this child if it does not exit. */async close(): Promise<void> {+ this.presentation.close();+ this.nativeVisualsAvailable = false;const client = this.client;const transport = this.transport;this.client = undefined;diff --git a/src/image-settings-command.ts b/src/image-settings-command.tsnew file mode 100644index 0000000..586836d--- /dev/null+++ b/src/image-settings-command.ts@@ -0,0 +1,39 @@+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";+import { CuaImageSettings } from "./cua-image-settings.ts";++const usage = "Usage: /computer-images [show | cap 64..4096 | scale 0.05..1 | hints on|off | guides off|grid|action|all | reset].";++/** Human-only controls for future captures; no viewer, listener, native process or image retention. */+export function registerImageSettingsCommand(pi: ExtensionAPI, images: CuaImageSettings): void {+ pi.registerCommand("computer-images", {+ description: "Show or change session-local screenshot sizing, hint labels and image guides for future captures.",+ getArgumentCompletions: prefix => ["show", "cap", "scale", "hints on", "hints off", "guides off", "guides grid", "guides action", "guides all", "reset"]+ .filter(value => value.startsWith(prefix)).map(value => ({ value, label: value })),+ handler: async (args, ctx) => {+ const parts = args.trim().toLowerCase().split(/\s+/).filter(Boolean);+ const [command = "show", value] = parts;+ try {+ const current = images.settings();+ if (command === "show" || command === "reset") {+ if (parts.length > 1) throw Error(usage);+ if (command === "reset") images.reset();+ } else if (command === "cap" || command === "scale") {+ if (parts.length !== 2 || !/^(?:\d+(?:\.\d+)?|\.\d+)$/.test(value)) throw Error(usage);+ images.update({ kernel: current.kernel, maxLongEdge: command === "cap" ? Number(value) : current.maxLongEdge,+ scale: command === "scale" ? Number(value) : current.scale });+ } else if (command === "hints") {+ if (parts.length !== 2 || (value !== "on" && value !== "off")) throw Error(usage);+ images.setHints(value === "on");+ } else if (command === "guides") {+ if (parts.length !== 2 || !["off", "grid", "action", "all"].includes(value)) throw Error(usage);+ images.setGuides({ grid: value === "grid" || value === "all", action: value === "action" || value === "all" });+ } else throw Error(usage);+ const settings = images.settings();+ ctx.ui.notify(`Computer images: ${settings.maxLongEdge} px maximum, scale ${settings.scale}, ${settings.kernel}; hints ${settings.hints === false ? "off" : "on"}, `+ + `grid ${settings.guides?.grid ? "on" : "off"}, action guide ${settings.guides?.action ? "on" : "off"}. Applies to future captures in this session.`, "info");+ } catch (error) {+ ctx.ui.notify(error instanceof Error ? error.message : usage, "warning");+ }+ },+ });+}diff --git a/src/index.ts b/src/index.tsindex d17b498..f9bd646 100644--- a/src/index.ts+++ b/src/index.ts@@ -1,11 +1,12 @@-import { registerVisionDebug } from "./vision-debug.ts";+import { registerImageSettingsCommand } from "./image-settings-command.ts";+import { registerVisualSettingsCommand } from "./visual-settings-command.ts";+import { VisualSettings } from "./native-visuals.ts";import { CuaImageSettings } from "./cua-image-settings.ts";import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";import { loadComputerConfig } from "./config.ts";import { McpDriver } from "./driver.ts";import { registerComputerMode } from "./mode.ts";import { registerSetupCommand } from "./setup-command.ts";-import { WindowHighlights } from "./window-highlights.ts";import { ComputerSession, type ComputerResult } from "./session.ts";import { registerComputerProvider } from "./provider.ts";import { registerStandaloneTools } from "./standalone-tools.ts";@@ -22,18 +23,19 @@ import { machineFactory, sandboxFactory } from "./machine.ts";export default async function computerUse(pi: ExtensionAPI): Promise<void> {let overviewKernel: "lanczos3" | "bilinear" = "lanczos3";const images = new CuaImageSettings(() => overviewKernel);- const debug = registerVisionDebug(pi, images);- const highlights = new WindowHighlights();- const getMode = registerComputerMode(pi, () => highlights.clear());+ const visuals = new VisualSettings();+ registerImageSettingsCommand(pi, images);+ registerVisualSettingsCommand(pi, visuals);+ const getMode = registerComputerMode(pi, async () => { visuals.clear(); });registerSetupCommand(pi);let session: ComputerSession | undefined;let loading: Promise<ComputerSession> | undefined;async function getSession(cwd: string): Promise<ComputerSession> {if (session) return session;if (!loading) loading = loadComputerConfig(cwd).then(async config => {- const driver = new McpDriver(config);+ const driver = new McpDriver(config, undefined, visuals);overviewKernel = driver.overviewKernel ?? "lanczos3";- session = new ComputerSession(driver, getMode, () => images.settings(), driver.platform === "windows" ? highlights.flash : undefined, image => debug.trace.retainOriginal(image),+ session = new ComputerSession(driver, getMode, () => images.settings(), undefined,driver.platform === "windows" ? (options, signal) => WebBrowser.launch({ ...options, interop: config.interop }, signal) : undefined,driver.platform === "windows" ? new MachineHost(localRunner(config.interop)) : undefined);return session;@@ -41,17 +43,14 @@ export default async function computerUse(pi: ExtensionAPI): Promise<void> {try { return await loading; }finally { loading = undefined; }}- const openSandbox = sandboxFactory({ settings: () => images.settings(), retainRaw: image => debug.trace.retainOriginal(image) });- const connectMachine = machineFactory({ getMode, settings: () => images.settings(), retainRaw: image => debug.trace.retainOriginal(image) });+ const openSandbox = sandboxFactory({ settings: () => images.settings(), visuals });+ const connectMachine = machineFactory({ getMode, settings: () => images.settings(), visuals });const provider = await registerComputerProvider(pi, getSession, async () => {const result: ComputerResult = session ? await session.close() : {content: [{ type: "text", text: "Not started." }],details: { status: "closed", mode: getMode() },};session = undefined;- try { await highlights.close(); }- catch { return { content: [{ type: "text", text: "Qt frame shutdown is unconfirmed." }],- details: { status: "unknown", mode: getMode() }, isError: true }; }return result;}, openSandbox, connectMachine, getMode);// Every extension has loaded by session_start, so an IDE that accepts the provider has done so by now.@@ -68,7 +67,8 @@ export default async function computerUse(pi: ExtensionAPI): Promise<void> {});pi.on("session_shutdown", async () => {if (loading) await loading.catch(() => undefined);- try { await session?.close(); } finally { await highlights.close(); }+ visuals.clear();+ await session?.close();session = undefined;});}diff --git a/src/machine.ts b/src/machine.tsindex 88878af..e048e62 100644--- a/src/machine.ts+++ b/src/machine.ts@@ -10,8 +10,9 @@ import { MachineHost } from "./machine-host.ts";import { localRunner, sshRunner, type CommandRunner } from "./runner.ts";import { WindowsSandbox, runnerHost, sandboxOptions, waitForGuestDriver } from "./sandbox.ts";import { IsolatedSession } from "./isolated-session.ts";-import { ComputerSession, type ComputerMode, type Content } from "./session.ts";+import { ComputerSession, type ComputerMode } from "./session.ts";import { WebBrowser } from "./web-browser.ts";+import type { VisualSettings } from "./native-visuals.ts";/** The configured machine for a connect command; unknown names refuse before any SSH work. */export async function machineConfig(cwd: string, name: string): Promise<NonNullable<ComputerConfig["machines"]>[string]> {@@ -46,7 +47,7 @@ export async function checkRemoteWorker(runner: CommandRunner, remotePackage: st* Each sandbox or isolated agent session gets its own guest driver and session: no host overlay, no browser launch, every guest window is the agent's own.* On another machine the sandbox host and its files are reached over SSH, and the guest relays through SSH tunnels.*/-export function sandboxFactory(options: { settings: () => CuaVisionSettings; retainRaw?: (image: Content) => void }): SandboxFactory {+export function sandboxFactory(options: { settings: () => CuaVisionSettings; visuals?: VisualSettings }): SandboxFactory {return async (cwd, command, signal, machineName) => {let parsed;try {@@ -60,10 +61,10 @@ export function sandboxFactory(options: { settings: () => CuaVisionSettings; ret: await IsolatedSession.open(runnerHost(runner), machine?.command ?? config.command, signal);// The endpoint selects the guest transport; ssh only makes the runtime hash check read the machine's own bytes.const driver = new McpDriver({ command: machine?.command ?? config.command, ...(machine?.runtime ?? config.runtime ? { runtime: machine?.runtime ?? config.runtime } : {}),- endpoint: sandbox.endpoint, ...(machine ? { ssh: machine.ssh } : {}) });+ endpoint: sandbox.endpoint, ...(machine ? { ssh: machine.ssh } : {}) }, undefined, options.visuals);try { await waitForGuestDriver(driver, signal); }catch (error) { await driver.close().catch(() => undefined); await sandbox.close(); throw error; }- const guest = new ComputerSession(driver, () => "yolo", options.settings, undefined, options.retainRaw);+ const guest = new ComputerSession(driver, () => "yolo", options.settings);return { ref: sandbox.ref, session: guest, describe: source => sandbox.describe(source), ...(machineName ? { machine: machineName } : {}),close: async () => { try { await guest.close(); } finally { await sandbox.close(); } } };};@@ -73,12 +74,12 @@ export function sandboxFactory(options: { settings: () => CuaVisionSettings; ret* Another machine's desktop gets its own driver and session over SSH. Safe/YOLO is the user's choice there too;* the host overlay is not shown, and browsers start on that machine.*/-export function machineFactory(options: { getMode: () => ComputerMode; settings: () => CuaVisionSettings; retainRaw?: (image: Content) => void }): MachineFactory {+export function machineFactory(options: { getMode: () => ComputerMode; settings: () => CuaVisionSettings; visuals?: VisualSettings }): MachineFactory {return async (cwd, name, signal) => {const machine = await machineConfig(cwd, name);const refresh = `setup part machine (user: /computer-setup machine ${machine.ssh} ${name})`;const runner = sshRunner(machine.ssh);- const driver = new McpDriver({ command: machine.command, ...(machine.runtime ? { runtime: machine.runtime } : {}), ssh: machine.ssh });+ const driver = new McpDriver({ command: machine.command, ...(machine.runtime ? { runtime: machine.runtime } : {}), ssh: machine.ssh }, undefined, options.visuals);try { await waitForGuestDriver(driver, signal, 30_000); }catch (error) { await driver.close().catch(() => undefined); throw error; }const launchWeb = machine.node && machine.package@@ -87,7 +88,7 @@ export function machineFactory(options: { getMode: () => ComputerMode; settings:return WebBrowser.launch({ ...launch, remote: { runner, node: machine.node!, worker: `${machine.package}/src/web-worker.ts` } }, webSignal);}: undefined;- const remote = new ComputerSession(driver, options.getMode, options.settings, undefined, options.retainRaw, launchWeb, new MachineHost(runner));+ const remote = new ComputerSession(driver, options.getMode, options.settings, undefined, launchWeb, new MachineHost(runner));return { ref: name, session: remote,describe: source => [`# Machine ${name}`, `Address: ${source}`, `SSH: ${machine.ssh}`,`Browser: ${launchWeb ? "available" : "not configured (node and package)"}`, ...machineGuide(source)].join("\n"),diff --git a/src/native-visuals.ts b/src/native-visuals.tsnew file mode 100644index 0000000..48d4b56--- /dev/null+++ b/src/native-visuals.ts@@ -0,0 +1,152 @@+import type { Content, WindowTarget } from "./session.ts";+import type { ScreenRect } from "./highlight-geometry.ts";++export interface VisualOptions {+ preset: "off" | "subtle" | "full";+ observations: boolean; previews: boolean; pointer: boolean; scroll_drag: boolean; keyboard: boolean;+ opacity: number; quality: "performance" | "balanced" | "high"; reduced_motion: boolean;+}+const defaults = (): VisualOptions => ({ preset: "full", observations: true, previews: true, pointer: true,+ scroll_drag: true, keyboard: true, opacity: 0.8, quality: "balanced", reduced_motion: false });+/** Human-owned, session-local presentation preferences. No model tool can write them. */+export class VisualSettings {+ private value = defaults();+ private listeners = new Set<(value: VisualOptions) => void>();+ private revokers = new Set<() => void>();+ current(): VisualOptions { return { ...this.value }; }+ update(patch: Partial<VisualOptions>): VisualOptions { return this.apply(patch, this.value); }+ restore(patch: Partial<VisualOptions>): VisualOptions { return this.apply(patch, defaults()); }+ private apply(patch: Partial<VisualOptions>, base: VisualOptions): VisualOptions {+ if (!patch || typeof patch !== "object" || Array.isArray(patch) || Object.keys(patch).some(k => !Object.hasOwn(base, k))) throw Error("Unknown visual setting.");+ const next = { ...base, ...patch };+ if (!["off", "subtle", "full"].includes(next.preset) || !["performance", "balanced", "high"].includes(next.quality) ||+ !Number.isFinite(next.opacity) || next.opacity < 0 || next.opacity > 1 ||+ [next.observations, next.previews, next.pointer, next.scroll_drag, next.keyboard, next.reduced_motion].some(v => typeof v !== "boolean"))+ throw Error("Invalid visual settings.");+ if (patch.preset === "subtle" && patch.previews === undefined) next.previews = false;+ if (patch.preset === "full" && patch.previews === undefined) next.previews = true;+ this.value = next;+ for (const listener of this.listeners) listener(this.current());+ return this.current();+ }+ subscribe(listener: (value: VisualOptions) => void): () => void { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }+ clear(): void { for (const revoke of this.revokers) revoke(); }+ onClear(revoke: () => void): () => void { this.revokers.add(revoke); return () => { this.revokers.delete(revoke); }; }+}++export interface HighlightMark { kind: "overview" | "detail" | "inspect" | "verify" | "action"; region?: ScreenRect; point?: { x: number; y: number } }+interface ActivityPayload {+ mark?: HighlightMark;+ image?: Extract<Content, { type: "image" }>;+ action?: string;+ outcome?: "delivered" | "refused" | "unknown" | "verified";+}+export interface WindowFrame extends ActivityPayload { target: WindowTarget; identity: string; bounds: ScreenRect }+export interface DisplayFrame extends ActivityPayload { display: string; bounds: ScreenRect }+export type HighlightFrame = WindowFrame | DisplayFrame | { clear: true };+/** Queued is admission intent, never proof that pixels were shown or that input succeeded. */+export interface HighlightResult { status: "queued" | "shown" | "skipped" | "unavailable"; kind?: HighlightMark["kind"]; reason?: string; region?: ScreenRect; point?: { x: number; y: number } }+export type WindowObserver = (frame: HighlightFrame, signal?: AbortSignal) => Promise<HighlightResult>;+type Send = (name: "present_agent_activity" | "set_agent_visuals", args: Record<string, unknown>) => Promise<boolean | undefined>;++const rectangle = (r: ScreenRect) => r && [r.x, r.y, r.width, r.height].every(Number.isSafeInteger) &&+ Math.abs(r.x) <= 2_000_000 && Math.abs(r.y) <= 2_000_000 && r.width > 0 && r.width <= 32768 && r.height > 0 && r.height <= 32768;+function valid(frame: WindowFrame | DisplayFrame): boolean {+ const { bounds, mark } = frame;+ if (!rectangle(bounds)) return false;+ if ("display" in frame) { if (!frame.display || frame.display.length > 256 || "target" in frame) return false; }+ else {+ const identity = /^PiComputerUse\.[a-f0-9]{32}:(\d+):([1-9]\d*)$/i.exec(frame.identity);+ if (!identity || Number(identity[1]) !== frame.target.pid || !Number.isSafeInteger(Number(identity[2])) ||+ ![frame.target.pid, frame.target.window_id].every(v => Number.isSafeInteger(v) && v > 0)) return false;+ }+ if (!mark) return true;+ if (!["overview", "detail", "inspect", "verify", "action"].includes(mark.kind) || (mark.region && mark.point)) return false;+ const r = mark.region; const p = mark.point;+ if (r) return rectangle(r) && r.x >= bounds.x && r.y >= bounds.y && r.x + r.width <= bounds.x + bounds.width && r.y + r.height <= bounds.y + bounds.height;+ if (p) return [p.x, p.y].every(Number.isSafeInteger) && p.x >= bounds.x && p.y >= bounds.y && p.x < bounds.x + bounds.width && p.y < bounds.y + bounds.height;+ return mark.kind !== "detail" && (mark.kind !== "action" || !!frame.action || !!frame.outcome);+}+/** Validation and fail-soft notification never grant capture or input authority. */+export async function notifyHighlight(observer: WindowObserver | undefined, frame: WindowFrame | DisplayFrame | undefined, signal?: AbortSignal): Promise<HighlightResult | undefined> {+ if (!observer) return;+ if (signal?.aborted) return { status: "skipped" };+ if (!frame || !valid(frame)) {+ try { await observer({ clear: true }, signal); } catch { /* Presentation is optional. */ }+ return { status: "unavailable", reason: "native_geometry_unavailable" };+ }+ try { return { ...await observer(frame, signal), kind: frame.mark?.kind ?? "overview",+ ...(frame.mark?.region ? { region: { ...frame.mark.region } } : {}), ...(frame.mark?.point ? { point: { ...frame.mark.point } } : {}) }; }+ catch { return { status: "unavailable", reason: "renderer_unavailable" }; }+}+export function notifyWindowObserver(observer: WindowObserver | undefined, target: WindowTarget | undefined, identity: string | undefined,+ bounds: unknown, signal?: AbortSignal, mark?: HighlightMark, image?: Content, action?: string, outcome?: ActivityPayload["outcome"]): Promise<HighlightResult | undefined> {+ return notifyHighlight(observer, target && identity && bounds ? { target: { ...target }, identity, bounds: bounds as ScreenRect,+ ...(mark ? { mark } : {}), ...(image?.type === "image" ? { image } : {}), ...(action ? { action } : {}), ...(outcome ? { outcome } : {}) } : undefined, signal);+}++/** One native admission in flight and one replaceable pending image. Native owns animation timing. */+export class NativeVisuals {+ private generation = 0;+ private pending?: { name: "present_agent_activity" | "set_agent_visuals"; args: Record<string, unknown>; time: number };+ private config?: VisualOptions;+ private busy = false;+ private closed = false;+ private unavailable = false;+ private release?: () => void;+ private readonly unsubscribe: () => void;+ constructor(private readonly send: Send, private readonly settings = new VisualSettings()) {+ const preferences = settings.subscribe(value => {+ ++this.generation; this.release?.(); this.release = undefined; this.pending = undefined;+ this.config = value; this.drain();+ });+ const revocation = settings.onClear(() => this.clear());+ this.unsubscribe = () => { preferences(); revocation(); };+ }+ readonly flash: WindowObserver = async (frame, signal) => {+ if ("clear" in frame) { this.clear(); return { status: "skipped" }; }+ if (this.closed || signal?.aborted || this.settings.current().preset === "off") return { status: "skipped" };+ if (this.unavailable) return { status: "unavailable", reason: "native_visuals_unavailable" };+ if (!valid(frame)) { this.clear(); return { status: "unavailable", reason: "native_geometry_unavailable" }; }+ const generation = ++this.generation;+ this.release?.(); this.release = undefined;+ const { mark, image } = frame;+ const source = structuredClone("display" in frame ? { display: frame.display, bounds: frame.bounds } : { ...frame.target, identity: frame.identity, bounds: frame.bounds });+ const options = this.settings.current();+ if (mark?.kind !== "action" && !options.observations) return { status: "skipped" };+ const args: Record<string, unknown> = { generation, kind: mark?.kind ?? "overview", source,+ ...(mark?.region ? { region: { ...mark.region } } : {}), ...(mark?.point ? { point: { ...mark.point } } : {}),+ ...(frame.action ? { action: frame.action } : {}), ...(frame.outcome ? { outcome: frame.outcome } : {}) };+ // Bound transient transport memory. Oversized images still get a source frame, without a preview.+ if (options.previews && image && ["image/png", "image/jpeg"].includes(image.mimeType) && image.data.length <= 4 * 1024 * 1024)+ args.image = { data: image.data, mime_type: image.mimeType };+ this.pending = { name: "present_agent_activity", args, time: performance.now() };+ const abort = () => { if (generation === this.generation) this.clear(); };+ if (signal) {+ signal.addEventListener("abort", abort, { once: true });+ const expiry = setTimeout(() => { if (generation === this.generation) { this.release?.(); this.release = undefined; } }, 1500);+ expiry.unref();+ this.release = () => { clearTimeout(expiry); signal.removeEventListener("abort", abort); };+ }+ this.drain();+ return { status: "queued" };+ };+ clear(): void {+ ++this.generation; this.release?.(); this.release = undefined;+ if (this.closed || this.unavailable) { this.pending = undefined; return; }+ this.pending = { name: "present_agent_activity", args: { kind: "clear", generation: this.generation }, time: performance.now() };+ this.drain();+ }+ private drain(): void {+ if (this.busy || this.closed || this.unavailable) return;+ const next = this.config ? { name: "set_agent_visuals" as const, args: { ...this.config }, time: performance.now() } : this.pending;+ if (!next) return;+ if (this.config) this.config = undefined; else this.pending = undefined;+ if (next.name === "present_agent_activity" && next.args.kind !== "clear" && performance.now() - next.time > 250) { this.drain(); return; }+ this.busy = true;+ void Promise.resolve().then(() => this.send(next.name, next.args)).then(ok => {+ if (ok === false) { this.unavailable = true; this.pending = undefined; this.config = undefined; }+ }).catch(() => { this.unavailable = true; this.pending = undefined; this.config = undefined; }).finally(() => { this.busy = false; this.drain(); });+ }+ close(): void { this.closed = true; ++this.generation; this.release?.(); this.pending = undefined; this.config = undefined; this.unsubscribe(); }+}diff --git a/src/provider.ts b/src/provider.tsindex 30972fd..1c85f2e 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -493,7 +493,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwasync tryResolve(request, context) {if (!owns(request.path)) return { kind: "not-handled" };try {- if (request.include !== undefined || request.exclude !== undefined || request.navigation !== undefined || request.caseSensitive || request.wholeWord)+ if (request.include !== undefined || request.exclude !== undefined || ("navigation" in request && request.navigation !== undefined) || request.caseSensitive || request.wholeWord)throw new ComputerRequestError("Computer Search uses native discovery/UI text predicates, not file globs or symbol navigation.");const result = await resources.search(resources.address(request.path), context.cwd, request.query, request.limit, context.signal);return { kind: "resolved", payload: result };diff --git a/src/session.ts b/src/session.tsindex e309b81..e3a04a1 100644--- a/src/session.ts+++ b/src/session.ts@@ -15,7 +15,7 @@ import { isCoordinateScroll, scrollReceipt } from "./scroll-actions.ts";import { validWindowStatus, windowRestoreStatus } from "./window-restore.ts";import { validWindowControlState, windowManagementStatus, type WindowManagementCommand } from "./window-management.ts";import { nativeInput, requestedInputTime, validateInput, type InputCommand, type InputStep } from "./complete-input.ts";-import { notifyWindowObserver, notifyHighlight, type WindowObserver, type HighlightResult } from "./window-overlay.ts";+import { notifyWindowObserver, notifyHighlight, type WindowObserver, type HighlightResult, type HighlightMark } from "./native-visuals.ts";import { screenRect, elementRegion, windowImageMapping, windowRegion, nativeClickPoint, type ScreenRect } from "./highlight-geometry.ts";import { observationSource, sourceIdentity, type ObservationSource, type SourceIdentity, type NativePlatform } from "./observation-source.ts";import { actionCapability, type ActionCapability } from "./action-capability.ts";@@ -71,6 +71,8 @@ export interface Driver {overviewKernel?: CuaVisionSettings["kernel"];/** A display-only window read can return one rectangle of the window, encoding only that rectangle. */windowCrop?: boolean;+ /** Human-only native presentation, owned by this driver and never an input route. */+ visuals?: WindowObserver;call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;/**@@ -162,6 +164,7 @@ interface WindowState { id: string; nativeId: string; kind: "restore" | "managem/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation;+ visualBounds?: ScreenRect;/** Size of the window's latest image capture: points of typed input are in its pixels. Outlives the observation. */seen?: { width: number; height: number };windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }@@ -290,8 +293,7 @@ export class ComputerSession {constructor(private readonly driver: Driver, private readonly getMode: () => ComputerMode = () => "yolo",private readonly settings: () => CuaVisionSettings = () => ({ kernel: "lanczos3", maxLongEdge: DEFAULT_MAX_LONG_EDGE, scale: 1, version: 0, route: "cua_native" }),- private readonly observer?: WindowObserver,- private readonly retainRaw?: (image: Content) => void,+ private readonly observer: WindowObserver | undefined = driver.visuals,private readonly launchWeb?: WebLauncher,/** Display modes, the owned file folder, process health and OCR on this desktop's machine; absent in a sandbox guest. */readonly host?: MachineHost) {}@@ -387,15 +389,25 @@ export class ComputerSession {return undefined;const identity = await this.driver.identity(target, true, signal);if (!identity) return undefined;+ const visualBounds = screenRect((target as WindowTarget & { bounds?: unknown }).bounds);for (const entry of this.entries.values()) {- if (entry.target.pid === target.pid && entry.target.window_id === target.window_id && entry.identity === identity) return entry;+ if (entry.target.pid === target.pid && entry.target.window_id === target.window_id && entry.identity === identity) {+ if (visualBounds) entry.visualBounds = visualBounds;+ return entry;+ }}- const entry: WindowEntry = { ref: "w-" + randomUUID(), target: { pid: target.pid, window_id: target.window_id }, identity, owned: false,+ const entry: WindowEntry = { ref: "w-" + randomUUID(), target: { pid: target.pid, window_id: target.window_id }, identity, owned: false, visualBounds,web: this.browsers.get(`${target.pid}:${target.window_id}`) };this.entries.set(entry.ref, entry);return entry;}+ /** Source-associated cue only: no UI strings, guessed coordinates, capture or input. */+ private activity(entry: WindowEntry, mark: HighlightMark, signal?: AbortSignal, image?: Content,+ outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string): Promise<HighlightResult | undefined> {+ if (!this.allowed(entry) || signal?.aborted) return Promise.resolve(undefined);+ return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome);+ }private async current(ref: string, signal?: AbortSignal): Promise<WindowEntry | undefined> {const entry = this.entries.get(ref);if (!entry || !this.allowed(entry)) return undefined;@@ -1109,9 +1121,9 @@ export class ComputerSession {const id = randomUUID(); const context = this.overview(state!, id, true, images);if (entry) { if (!context) throw new Error("CUA returned invalid selected-display raster geometry."); entry.detail = context; }else this.displayOverview = context;- // The installed optional human overlay understands only primary. Clear it for selected monitors rather than drawing on the wrong source.- const highlight = await notifyHighlight(this.observer, !entry && context ? { display: "primary", bounds: { x: 0, y: 0, width: context.originalWidth, height: context.originalHeight } } : undefined, signal);- if (entry && highlight) highlight.reason = "selected_display_overlay_unsupported";+ const highlight = await notifyHighlight(this.observer, context ? { display: entry?.native.display_id ?? "primary",+ bounds: entry?.native.bounds ?? { x: 0, y: 0, width: context.originalWidth, height: context.originalHeight },+ ...(images[0]?.type === "image" ? { image: images[0] } : {}) } : undefined, signal);if (this.getMode() !== "yolo") return denied();if (signal?.aborted) { retire(); return this.result("refused", "Cancelled during display frame; pixels withheld."); }return this.result("observed", `Native display ${entry?.native.display_id ?? "primary"} overview: ${state!.screenshot_width}x${state!.screenshot_height} local pixels. Read only; zoom uses this image_id. Paint freshness is unverified.`,@@ -1156,10 +1168,9 @@ export class ComputerSession {coordinate_space: "detail_image_pixels", configuration: { ...configuration, route: "adapter_crop_resize" },region: { overview: { image_id: overview!.id, dimensions: { width: overview!.width, height: overview!.height }, x: request.x, y: request.y, width: request.width, height: request.height }, native: region },native: { resolution: "full", intermediate_pixels: "withheld" }, input: "none" };- const highlight = await notifyHighlight(this.observer, entry ? undefined : { display: "primary",- bounds: { x: 0, y: 0, width: overview!.originalWidth, height: overview!.originalHeight },- mark: { kind: "detail", region: { x: region.left, y: region.top, width: region.width, height: region.height } } }, signal);- if (entry && highlight) highlight.reason = "selected_display_overlay_unsupported";+ const bounds = entry?.native.bounds ?? { x: 0, y: 0, width: overview!.originalWidth, height: overview!.originalHeight };+ const highlight = await notifyHighlight(this.observer, { display: entry?.native.display_id ?? "primary", bounds,+ mark: { kind: "detail", region: { x: bounds.x + region.left, y: bounds.y + region.top, width: region.width, height: region.height } }, image: detail.image }, signal);if (this.getMode() !== "yolo" || signal?.aborted) return this.result("refused", "Display detail revoked or cancelled during frame; pixels withheld.");return this.result("observed", `Fresh display ${entry?.native.display_id ?? "primary"} crop: native-local ${region.left},${region.top} ${region.width}x${region.height}, output ${detail.width}x${detail.height}. Read only; no nested crop or input. Paint freshness is unverified.`,undefined, { source, agent_image, ...(highlight ? { highlight } : {}) }, [detail.image]);@@ -1192,7 +1203,7 @@ export class ComputerSession {const bounds = screenRect(window.bounds);Object.assign(source, this.domained(sourceIdentity(source.identity, null, window.virtual_desktop, this.platform)));const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,- { kind: "detail", region: bounds ? windowRegion(request, overview!.windowMapping, bounds) : undefined });+ { kind: "detail", region: bounds ? windowRegion(request, overview!.windowMapping, bounds) : undefined }, image);if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Window detail revoked or cancelled during frame; pixels withheld.");return this.result("observed", "Fresh native window zoom. CUA owns JPEG padding and geometry. Read only; take a new overview before input or another detail.",undefined, { ...this.addressed(entry, window.virtual_desktop), ...(highlight ? { highlight } : {}), source, agent_image: { route: "cua_fresh_window_zoom", coordinate_space: "zoom_image_pixels",@@ -1261,6 +1272,7 @@ export class ComputerSession {const full = [element.label, element.value].filter((item): item is string => typeof item === "string" && item.length > 1024);if (full.length && ui.elements[index]) texts.set(ui.elements[index].hint, full.join("\n").slice(0, ELEMENT_TEXT_LIMIT));});+ entry.visualBounds = screenRect(state.window_bounds);// Only an image the agent is shown sets the space of its points; successors and other quiet captures keep it.if (hasImage && !quiet && typeof state.screenshot_width === "number" && typeof state.screenshot_height === "number") entry.seen = { width: state.screenshot_width, height: state.screenshot_height };entry.observation = { id, ui, settings, ...(texts.size ? { texts } : {}),@@ -1277,15 +1289,19 @@ export class ComputerSession {const source = this.source("window", state, { session_id: this.sessionId, window_ref: entry.ref, ...entry.target });if (!hasUi) source.semantic_scope = "none";if (!hasImage) source.capabilities.detail = "unsupported";- // Repeated verification samples do not flash the human frame on every walk.- const highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal);+ // Publish only the final operation image, after annotation. Quiet verification samples stay silent.+ let highlight: HighlightResult | undefined;if (!this.allowed(entry)) { entry.observation = undefined; entry.detail = undefined; return this.denied(entry.ref); }if (signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Cancelled during window frame; captured data withheld."); }const observed = entry.observation!;observed.source = source;- const extra = { ...this.addressed(entry, state.virtual_desktop), ...(highlight ? { highlight } : {}), observation_id: id,+ const extra = { ...this.addressed(entry, state.virtual_desktop), observation_id: id,source, granularity, captured: { image: hasImage, ui: hasUi } };- if (!hasImage) return this.result("observed", `Fresh UI-only window observation: ${ui!.elements.length} UI elements. No pixels or image detail; current hints/tokens only.`, state, { ...extra, ui: structuredClone(ui!) });+ if (!hasImage) {+ highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; data withheld.");+ return this.result("observed", `Fresh UI-only window observation: ${ui!.elements.length} UI elements. No pixels or image detail; current hints/tokens only.`, state, { ...extra, ...(highlight ? { highlight } : {}), ui: structuredClone(ui!) });+ }const raw = images[0];if (raw?.type === "image" && Buffer.byteLength(raw.data, "base64") <= 12 * 1024 * 1024) {observed.rawImage = raw;@@ -1300,12 +1316,13 @@ export class ComputerSession {const rendered = await annotateUi(raw, annotationUi, observed.width, observed.height, hasUi && (options.hints ?? settings.hints ?? true), undefined, annotationUi.elements,{ options: settings.guides, action: observed.actionMark });if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled during annotation; pixels withheld."); }- if (rendered.image !== raw) this.retainRaw?.(raw);+ highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);+ if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled; pixels withheld."); }const metadata = this.imageMetadata([rendered.image], state, settings);const message = hasUi? `Fresh full window observation: ${ui!.elements.length} UI elements; ${rendered.annotation.labels.length} image hints. Structure ${ui!.complete === true ? "complete" : ui!.complete === false ? "partial" : "completeness unknown"}; paint freshness is unverified. Hints belong to this observation only.`: "Fresh image-only window observation. Accessibility was not collected; no current UI hints/tokens. Pixel input uses this window image; paint freshness is unverified.";- return this.result("observed", message, state, { ...extra, ...(ui ? { ui: structuredClone(ui) } : {}),+ return this.result("observed", message, state, { ...extra, ...(highlight ? { highlight } : {}), ...(ui ? { ui: structuredClone(ui) } : {}),agent_image: { ...metadata, ...(rendered.image !== raw ? { route: "cua_native_ui_hints" } : {}), annotation: rendered.annotation, image_id: id } }, [rendered.image]);}@@ -1345,6 +1362,8 @@ export class ComputerSession {entry.dom = undefined; entry.windowState = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("snapshot", options, signal));+ await this.activity(entry, { kind: "inspect" }, signal);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; DOM withheld.");return this.result("observed", "Fresh Playwright aria snapshot. Use [ref=...] values as target.ref while this stays the latest snapshot of the window.",undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });} catch (error) { return this.webFailure(entry, error); }@@ -1387,6 +1406,10 @@ export class ComputerSession {{ ...(target ? { target } : {}), highlight: kind === "highlight", ...(css ? { css } : {}) }, signal);const bytes = Buffer.from(shot.png, "base64");if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw Error("Playwright did not return a PNG.");+ const image: Content = { type: "image", data: shot.png, mimeType: "image/png" };+ // CSS boxes do not prove a Windows screen transform. Associate the preview with its window only.+ await this.activity(entry, { kind: "inspect" }, signal, image);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; pixels withheld.");return this.result("observed", `Playwright ${kind === "page" ? "page viewport" : kind === "crop" ? "element" : "highlighted viewport"} screenshot. These are page pixels, not native window coordinates; they grant no input.`,{ url: shot.url, title: shot.title, ...(shot.box ? { box_css: shot.box } : {}) },{ ...this.addressed(entry), ...(id !== undefined ? { observation_id: id } : {}), agent_image: { route: "playwright_screenshot", kind, coordinate_space: "page_image_pixels", input_authority: false,@@ -1456,11 +1479,13 @@ export class ComputerSession {if (!this.allowed(entry)) return this.denied(entry.ref);let samples = 0, stable = 0, results: ConditionResult[] = [], last: ComputerResult | undefined;entry.dom = undefined; entry.windowState = undefined;- const finish = (verdict: WaitResult["verdict"], timedOut: boolean, final: ComputerResult | undefined, why?: string): ComputerResult => {+ const finish = async (verdict: WaitResult["verdict"], timedOut: boolean, final: ComputerResult | undefined, why?: string): Promise<ComputerResult> => {const verification = { verdict, conditions: results, samples, elapsed_ms: Math.round(performance.now() - started), timed_out: timedOut,...(why ? { reason: why } : {}), spec };const message = `Verification ${verdict} after ${samples} sample(s) in ${verification.elapsed_ms} ms${timedOut ? " (timed out)" : ""}.${why ? ` ${why}` : ""}`;const d = final?.details;+ await this.activity(entry!, { kind: "verify" }, signal, undefined, verdict === "satisfied" ? "verified" : "unknown");+ if (!this.allowed(entry!) || signal?.aborted) return this.result("refused", "Verification revoked or cancelled; data withheld.");return this.result("observed", message, undefined, { ...this.addressed(entry!), verification,...(d?.observation_id ? { observation_id: d.observation_id } : {}), ...(d?.ui ? { ui: d.ui } : {}), ...(d?.dom ? { dom: d.dom } : {}) });};@@ -1557,9 +1582,9 @@ export class ComputerSession {if (images.length !== 1 || !Number.isSafeInteger(width) || !Number.isSafeInteger(height) || (width as number) <= 0 || (height as number) <= 0)throw new Error("Driver returned no window image.");if (!await this.current(entry.ref, signal) || !this.allowed(entry)) return this.result("refused", "Window changed during capture. Discover again.");- const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal);- return this.result("observed", `Full-resolution window pixels: ${width}x${height}.`, undefined,- { ...this.addressed(entry, state.virtual_desktop), ...(highlight ? { highlight } : {}) }, images);+ entry.visualBounds = screenRect(state.window_bounds);+ return this.result("observed", `Full-resolution window pixels: ${width}x${height}.`, { window_bounds: entry.visualBounds, screenshot_width: width, screenshot_height: height },+ this.addressed(entry, state.virtual_desktop), images);} catch (error) { return this.failed(error, false); }});}@@ -1840,7 +1865,10 @@ export class ComputerSession {observed.settings.hints ?? true, request.operation === "highlight" ? request.hint : undefined, observed.ui.elements,{ options: observed.settings.guides, action: observed.actionMark });if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; pixels withheld.");- if (rendered.image !== observed.rawImage) this.retainRaw?.(observed.rawImage);+ const selected = request.hint && observed.ui.elements.find(element => element.hint === request.hint);+ const region = selected && observed.frames.get(selected.element_token);+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal, rendered.image);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; pixels withheld.");const metadata = this.imageMetadata([rendered.image], { screenshot_width: observed.width, screenshot_height: observed.height }, observed.settings);return this.result("observed", `Read-only ${request.operation} of the existing UI snapshot: ${ui.elements.length}/${ui.matches} elements. No native reread or input; pixels are from the original observation.`,undefined, { ...this.addressed(entry), observation_id: observed.id, ui, source: observed.source,@@ -2024,10 +2052,8 @@ export class ComputerSession {basis: target ? "completed_native_token_target" : "completed_native_click_receipt" }: { reason: "Native action did not report usable target geometry." };}- if (status === "delivered" && step.x !== undefined && step.action === "click" && observed.bounds) {- const point = resolvedPoint;- highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal, { kind: "action", point });- }+ highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal,+ { kind: "action", ...(resolvedPoint ? { point: resolvedPoint } : step.element_token && mark.region ? { region: mark.region } : {}) }, undefined, undefined, status);if (!this.allowed(entry) || (!batch && signal?.aborted)) {entry.lastAction = { reason: "In-flight pointer outcome was withheld after revocation or cancellation." };return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");diff --git a/src/vision-debug.ts b/src/vision-debug.tsdeleted file mode 100644index d59f893..0000000--- a/src/vision-debug.ts+++ /dev/null@@ -1,68 +0,0 @@-import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";-import { VisionTrace } from "./vision-trace.ts";-import { CuaImageSettings } from "./cua-image-settings.ts";-import { startVisionServer, type VisionServer } from "./vision-server.ts";--/** Live debug state for trusted host integrations and verification, never model control. */-export interface VisionDebug { trace: VisionTrace; url(): string | undefined; close(): Promise<void> }--/** Mirror existing Pi hooks and expose session image settings. No provider override or new agent tool. */-export function registerVisionDebug(pi: ExtensionAPI, images: CuaImageSettings = new CuaImageSettings()): VisionDebug {- const trace = new VisionTrace("unstarted");- let server: VisionServer | undefined;- let loading: Promise<VisionServer> | undefined;- let sessionId: string | undefined;- let enabled = false;- let imageSession: string | undefined;- const close = async () => {- enabled = false;- if (loading) await loading.catch(() => undefined);- await server?.close(); server = undefined; sessionId = undefined;- trace.reset("closed");- };- const start = async (ctx: ExtensionContext) => {- const id = ctx.sessionManager.getSessionId();- if (imageSession !== id) { images.reset(); imageSession = id; }- if (id !== sessionId) { await close(); sessionId = id; trace.reset(id); }- if (!server) {- loading ??= startVisionServer(trace, images);- try { server = await loading; } finally { loading = undefined; }- }- enabled = true;- pi.appendEntry("vision-debug-viewer", { url: server.url, sessionId: id, boundary: "application; final delivery unconfirmed" });- ctx.ui.setStatus("vision-debug", "vision debug · application boundary");- ctx.ui.notify(`Vision debug: ${server.url}\nApplication boundary only; final delivery is unconfirmed.`, "info");- };- pi.on("session_start", async (_event, ctx) => { await start(ctx); });- pi.on("session_shutdown", async (_event, ctx) => {- await close(); ctx.ui.setStatus("vision-debug", undefined);- });- // Computer actions arrive as standalone computer_* tools or, with Pi Agent IDE, as ordinary tools on computer: paths.- const tracked = (name: string, input: unknown) => name.startsWith("computer_")- || (["read", "search", "write", "insert"].includes(name) && typeof (input as { path?: unknown } | undefined)?.path === "string"- && (input as { path: string }).path.startsWith("computer:"));- pi.on("tool_call", (event) => {- if (enabled && tracked(event.toolName, event.input)) trace.startAction(event.toolCallId, event.toolName, event.input, event.parentToolCallId);- });- pi.on("tool_result", (event) => {- if (enabled && tracked(event.toolName, event.input)) trace.finishAction(event.toolCallId, event.content, event.details, event.isError);- });- pi.on("before_provider_request", (event, ctx) => {- if (!enabled) return;- const model = ctx.model;- trace.observePayload(event.payload, { provider: model?.provider ?? "unknown", api: model?.api ?? "unknown", model: model?.id ?? "unknown" });- // Observation-only: returning data here would replace the provider payload.- });- pi.on("after_provider_response", (event) => { if (enabled) trace.response(event.status); });- pi.registerCommand("computer-debug", {- description: "Show, start or stop the session's vision debug viewer and image controls",- handler: async (args, ctx) => {- const value = args.trim();- if (value === "off") { await close(); ctx.ui.setStatus("vision-debug", undefined); ctx.ui.notify("Vision debug stopped; retained pixels cleared.", "info"); }- else if (value === "on") await start(ctx);- else if (!value || value === "status") ctx.ui.notify(server ? `Vision debug: ${server.url}\nFinal delivery is unconfirmed.` : "Vision debug is off. Use /computer-debug on.", "info");- else ctx.ui.notify("Usage: /computer-debug [on|off|status]", "warning");- },- });- return { trace, url: () => server?.url, close };-}diff --git a/src/vision-panel.ts b/src/vision-panel.tsdeleted file mode 100644index d6afae3..0000000--- a/src/vision-panel.ts+++ /dev/null@@ -1,206 +0,0 @@-/** Static viewer. All observed text goes through textContent, never HTML evaluation. */-export const visionPanel = `<!doctype html>-<html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width">-<title>Pi vision debug</title>-<link rel="stylesheet" href="style.css">-<main>-<header><div><p class="eyebrow">PI · COMPUTER USE</p><h1>Vision debug</h1></div><span id="connection">Connecting…</span></header>-<p class="notice"><strong>Final delivery is unconfirmed.</strong> These are exact image bytes at Pi's existing application-boundary hook, not confirmed wire payloads or model embeddings.</p>-<p id="session"></p>-<section class="controls"><h2>Agent image size</h2>-<p class="muted" id="route-note">Native CUA window/display PNG overviews; browser captures include chrome. The selected native runtime fixes the overview filter (Lanczos3 or Bilinear); default long edge is 1280px. Window detail is fresh padded JPEG; display detail crops fresh full-resolution CUA pixels in the adapter. Raw returned PNGs are retained only for private hint comparison; no full-resolution originals. These controls apply to the next overview, not per-request detail settings.</p>-<form id="image-settings">-<label>Driver kernel <select id="kernel" disabled><option value="lanczos3" selected>Lanczos3</option><option value="bilinear">Bilinear</option></select></label>-<label>Max long edge (px) <input id="long-edge" type="text" inputmode="numeric" pattern="[0-9]+" value="1280" required></label>-<label>Scale slider <input id="scale" type="range" min="0.05" max="1" step="any" value="1"><output id="scale-value">1</output></label>-<label>Scale exact <input id="scale-number" type="text" inputmode="decimal" value="1" required></label>-<button type="submit">Apply to next capture</button>-</form><p id="settings-status" role="status"></p></section>-<section class="controls"><h2>UI hints</h2><label><input id="hints" type="checkbox" checked> Draw short UI hints on the next window overview</label><p id="hints-status" role="status">Raw returned PNG and sent annotated PNG stay separate. No extra image is sent.</p></section>-<section class="controls"><h2>Image guides</h2>-<label><input id="guide-grid" type="checkbox"> Coordinate grid</label>-<label><input id="guide-action" type="checkbox"> Last completed pointer target</label>-<p id="guides-status" role="status">Window overview pixels only. Later captures change; sent images stay frozen. Missing native geometry is not guessed.</p></section>-<section><h2>Last three boundary images</h2><p class="muted">Oldest → newest. Only this viewer is limited to three cards. Images stay in Pi context until normal compaction; these cards mirror the provider hook, not confirmed transport delivery.</p><div id="frames" class="frames"></div></section>-<section><h2>Temporal frames at the boundary</h2><p class="muted">Up to 40 sequence image blocks from the latest request, in payload order. Selection alone is not forwarding. Matching uses exact image bytes; identical frames can have several possible source timestamps.</p><div id="sequence-images"></div></section>-<section><h2>Observed request text</h2><p class="muted">Exact bounded text excerpts at the same hook. Adapter structures below are separate, not proof of forwarded content.</p><div id="request"></div></section>-<section><h2>Computer-use action trace</h2><p class="muted">Includes nested Codemode calls, captured-only images, errors and adapter metadata.</p><div id="actions"></div></section>-<section><h2>Limits</h2><ul id="limits"></ul></section>-</main><script src="panel.js"></script></html>`;--export const visionStyle = `-:root{color-scheme:dark;font:15px/1.5 system-ui;background:#111722;color:#e6edf7}-*{box-sizing:border-box}body{margin:0}main{max-width:1480px;margin:auto;padding:28px}-header{display:flex;justify-content:space-between;align-items:center;gap:16px}h1{margin:0;font-size:30px}-h2{font-size:21px;margin:0 0 8px}.eyebrow{letter-spacing:.13em;color:#7db9ff;margin:0;font-size:12px}-section{margin-top:30px}.notice{background:#332b17;border:1px solid #716033;padding:14px;border-radius:9px}-.controls form{display:flex;gap:20px;flex-wrap:wrap;align-items:end}.controls label{display:flex;flex-direction:column;gap:6px}select,input,button{font:inherit;color:inherit;background:#1b2432;border:1px solid #566b89;border-radius:6px;padding:8px}input[type=range]{padding:0;width:220px}button{cursor:pointer;background:#24446b}#settings-status{color:#a9d6b7}-.muted,#session,#connection{color:#aebbd0}.frames{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:16px}-.card,details{background:#1b2432;border:1px solid #344258;border-radius:10px;padding:14px;min-width:0}-.card img{display:block;width:100%;height:230px;object-fit:contain;background:#0c111a;margin:12px 0}-.card h3{margin:0;font-size:16px}.meta{font:12px/1.6 ui-monospace,monospace;overflow-wrap:anywhere}-pre{font:12px/1.5 ui-monospace,monospace;white-space:pre-wrap;overflow-wrap:anywhere;max-height:400px;overflow:auto}-details{margin:10px 0}summary{cursor:pointer}.error{border-color:#d56b75}.empty{padding:28px;color:#aebbd0;border:1px dashed #344258;border-radius:8px}-@media(max-width:850px){.frames{grid-template-columns:1fr}.card img{height:280px}main{padding:18px}header{align-items:flex-start}}-`;--export const visionScript = `-const el=(tag,text,cls)=>{const node=document.createElement(tag);if(text!==undefined)node.textContent=text;if(cls)node.className=cls;return node};-let revision=-1,session,settingsVersion=-1,settingsDirty=false,settingsBusy=false,controlsReady=false,hintsBusy=false,guidesBusy=false;-function settingsDraft(){- return {kernel:document.getElementById("kernel").value,maxLongEdge:Number(document.getElementById("long-edge").value),scale:Number(document.getElementById("scale-number").value)};-}-function showDraft(){- const draft=settingsDraft();- document.getElementById("scale-value").textContent=String(draft.scale);- document.getElementById("settings-status").textContent="Draft cap: "+Math.floor(draft.maxLongEdge*draft.scale)+" px · apply to the next capture.";-}-function setupSettings(){- for(const key of ["grid","action"])document.getElementById("guide-"+key).addEventListener("change",async ()=>{- if(guidesBusy)return;guidesBusy=true;- const guides=Object.fromEntries(["grid","action"].map(k=>[k,document.getElementById("guide-"+k).checked]));- try{- const response=await fetch("guides",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(guides)});- if(!response.ok)throw Error(await response.text());await response.json();settingsVersion=-1;- document.getElementById("guides-status").textContent="Applied to later window captures. Earlier images stay unchanged.";- }catch(error){document.getElementById("guides-status").textContent="Not applied: "+error.message}- finally{guidesBusy=false}- });- document.getElementById("hints").addEventListener("change",async ()=>{- if(hintsBusy)return;hintsBusy=true;- try{- const response=await fetch("hints",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({enabled:document.getElementById("hints").checked})});- if(!response.ok)throw Error(await response.text());- const applied=await response.json();- document.getElementById("hints-status").textContent="Hints "+(applied.hints?"on":"off")+" for the next overview. Earlier images are unchanged.";- settingsVersion=-1;- }catch(error){document.getElementById("hints-status").textContent="Not applied: "+error.message}- finally{hintsBusy=false}- });- for(const id of ["kernel","long-edge"])document.getElementById(id).addEventListener("input",()=>{settingsDirty=true;showDraft()});- for(const [id,other] of [["scale","scale-number"],["scale-number","scale"]])document.getElementById(id).addEventListener("input",()=>{document.getElementById(other).value=document.getElementById(id).value;settingsDirty=true;showDraft()});- document.getElementById("image-settings").addEventListener("submit",async event=>{- event.preventDefault();if(settingsBusy)return;settingsBusy=true;- const submitted=settingsDraft();- try{- const response=await fetch("settings",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(submitted)});- if(!response.ok)throw Error(await response.text());- const applied=await response.json();- settingsDirty=JSON.stringify(settingsDraft())!==JSON.stringify(submitted);- document.getElementById("settings-status").textContent="Applied v"+applied.version+": "+applied.kernel+" · cap "+Math.floor(applied.maxLongEdge*applied.scale)+" px · next capture.";- settingsVersion=-1;- }catch(error){document.getElementById("settings-status").textContent="Not applied: "+error.message}- finally{settingsBusy=false}- });-}-function renderSettings(settings){- if(!settings)return;- if(!hintsBusy)document.getElementById("hints").checked=settings.hints!==false;- if(!guidesBusy)for(const key of ["grid","action"])document.getElementById("guide-"+key).checked=settings.guides?.[key]===true;- if(!settingsDirty&&!settingsBusy){- document.getElementById("kernel").value=settings.kernel;- document.getElementById("long-edge").value=String(settings.maxLongEdge);- document.getElementById("scale").value=String(settings.scale);- document.getElementById("scale-number").value=String(settings.scale);- document.getElementById("scale-value").textContent=String(settings.scale);- document.getElementById("settings-status").textContent="Active v"+settings.version+": "+settings.kernel+" · "+settings.maxLongEdge+" × "+settings.scale+" = cap "+Math.floor(settings.maxLongEdge*settings.scale)+" px. No upsampling.";- }-}-function render(state){- renderSettings(state.imageSettings);- document.getElementById("session").textContent="Session "+state.sessionId+" · revision "+state.revision;- const frames=document.getElementById("frames");- const existingFrames=new Map([...frames.children].map(card=>[card.dataset.imageKey,card]));- const frameRows=[],frameCounts=new Map();- for(const [i,frame] of state.frames.entries()){- const base=frame.hash+":"+(frame.originalHash??"")+":"+frame.stage;- const occurrence=frameCounts.get(base)??0;frameCounts.set(base,occurrence+1);- const key=base+":"+occurrence;- let card=existingFrames.get(key);- if(!card){- card=el("article",undefined,"card");card.dataset.imageKey=key;- card.append(el("h3"));- if(frame.stage!=="evicted"){- const img=el("img");img.src="images/"+frame.hash;card.append(img);- if(frame.original==="available"&&frame.originalHash){- const mode=el("select");- for(const [value,label] of [[frame.hash,"Sent image"],[frame.originalHash,"Raw returned capture"]]){const option=el("option",label);option.value=value;mode.append(option)}- mode.addEventListener("change",()=>{img.src="images/"+mode.value});card.append(mode);- }- }- for(let n=0;n<4;n++)card.append(el("p",undefined,n===3?"muted":"meta"));- card.append(el("pre"));- }- card.querySelector("h3").textContent="t"+(i-state.frames.length+1)+" · "+frame.stage;- const img=card.querySelector("img");if(img)img.alt="Application-boundary frame "+frame.id;- const mode=card.querySelector("select");if(mode)mode.setAttribute("aria-label","Image comparison "+frame.id);- const paragraphs=card.querySelectorAll("p");- paragraphs[0].textContent=frame.time+" · "+(frame.width??"?")+"×"+(frame.height??"?")+" · "+frame.bytes+" bytes";- paragraphs[1].textContent="Request "+frame.requestId+" · action "+(frame.actionId??"unmatched");- paragraphs[2].textContent="SHA-256 "+frame.hash;- paragraphs[3].textContent=frame.original==="available"?"Compare the exact raw returned capture with the sent annotation. No full-resolution original is retained.":"Native CUA capture; no separate raw comparison is retained.";- card.querySelector("pre").textContent=JSON.stringify({path:frame.path,detail:frame.detail,source:frame.source,matchingActionIds:frame.matchingActionIds,configuration:frame.configuration,originalHash:frame.originalHash,finalDelivery:frame.finalDelivery},null,2);- frameRows.push(card);- }- if(!state.frames.length)frameRows.push(el("p","No boundary images yet. Captured-only frames do not appear here.","empty"));- for(const child of [...frames.children])if(!frameRows.includes(child))child.remove();- for(const [i,card] of frameRows.entries())if(frames.children[i]!==card)frames.insertBefore(card,frames.children[i]??null);- const request=document.getElementById("request");request.replaceChildren();- const latest=state.requests.at(-1);- if(latest){- request.append(el("p",latest.id+" · "+latest.provider+"/"+latest.model+" · "+latest.imageCount+" image blocks · response "+(latest.responseStatus??"unobserved")+" · final payload unconfirmed","meta"));- request.append(el("p",latest.textTruncated?"Text excerpts are truncated.":"Retained text blocks: "+latest.texts.length,"muted"));- request.append(el("pre",latest.texts.map(item=>item.path+"\\n"+item.text).join("\\n\\n")));- }else request.append(el("p","No request observed yet.","empty"));- const temporal=document.getElementById("sequence-images");- const temporalFrames=latest?.sequenceImages??[];- const temporalSignature=JSON.stringify(temporalFrames);- if(temporal.dataset.signature!==temporalSignature){- const previous=temporal.querySelector("select")?.value;- temporal.dataset.signature=temporalSignature;temporal.replaceChildren();- if(temporalFrames.length){- const card=el("article",undefined,"card"),select=el("select"),img=el("img"),meta=el("pre");- select.setAttribute("aria-label","Temporal frame in payload order");- for(const [i,frame] of temporalFrames.entries()){- const names=frame.matches.map(m=>"frame "+m.index+" @ "+m.timestamp_ms.toFixed(1)+" ms");- const option=el("option","Payload "+i+" · "+(names.length===1?names[0]:names.length+" byte-matched sources")+(frame.available?"":" · evicted"));option.value=String(i);select.append(option);- }- select.value=previous!==undefined&&Number(previous)<temporalFrames.length?previous:"0";- const show=()=>{const frame=temporalFrames[Number(select.value)];img.src=frame.available?"images/"+frame.hash:"";img.alt=frame.available?"Exact application-boundary temporal image":"Image bytes evicted";meta.textContent=JSON.stringify(frame,null,2)};- select.addEventListener("change",show);show();card.append(select,img,meta);temporal.append(card);- }else temporal.append(el("p","No sequence images in the latest request. Capture resources alone send no pixels.","empty"));- }- const actions=document.getElementById("actions");- const existing=new Map([...actions.children].map(item=>[item.dataset.actionId,item]));- const rows=[];- for(const action of [...state.actions].reverse()){- const captures=state.captures.filter(frame=>frame.actionId===action.id);- const signature=JSON.stringify({action,captures});- const previous=existing.get(action.id);- if(previous?.dataset.signature===signature){rows.push(previous);continue}- const item=el("details",undefined,action.isError?"error":"");- item.dataset.actionId=action.id;item.dataset.signature=signature;item.open=previous?.open??false;- item.append(el("summary",action.tool+" · "+action.status+" · "+(action.durationMs===undefined?"running":action.durationMs.toFixed(1)+" ms")+" · "+action.id));- item.append(el("pre",JSON.stringify(action,null,2)));- for(const capture of captures){- item.append(el("p","Adapter image "+capture.hash+" · "+capture.stage+" · delivery unconfirmed","meta"));- if(capture.stage!=="evicted"){const link=el("a","Open adapter image");link.href="images/"+capture.hash;link.target="_blank";link.rel="noopener";item.append(link)}- }- rows.push(item);- }- actions.replaceChildren(...rows);- if(!state.actions.length)actions.append(el("p","No computer-use actions yet.","empty"));- document.getElementById("limits").replaceChildren(...state.limitations.map(value=>el("li",value)));-}-async function poll(){- if(!controlsReady){setupSettings();controlsReady=true}- try{- const response=await fetch("state",{cache:"no-store"});if(!response.ok)throw Error("HTTP "+response.status);- const state=await response.json();- if(state.revision!==revision||state.sessionId!==session||state.imageSettings?.version!==settingsVersion){render(state);revision=state.revision;session=state.sessionId;settingsVersion=state.imageSettings?.version}- document.getElementById("connection").textContent="Connected · "+new Date().toLocaleTimeString();- }catch(error){document.getElementById("connection").textContent="Disconnected · retrying ("+error.message+")"}- setTimeout(poll,1000);-}-poll();-`;diff --git a/src/vision-server.ts b/src/vision-server.tsdeleted file mode 100644index 573b6b6..0000000--- a/src/vision-server.ts+++ /dev/null@@ -1,76 +0,0 @@-import { randomBytes } from "node:crypto";-import { createServer } from "node:http";-import { VisionTrace } from "./vision-trace.ts";-import { CuaImageSettings } from "./cua-image-settings.ts";-import { visionPanel, visionScript, visionStyle } from "./vision-panel.ts";--/** A private loopback viewer with bounded native capture settings. */-export interface VisionServer { url: string; close(): Promise<void> }--/** Start from a session/command. Settings affect following captures, not historical frames. */-export async function startVisionServer(trace: VisionTrace, images: CuaImageSettings = new CuaImageSettings()): Promise<VisionServer> {- const token = randomBytes(24).toString("hex");- let origin = "";- const server = createServer(async (request, response) => {- response.setHeader("Cache-Control", "no-store");- response.setHeader("X-Content-Type-Options", "nosniff");- response.setHeader("Referrer-Policy", "no-referrer");- response.setHeader("Content-Security-Policy", "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'");- if (request.headers.host !== new URL(origin).host || (request.headers.origin && request.headers.origin !== origin)) {- response.writeHead(403).end("Foreign host or origin."); return;- }- const url = new URL(request.url ?? "/", origin);- const base = `/${token}/`;- if (!url.pathname.startsWith(base)) { response.writeHead(404).end("Not found."); return; }- const route = url.pathname.slice(base.length);- if (route === "trial" || route.startsWith("trial/") || route === "trial.js" || route === "trial.css") {- response.writeHead(410).end("The old resampling trial is retired on the native CUA route. Native overview uses Lanczos3; detail uses fresh padded zoom."); return;- }- if (request.method === "POST" && ["settings", "hints", "guides"].includes(route)) {- const limit = 1024;- if (request.headers.origin !== origin || request.headers["content-type"] !== "application/json") {- response.writeHead(403).end("Settings require same-origin JSON."); request.resume(); return;- }- try {- if (Number(request.headers["content-length"]) > limit) { response.writeHead(413).end("Settings body exceeds limit."); request.resume(); return; }- const chunks: Buffer[] = []; let length = 0;- for await (const chunk of request) {- length += chunk.length;- if (length > limit) { response.writeHead(413).end("Settings body exceeds limit."); request.resume(); return; }- chunks.push(Buffer.from(chunk));- }- const input: unknown = JSON.parse(Buffer.concat(chunks).toString("utf8"));- if (route === "hints" && (!input || typeof input !== "object" || Array.isArray(input) ||- Object.keys(input).join(",") !== "enabled" || typeof (input as { enabled?: unknown }).enabled !== "boolean")) throw Error("Expected only boolean enabled.");- const applied = route === "hints" ? images.setHints((input as { enabled: boolean }).enabled) : route === "guides" ? images.setGuides(input) : images.update(input);- response.setHeader("Content-Type", "application/json"); response.end(JSON.stringify(applied));- } catch (error) { response.writeHead(400).end(error instanceof Error ? error.message : String(error)); }- return;- }- if (!["GET", "HEAD"].includes(request.method ?? "")) { response.writeHead(405).end("Only settings can be written."); return; }- let body: string | Buffer | undefined;- let mime = "text/plain; charset=utf-8";- if (route === "") { body = visionPanel; mime = "text/html; charset=utf-8"; }- else if (route === "panel.js") { body = visionScript; mime = "text/javascript; charset=utf-8"; }- else if (route === "style.css") { body = visionStyle; mime = "text/css; charset=utf-8"; }- else if (route === "state") { body = JSON.stringify({ ...trace.snapshot(), imageSettings: images.settings() }); mime = "application/json"; }- else if (/^images\/[a-f0-9]{64}$/.test(route)) {- const hash = route.slice(7); body = trace.asset(hash); mime = trace.assetMime(hash) ?? "image/png";- }- if (body === undefined) { response.writeHead(404).end("Not retained in this session."); return; }- response.setHeader("Content-Type", mime);- response.end(request.method === "HEAD" ? undefined : body);- });- await new Promise<void>((resolve, reject) => {- server.once("error", reject);- server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); });- });- const address = server.address();- if (!address || typeof address === "string") { server.close(); throw new Error("No loopback server address."); }- origin = `http://127.0.0.1:${address.port}`;- let closing: Promise<void> | undefined;- return { url: `${origin}/${token}/`, close: () => closing ??= new Promise<void>((resolve, reject) => {- server.close(error => error ? reject(error) : resolve());- server.closeAllConnections();- }) };-}diff --git a/src/vision-trace.ts b/src/vision-trace.tsdeleted file mode 100644index e280d89..0000000--- a/src/vision-trace.ts+++ /dev/null@@ -1,244 +0,0 @@-import { createHash } from "node:crypto";--/** Provider identity at Pi's existing, mutable application-boundary hook. */-export interface VisionModel { provider: string; api: string; model: string }-interface ImageBytes { bytes: Buffer; mime: string; hash: string; width?: number; height?: number }-interface SequenceMatch { sequence_id: string; index: number; timestamp_ms: number }-interface Frame {- id: string; hash: string; mime: string; bytes: number; width?: number; height?: number;- time: string; stage: "captured" | "prepared" | "evicted"; finalDelivery: "unconfirmed";- original: "available" | "unavailable"; originalHash?: string; actionId?: string; matchingActionIds?: string[]; requestId?: string; path?: string; detail?: string;- source?: unknown; configuration?: unknown;- sequenceFrame?: SequenceMatch;-}-interface Action {- id: string; parentId?: string; tool: string; input: unknown; time: string; started: number;- durationMs?: number; status: string; isError?: boolean; text?: string; adapterStructure?: unknown;-}-interface Request extends VisionModel {- id: string; time: string; imageCount: number; unavailableImages: number;- texts: { path: string; text: string }[]; textTruncated: boolean;- frameIds: string[]; finalPayload: "unconfirmed"; responseStatus?: number;- sequenceImages: { hash: string; path: string; matches: SequenceMatch[]; available?: boolean }[];-}-const limitations = [- "Final delivery is unconfirmed: later handlers can replace this application-boundary payload.",- "Codex WebSocket may send a delta; wire payload and effective server context are not observed.",- "The three-card viewer does not limit the model context.",- "Native CUA exposes returned PNG/JPEG bytes, not retained full-resolution originals.",- "Native JPEG dimensions are reported by matching adapter metadata. Provider-internal resize and final delivery are unobserved.",- "Action links match image bytes; identical screenshots can have more than one source.",-];-const maxImageBytes = 12 * 1024 * 1024;-const maxAssetBytes = 48 * 1024 * 1024;--function imageBytes(data: string, mime: string): ImageBytes | undefined {- if (!["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mime) ||- data.length > Math.ceil(maxImageBytes / 3) * 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(data)) return;- const bytes = Buffer.from(data, "base64");- if (!bytes.length || bytes.length > maxImageBytes) return;- const png = mime === "image/png" && bytes.length >= 24 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]));- return { bytes, mime, hash: createHash("sha256").update(bytes).digest("hex"),- ...(png ? { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) } : {}) };-}-function dataUrl(value: unknown): ImageBytes | undefined {- if (typeof value !== "string") return;- const match = /^data:(image\/[a-z]+);base64,([\s\S]*)$/.exec(value);- return match ? imageBytes(match[2], match[1]) : undefined;-}-/** Bound adapter metadata, strip image data and common secret fields, and reject cycles. */-function metadata(value: unknown): unknown {- try {- const json = JSON.stringify(value, function (key, item) {- // A source's image field is geometry, not an image payload.- if (key === "image" && ["windows-host", "linux-host", "macos-host", "windows-sandbox", "windows-session", "windows-remote"].includes(this.domain) && ["display", "window"].includes(this.kind)) return item;- return ["image", "png", "data", "authorization", "apiKey", "access_token", "refresh_token", "password"].includes(key) ? undefined : item;- });- if (json === undefined) return undefined;- return json.length <= 32_000 ? JSON.parse(json) : { truncated: true, excerpt: json.slice(0, 32_000) };- } catch { return { unavailable: "Metadata could not be serialized." }; }-}--/** Bounded, session-local mirror. Never mutates context or asserts confirmed transport delivery. */-export class VisionTrace {- private frames: Frame[] = [];- private captures: Frame[] = [];- private actions: Action[] = [];- private requests: Request[] = [];- private assets = new Map<string, ImageBytes>();- private originals: string[] = [];- private sequence = 0;- private revision = 0;- constructor(private sessionId: string) {}-- /** Drop all previous-session pixels, metadata and correlation state. */- reset(sessionId: string): void {- this.sessionId = sessionId;- this.frames = []; this.captures = []; this.actions = []; this.requests = [];- this.assets.clear(); this.originals = []; this.sequence = 0; this.revision++;- }-- /** Keep recent raw returned captures and visible-card comparisons within the asset budget; never send another image. */- retainOriginal(value: { type: string; data?: string; mimeType?: string }): void {- if (value.type !== "image" || typeof value.data !== "string" || typeof value.mimeType !== "string") return;- const image = imageBytes(value.data, value.mimeType);- if (!image) return;- this.assets.set(image.hash, image);- this.originals = [...this.originals.filter(hash => hash !== image.hash), image.hash].slice(-3);- this.prune();- }- /** Record a computer-use call, including its Codemode parent when available. */- startAction(id: string, tool: string, input: unknown, parentId?: string): void {- this.actions.push({ id, parentId, tool, input: metadata(input), time: new Date().toISOString(), started: performance.now(), status: "running" });- this.actions = this.actions.slice(-100); this.revision++;- }-- /** Record adapter output separately from what a parent script forwards to the model. */- finishAction(id: string, content: readonly unknown[], details: unknown, isError: boolean): void {- const action = this.actions.find(item => item.id === id);- if (!action) return;- action.durationMs = performance.now() - action.started;- action.isError = isError;- action.adapterStructure = metadata(details);- const state = details && typeof details === "object" ? details as Record<string, unknown> : {};- action.status = typeof state.status === "string" ? state.status : isError ? "error" : "completed";- action.text = content.map(block => {- const value = block as Record<string, unknown>;- return value?.type === "text" && typeof value.text === "string" ? value.text : "";- }).filter(Boolean).join("\n").slice(0, 16_000);- const reported = state.agent_image as { output?: { hash?: unknown; width?: unknown; height?: unknown }; route?: unknown; coordinate_space?: unknown; native?: unknown; origin?: unknown } | undefined;- const annotation = (state.agent_image as { annotation?: { raw?: { hash?: unknown } } } | undefined)?.annotation;- const rawHash = annotation?.raw?.hash;- const originalHash = typeof rawHash === "string" && this.assets.has(rawHash) ? rawHash : undefined;- const selectedFrames = Array.isArray(state.selected_frames) ? state.selected_frames as { index: number; timestamp_ms: number; source?: unknown }[] : [];- const sequenceId = (action.input as { sequence_id?: string } | undefined)?.sequence_id;- let imageIndex = 0;- for (const block of content) {- const value = block as Record<string, unknown>;- if (value?.type !== "image" || typeof value.data !== "string" || typeof value.mimeType !== "string") continue;- const image = imageBytes(value.data, value.mimeType);- const temporal = selectedFrames[imageIndex++];- if (!image) continue;- // JPEG geometry comes from matching native output metadata, not a new decoder.- if (reported?.output?.hash === image.hash) {- const { width, height } = reported.output;- if (Number.isSafeInteger(width) && (width as number) > 0 && Number.isSafeInteger(height) && (height as number) > 0) {- image.width ??= width as number; image.height ??= height as number;- }- }- this.assets.set(image.hash, image);- this.captures.push({ ...this.frame(image, "captured"), actionId: id, ...(originalHash ? { original: "available", originalHash } : {}),- ...(temporal && sequenceId ? { sequenceFrame: { sequence_id: sequenceId, index: temporal.index, timestamp_ms: temporal.timestamp_ms } } : {}),- source: metadata({ observation: temporal?.source ?? state.source, tab: state.tab, window_ref: state.window_ref, observation_id: state.observation_id, screenshot: state.screenshot, origin: reported?.origin, route: reported?.route, coordinate_space: reported?.coordinate_space, native: reported?.native, annotation }),- configuration: metadata({ requested: action.input, effective: state.agent_image ?? "unreported by current adapter", timings: state.timings }) });- this.captures = [...this.captures.filter(f => f.sequenceFrame).slice(-40), ...this.captures.filter(f => !f.sequenceFrame).slice(-3)];- }- this.prune(); this.revision++;- }-- private frame(image: ImageBytes, stage: Frame["stage"]): Frame {- return { id: `frame-${++this.sequence}`, hash: image.hash, mime: image.mime, bytes: image.bytes.length,- width: image.width, height: image.height, time: new Date().toISOString(), stage,- finalDelivery: "unconfirmed", original: "unavailable" };- }-- /** Snapshot the existing hook payload, not final HTTP bytes or provider embeddings. */- observePayload(payload: unknown, model: VisionModel): void {- const request: Request = { ...model, id: `request-${++this.sequence}`, time: new Date().toISOString(), imageCount: 0,- unavailableImages: 0, texts: [], textTruncated: false, frameIds: [], sequenceImages: [], finalPayload: "unconfirmed" };- const selected: { image: ImageBytes; path: string; detail?: string }[] = [];- let nodes = 0;- let textBytes = 0;- const seen = new WeakSet<object>();- const visit = (value: unknown, path: string, depth: number): void => {- if (!value || typeof value !== "object" || depth > 40 || ++nodes > 100_000 || seen.has(value)) return;- seen.add(value);- if (Array.isArray(value)) { value.forEach((item, index) => visit(item, `${path}[${index}]`, depth + 1)); seen.delete(value); return; }- const node = value as Record<string, unknown>;- let image: ImageBytes | undefined;- const inline = (node.inlineData ?? node.inline_data) as Record<string, unknown> | undefined;- const isImage = ["input_image", "image_url", "image"].includes(String(node.type)) || !!inline;- if (isImage) {- request.imageCount++;- const source = node.source as Record<string, unknown> | undefined;- if (typeof node.image_url === "string") image = dataUrl(node.image_url);- else if (node.image_url && typeof node.image_url === "object") image = dataUrl((node.image_url as Record<string, unknown>).url);- else if (source?.type === "base64" && typeof source.data === "string" && typeof source.media_type === "string") image = imageBytes(source.data, source.media_type);- else if (inline && typeof inline.data === "string") image = imageBytes(inline.data, String(inline.mimeType ?? inline.mime_type));- else if (typeof node.data === "string" && typeof node.mimeType === "string") image = imageBytes(node.data, node.mimeType);- if (image) {- selected.push({ image, path, detail: typeof node.detail === "string" ? node.detail : undefined });- const matches = [...new Map(this.captures.filter(f => f.hash === image!.hash && f.sequenceFrame)- .map(f => { const frame = f.sequenceFrame!; return [`${frame.sequence_id}:${frame.index}`, frame] as const; })).values()];- if (matches.length) {- request.sequenceImages.push({ hash: image.hash, path, matches });- request.sequenceImages = request.sequenceImages.slice(-40);- this.assets.set(image.hash, image);- }- if (selected.length > 3) selected.shift();- } else request.unavailableImages++;- seen.delete(value); return;- }- const candidate = ["input_text", "output_text", "text"].includes(String(node.type)) ? node.text- : ["function_call_output", "custom_tool_call_output"].includes(String(node.type)) ? node.output- : node.role === "tool" && typeof node.content === "string" ? node.content : undefined;- if (typeof candidate === "string") {- const excerpt = candidate.slice(0, 16_000);- if (excerpt) request.texts.push({ path, text: excerpt });- textBytes += excerpt.length;- while (textBytes > 64_000) {- textBytes -= request.texts.shift()!.text.length;- request.textTruncated = true;- }- if (excerpt.length !== candidate.length) request.textTruncated = true;- }- for (const [key, item] of Object.entries(node)) visit(item, `${path}.${key}`, depth + 1);- seen.delete(value);- };- visit(payload, "$", 0);- if (nodes > 100_000) request.textTruncated = true;- for (const { image, path, detail } of selected) {- const matches = this.captures.filter(item => item.hash === image.hash);- const capture = matches.length === 1 ? matches[0] : undefined;- const frame = { ...this.frame(image, "prepared"), width: image.width ?? capture?.width, height: image.height ?? capture?.height, actionId: capture?.actionId, source: capture?.source,- configuration: capture?.configuration, original: capture?.original ?? "unavailable", originalHash: capture?.originalHash, matchingActionIds: matches.map(item => item.actionId!).filter(Boolean), requestId: request.id, path, detail };- this.assets.set(image.hash, image);- this.frames.push(frame); request.frameIds.push(frame.id);- }- this.frames = this.frames.slice(-3);- this.requests.push(request); this.requests = this.requests.slice(-20);- this.prune(); this.revision++;- }-- /** A provider response is useful evidence, but not proof of unchanged image delivery. */- response(status: number): void {- const request = this.requests.at(-1);- if (request) { request.responseStatus = status; this.revision++; }- }-- private prune(): void {- const retained = new Set([...this.originals, ...(this.requests.at(-1)?.sequenceImages.map(f => f.hash) ?? []),- ...[...this.frames, ...this.captures].flatMap(frame => [frame.hash, ...(frame.originalHash ? [frame.originalHash] : [])])]);- for (const hash of this.assets.keys()) if (!retained.has(hash)) this.assets.delete(hash);- let size = [...this.assets.values()].reduce((sum, image) => sum + image.bytes.length, 0);- for (const [hash, image] of this.assets) {- if (size <= maxAssetBytes) break;- this.assets.delete(hash); size -= image.bytes.length;- }- for (const frame of [...this.frames, ...this.captures]) {- if (!this.assets.has(frame.hash)) frame.stage = "evicted";- if (frame.originalHash && !this.assets.has(frame.originalHash)) frame.original = "unavailable";- }- }-- /** Return retained bytes only; never download an external image URL. */- asset(hash: string): Buffer | undefined { return this.assets.get(hash)?.bytes; }- /** Content type for a retained image asset. */- assetMime(hash: string): string | undefined { return this.assets.get(hash)?.mime; }- /** JSON-safe viewer state without image base64. Retained UI text can still be sensitive. */- snapshot() {- return structuredClone({ sessionId: this.sessionId, revision: this.revision, limitations,- frames: this.frames, captures: this.captures, actions: this.actions,- requests: this.requests.map(request => ({ ...request, sequenceImages: request.sequenceImages.map(f => ({ ...f, available: this.assets.has(f.hash) })) })) });- }-}diff --git a/src/visual-settings-command.ts b/src/visual-settings-command.tsnew file mode 100644index 0000000..96a691d--- /dev/null+++ b/src/visual-settings-command.ts@@ -0,0 +1,32 @@+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";+import { VisualSettings, type VisualOptions } from "./native-visuals.ts";++/** Human-only controls; persistence contains preferences, never images or input text. */+export function registerVisualSettingsCommand(pi: ExtensionAPI, settings: VisualSettings): void {+ const restore = (ctx: ExtensionContext) => {+ let saved: Partial<VisualOptions> = { preset: "full" };+ for (const entry of ctx.sessionManager.getBranch()) {+ if (entry.type === "custom" && entry.customType === "computer-use-visuals") saved = entry.data as Partial<VisualOptions>;+ }+ try { settings.restore(saved); } catch { settings.restore({ preset: "off" }); }+ };+ pi.on("session_start", (_event, ctx) => restore(ctx));+ pi.on("session_tree", (_event, ctx) => restore(ctx));+ pi.registerCommand("computer-visuals", {+ description: "Show or change native computer visuals: off, subtle, full, or JSON presentation preferences. No input or capture changes.",+ getArgumentCompletions: prefix => ["off", "subtle", "full"].filter(value => value.startsWith(prefix)).map(value => ({ value, label: value })),+ handler: async (args, ctx) => {+ try {+ const text = args.trim();+ if (text) {+ const patch = ["off", "subtle", "full"].includes(text) ? { preset: text as VisualOptions["preset"] } : JSON.parse(text);+ pi.appendEntry("computer-use-visuals", settings.update(patch));+ }+ const value = settings.current();+ ctx.ui.notify(`Computer visuals: ${value.preset}. Previews ${value.previews ? "on" : "off"}; ${value.quality} quality; opacity ${Math.round(value.opacity * 100)}%; reduced motion ${value.reduced_motion ? "on" : "off"}.`, "info");+ } catch {+ ctx.ui.notify('Use /computer-visuals off|subtle|full, or JSON such as {"previews":false,"reduced_motion":true}. Preferences were not changed.', "warning");+ }+ },+ });+}diff --git a/src/window-highlights.ts b/src/window-highlights.tsdeleted file mode 100644index 560eb23..0000000--- a/src/window-highlights.ts+++ /dev/null@@ -1,42 +0,0 @@-import { readdir, stat } from "node:fs/promises";-import path from "node:path";-import { WindowOverlay, type OverlayPeer, type WindowObserver } from "./window-overlay.ts";-import { startOverlayPeer } from "./window-overlay-peer.ts";--/** Find only a trusted user-installed Windows rendering helper. No model-supplied executable or auto-install. */-export async function findWindowOverlay(): Promise<string> {- if (process.env.PI_WINDOW_OVERLAY_PATH) return process.env.PI_WINDOW_OVERLAY_PATH;- const suffix = ["Programs", "PiComputerUsePoC", "window-overlay-0.1.0", "pi-window-overlay.exe"];- const candidates = process.platform === "win32"- ? [path.join(process.env.LOCALAPPDATA ?? "", ...suffix)]- : (await readdir("/mnt/c/Users").catch(() => [] as string[]))- .map(user => path.join("/mnt/c/Users", user, "AppData", "Local", ...suffix));- const found: string[] = [];- for (const candidate of candidates) if ((await stat(candidate).catch(() => undefined))?.isFile()) found.push(candidate);- if (found.length !== 1) throw new Error("Install one qualified Windows window overlay, or set PI_WINDOW_OVERLAY_PATH.");- return found[0];-}--/** Share one lazy rendering peer. Explicit confirmed close permits a fresh peer; unknown shutdown never does. */-export class WindowHighlights {- private current: WindowOverlay;- private stopping?: Promise<void>;- constructor(private readonly createPeer: () => Promise<OverlayPeer> = async () => startOverlayPeer(await findWindowOverlay())) {- this.current = new WindowOverlay(createPeer);- }-- /** Only authorized observation/action events draw; clearing an unused peer never starts a helper. */- readonly flash: WindowObserver = (frame, signal) => this.stopping- ? Promise.resolve({ status: "skipped" }) : this.current.flash(frame, signal);-- /** Revoke the current frame, including an in-flight startup, without starting a helper. */- clear(): Promise<void> { return this.current.clear(); }-- /** Close only the helper. A failed cleanup stays blocked until extension reload. */- close(): Promise<void> {- if (this.stopping) return this.stopping;- return this.stopping = this.current.close().then(() => {- this.current = new WindowOverlay(this.createPeer); this.stopping = undefined;- });- }-}diff --git a/src/window-overlay-peer.ts b/src/window-overlay-peer.tsdeleted file mode 100644index dbefec6..0000000--- a/src/window-overlay-peer.ts+++ /dev/null@@ -1,79 +0,0 @@-import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";-import { createInterface } from "node:readline";-import type { OverlayCommand, OverlayPeer, OverlayReply } from "./window-overlay.ts";--/** Spawn only a trusted helper binary. It speaks bounded JSON-line requests and exits on stdin EOF. */-export async function startOverlayPeer(command: string, args: string[] = [], interop?: string): Promise<NativeOverlayPeer> {- const peer = new NativeOverlayPeer(spawn(command, args, { stdio: "pipe", windowsHide: true,- env: { ...process.env, ...(interop ? { WSL_INTEROP: interop } : {}) } }));- try { await peer.started; return peer; }- catch (error) { await peer.close().catch(() => undefined); throw error; }-}--/** Own the rendering child's protocol and cleanup, never a source application or a shared Windows process. */-export class NativeOverlayPeer implements OverlayPeer {- readonly started: Promise<void>;- ready?: Record<string, unknown>;- private pending = new Map<number, { finish(error?: Error, reply?: OverlayReply): void }>();- private closed = false;- private failure?: Error;- private closePromise?: Promise<void>;-- constructor(private readonly child: ChildProcessWithoutNullStreams) {- this.started = new Promise<void>((resolve, reject) => {- const timer = setTimeout(() => fail(new Error("Qt frame startup timed out.")), 10_000);- const fail = (error: Error) => {- clearTimeout(timer); this.failure = error; reject(error);- for (const item of [...this.pending.values()]) item.finish(error);- this.child.stdin.end();- };- child.once("error", fail);- child.once("exit", () => fail(new Error("Qt frame process exited.")));- child.stderr.resume();- createInterface({ input: child.stdout }).on("line", line => {- try {- if (line.length > 16384) throw new Error("Oversized Qt frame reply.");- const reply = JSON.parse(line) as Record<string, unknown>;- if (!reply || typeof reply !== "object" || typeof reply.status !== "string" || !Number.isSafeInteger(reply.generation))- throw new Error("Invalid Qt frame reply.");- if (reply.generation === 0 && !this.ready) {- if (reply.status !== "ready" || reply.protocol_version !== 2 || reply.capture_affinity !== 0x11 || !Number.isSafeInteger(reply.pid) || !Number.isSafeInteger(reply.overlay_hwnd))- throw new Error("Qt frame capture exclusion is unavailable.");- this.ready = reply; clearTimeout(timer); resolve();- } else this.pending.get(reply.generation as number)?.finish(undefined, reply as unknown as OverlayReply);- } catch { fail(new Error("Qt frame protocol failed.")); }- });- });- }-- /** Wait for the exact generation acknowledgement; never repeat a timed-out request. */- async send(command: OverlayCommand): Promise<OverlayReply> {- await this.started;- if (this.closed || this.failure || this.child.exitCode !== null) throw new Error("Qt frame peer is unavailable.");- if (this.pending.has(command.generation)) throw new Error("Duplicate Qt frame generation.");- return new Promise((resolve, reject) => {- const finish = (error?: Error, reply?: OverlayReply) => {- clearTimeout(timer); this.pending.delete(command.generation);- if (error) reject(error); else resolve(reply!);- };- const timer = setTimeout(() => {- this.failure = new Error("Qt frame acknowledgement timed out."); this.child.stdin.end(); finish(this.failure);- }, 2000);- this.pending.set(command.generation, { finish });- this.child.stdin.write(JSON.stringify(command) + "\n", error => { if (error) finish(error); });- });- }-- /** EOF first; report uncertain shutdown instead of claiming the native process is gone. */- close(): Promise<void> {- if (this.closePromise) return this.closePromise;- this.closed = true;- for (const item of [...this.pending.values()]) item.finish(new Error("Qt frame peer closed."));- return this.closePromise = new Promise<void>((resolve, reject) => {- if (this.child.pid === undefined || this.child.exitCode !== null || this.child.signalCode !== null) { resolve(); return; }- const timer = setTimeout(() => { this.child.kill(); reject(new Error("Qt frame shutdown is unconfirmed.")); }, 5000);- this.child.once("exit", () => { clearTimeout(timer); resolve(); });- this.child.stdin.end();- });- }-}diff --git a/src/window-overlay.ts b/src/window-overlay.tsdeleted file mode 100644index b711f9e..0000000--- a/src/window-overlay.ts+++ /dev/null@@ -1,121 +0,0 @@-import type { WindowTarget } from "./session.ts";-import type { ScreenRect } from "./highlight-geometry.ts";--/** An internal human-only primitive in physical screen coordinates. */-export interface HighlightMark { kind: "overview" | "detail" | "action"; region?: ScreenRect; point?: { x: number; y: number } }-/** A successfully authorized window lifetime, never a model-supplied drawing target. */-export interface WindowFrame {- target: WindowTarget;- identity: string;- bounds: ScreenRect;- mark?: HighlightMark;-}-/** Display marks require an authorized primary-display read; they grant no input. */-export interface DisplayFrame { display: "primary"; bounds: ScreenRect; mark?: HighlightMark }-export type HighlightFrame = WindowFrame | DisplayFrame | { clear: true };-/** Private helper messages. */-export type OverlayCommand = { kind: "flash"; generation: number; duration_ms: 2000 } & (WindowFrame | DisplayFrame)- | { kind: "hide"; generation: number };-/** A reply is not proof of click-through or capture exclusion; those need native checks. */-export interface OverlayReply { status: string; generation: number; reason?: string }-/** A single owned rendering peer, with no capture or input operations. */-export interface OverlayPeer { send(command: OverlayCommand): Promise<OverlayReply>; close(): Promise<void> }-/** Drawing failure never changes or replays the underlying capture/input. */-export interface HighlightResult { status: "shown" | "skipped" | "unavailable"; duration_ms?: number; kind?: HighlightMark["kind"]; reason?: string; region?: ScreenRect; point?: { x: number; y: number } }-export type WindowObserver = (frame: HighlightFrame, signal?: AbortSignal) => Promise<HighlightResult>;--/** Fail-soft notification. An unavailable target clears the previous mark instead of inventing a frame. */-export async function notifyHighlight(observer: WindowObserver | undefined, frame: WindowFrame | DisplayFrame | undefined,- signal?: AbortSignal): Promise<HighlightResult | undefined> {- if (!observer) return undefined;- if (signal?.aborted) return { status: "skipped" };- if (!frame || !valid(frame)) {- try { await observer({ clear: true }, signal); } catch { /* Optional visual aid. */ }- return { status: "unavailable", reason: "native_geometry_unavailable" };- }- try {- const result = await observer(frame, signal);- const mark = frame.mark;- return { ...result, kind: mark?.kind ?? "overview", ...(mark?.region ? { region: { ...mark.region } } : {}),- ...(mark?.point ? { point: { ...mark.point } } : {}) };- } catch { return { status: "unavailable", reason: "renderer_unavailable" }; }-}--/** Keep the retained browser research adapter's whole-window observer contract. */-export async function notifyWindowObserver(observer: WindowObserver | undefined, target: WindowTarget | undefined,- identity: string | undefined, bounds: unknown, signal?: AbortSignal, mark?: HighlightMark): Promise<HighlightResult | undefined> {- return notifyHighlight(observer, target && identity && bounds ? { target: { ...target }, identity, bounds: bounds as ScreenRect, ...(mark ? { mark } : {}) } : undefined, signal);-}-function valid(frame: WindowFrame | DisplayFrame): boolean {- const { bounds, mark } = frame;- const rectangle = (r: ScreenRect) => r && [r.x, r.y, r.width, r.height].every(Number.isSafeInteger) &&- Math.abs(r.x) <= 2_000_000 && Math.abs(r.y) <= 2_000_000 && r.width > 0 && r.width <= 32768 && r.height > 0 && r.height <= 32768;- if (!rectangle(bounds)) return false;- if ("display" in frame) { if (frame.display !== "primary" || "target" in frame || bounds.x !== 0 || bounds.y !== 0) return false; }- else {- const identity = /^PiComputerUse\.[a-f0-9]{32}:(\d+):([1-9]\d*)$/i.exec(frame.identity);- if (!identity || Number(identity[1]) !== frame.target.pid || !Number.isSafeInteger(Number(identity[2])) || Number(identity[2]) <= 0 ||- ![frame.target.pid, frame.target.window_id].every(v => Number.isSafeInteger(v) && v > 0)) return false;- }- if (!mark) return true;- if (!["overview", "detail", "action"].includes(mark.kind)) return false;- if (mark.kind === "overview") return !mark.region && !mark.point;- if (!!mark.region === !!mark.point || (mark.kind === "detail" && !mark.region)) return false;- const r = mark.region; const p = mark.point;- return r ? rectangle(r) && r.x >= bounds.x && r.y >= bounds.y &&- r.x + r.width <= bounds.x + bounds.width && r.y + r.height <= bounds.y + bounds.height :- !!p && [p.x, p.y].every(Number.isSafeInteger) && p.x >= bounds.x && p.y >= bounds.y &&- p.x < bounds.x + bounds.width && p.y < bounds.y + bounds.height;-}--/** Lazily own one Qt frame process. Generations prevent late startup, cancellation or old reads repainting a newer frame. */-export class WindowOverlay {- private starting?: Promise<OverlayPeer>;- private generation = 0;- private release?: () => void;- private closed = false;-- constructor(private readonly createPeer: () => Promise<OverlayPeer>) {}-- /** Flash a validated source/primitive. Native failure is fail-soft and never retried during this session. */- async flash(frame: HighlightFrame, signal?: AbortSignal): Promise<HighlightResult> {- if ("clear" in frame) { await this.clear(); return { status: "skipped" }; }- if (this.closed || signal?.aborted) return { status: "skipped" };- if (!valid(frame)) { await this.clear(); return { status: "unavailable", reason: "native_geometry_unavailable" }; }- const generation = ++this.generation;- this.release?.(); this.release = undefined;- this.starting ??= Promise.resolve().then(() => this.createPeer());- try {- const peer = await this.starting;- if (generation !== this.generation || signal?.aborted || this.closed) return { status: "skipped" };- const abort = () => { if (generation === this.generation) void this.clear(); };- signal?.addEventListener("abort", abort, { once: true });- const expiry = setTimeout(() => { if (generation === this.generation) { this.release?.(); this.release = undefined; } }, 2000);- expiry.unref();- this.release = () => { clearTimeout(expiry); signal?.removeEventListener("abort", abort); };- const reply = await peer.send({ kind: "flash", generation, ...frame, duration_ms: 2000 });- if (generation !== this.generation || signal?.aborted || this.closed) return { status: "skipped" };- return reply.status === "shown" && reply.generation === generation ? { status: "shown", duration_ms: 2000 } : { status: "unavailable", reason: reply.reason ?? "renderer_refused" };- } catch { return { status: "unavailable" }; }- }-- /** Invalidate pending flashes and hide a visible frame without starting an unused helper. */- async clear(): Promise<void> {- const generation = ++this.generation;- this.release?.(); this.release = undefined;- if (!this.starting) return;- try {- const peer = await this.starting;- if (generation === this.generation) await peer.send({ kind: "hide", generation });- } catch { /* A missing rendering aid must not block permission revocation. */ }- }-- /** Stop this helper only. EOF owns native cleanup; never terminate source applications. */- async close(): Promise<void> {- this.closed = true; ++this.generation;- this.release?.(); this.release = undefined;- // Startup failure has no usable peer; uncertain shutdown of an existing peer must propagate.- const peer = await this.starting?.catch(() => undefined);- await peer?.close();- }-}diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.tsindex 470eaf3..5d5c6f4 100644--- a/test/agent-cursor.test.ts+++ b/test/agent-cursor.test.ts@@ -7,7 +7,7 @@ import { McpDriver } from "../src/driver.ts";const dirs: string[] = [];afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });-async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay") {+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals") {const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));dirs.push(dir);const log = path.join(dir, "calls.jsonl");@@ -28,6 +28,17 @@ test("the driver makes the agent cursor jump and checks the setting before any o} finally { await driver.close(); }});+test("a native visual driver receives presentation settings without forcing a one-millisecond glide", async () => {+ const { driver, calls } = await start("visuals");+ try {+ await driver.call("list_windows", {});+ expect((await calls()).map(call => call.name)).toEqual(["set_agent_visuals", "list_windows"]);+ expect((await calls())[0].arguments).toMatchObject({ preset: "full", previews: true });+ await driver.visuals({ target: { pid: 42, window_id: 101 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`, bounds: { x: 0, y: 0, width: 800, height: 600 } });+ for (let n = 0; n < 20 && (await calls()).length < 3; n++) await new Promise(resolve => setTimeout(resolve, 5));+ expect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview", generation: 1 } });+ } finally { await driver.close(); }+});test("a refused cursor setting stops the driver before any input", async () => {const { driver, calls } = await start("refuses");await expect(driver.call("list_windows", {})).rejects.toThrow("refused the agent cursor glide setting");diff --git a/test/attach-browser.test.ts b/test/attach-browser.test.tsindex 9e2f3d3..ee36758 100644--- a/test/attach-browser.test.ts+++ b/test/attach-browser.test.ts@@ -15,7 +15,7 @@ function fixture() {const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({ url: "data:", title: "Attach target", snapshot: "- textbox \"Field\"", tabs: [], events: [] })) };const launch = vi.fn<WebLauncher>(async () => ({ browser: browser as unknown as WebBrowser,opened: { pid: window.pid, version: "155", url: "data:", title: "Attach target", via: "cdp" } }));- const session = new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }, () => mode, undefined, undefined, undefined, launch);+ const session = new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }, () => mode, undefined, undefined, launch);const ref = async () => ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;return { session, launch, browser, ref, setMode(value: ComputerMode) { mode = value; } };}diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex c185871..5ec3e3c 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -158,7 +158,7 @@ function browserFixture() {const browser = { close: vi.fn(async () => {}), page } as unknown as WebBrowser;const launch = vi.fn<WebLauncher>(async () => ({ browser,opened: { pid: target.pid, version: "155", url: "https://example.com", title: "Page", via: "cdp" } }));- const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, launch);+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, launch);const resources = new ComputerResources(async () => session, async () => session.close());const window = async () => {const ref = ((await resources.search(at("computer:machine/local/windows"), cwd, "Chrome")).details.state!.windows as { window_ref: string }[])[0].window_ref;@@ -275,7 +275,7 @@ function fixture(outcome: "closed" | "open" | "refused") {}),};const closeWindow = vi.fn(async () => { if (outcome === "closed") open = false; return outcome; });- const session = new ComputerSession(driver, () => mode, undefined, undefined, undefined, undefined, { closeWindow, removeFiles: async () => {} } as unknown as MachineHost);+ const session = new ComputerSession(driver, () => mode, undefined, undefined, undefined, { closeWindow, removeFiles: async () => {} } as unknown as MachineHost);const resources = new ComputerResources(async () => session, async () => session.close());const window = async () => ((await resources.search(at("computer:machine/local/windows"), cwd, "Editor")).details.state!.windows as { window_ref: string }[])[0].window_ref;const remove = (source: string) => deleteComputer(resources, source, { path: source }, cwd);@@ -328,7 +328,7 @@ test("computer_record video_stop says where it fetched the video in the summary};const video = Buffer.from("mp4 bytes");const host = { files: async () => ({ windows: "C:\\pi", wsl: "/mnt/c/pi" }), rename: vi.fn(async () => {}), get: vi.fn(async () => video), removeFiles: async () => {} };- const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, undefined, host as unknown as MachineHost);+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, host as unknown as MachineHost);const resources = new ComputerResources(async () => session, async () => session.close());const workspace = await mkdtemp(join(tmpdir(), "pi-record-"));const run = (command: Record<string, unknown>) => computerAction(resources, "computer:machine/local", String(command.action), () => at("computer:machine/local"),diff --git a/test/cua-vision-settings.test.ts b/test/cua-vision-settings.test.tsindex 4df15eb..7db43c8 100644--- a/test/cua-vision-settings.test.ts+++ b/test/cua-vision-settings.test.ts@@ -1,10 +1,7 @@-import { readFile } from "node:fs/promises";import { expect, test } from "vitest";-import { CuaImageSettings } from "../src/cua-image-settings.ts";-import { startVisionServer } from "../src/vision-server.ts";-import { VisionTrace } from "../src/vision-trace.ts";+import { CuaImageSettings, nativeImageCap } from "../src/cua-image-settings.ts";-test("the private viewer reports the selected native overview filter without offering a filter switch", () => {+test("image settings follow the selected native filter without offering a filter switch", () => {let kernel: "lanczos3" | "bilinear" = "bilinear";const images = new CuaImageSettings(() => kernel);const frozen = images.settings();@@ -16,122 +13,56 @@ test("the private viewer reports the selected native overview filter without offexpect(images.settings().kernel).toBe("lanczos3");expect(frozen.kernel).toBe("bilinear");});-// Native controls must not imply that a second image engine still runs.-test("guide writes require a complete same-origin tuple and keep prior settings frozen", async () => {++test("guide writes require a complete tuple and keep previous captures frozen", () => {const images = new CuaImageSettings(); const original = images.settings();- const server = await startVisionServer(new VisionTrace("guides"), images);- const url = new URL("guides", server.url); const headers = { Origin: url.origin, "Content-Type": "application/json" };const guides = { grid: true, action: true };- try {- for (const input of [{ grid: true }, { ...guides, mode: "yolo" }, { ...guides, action: 1 }])- expect((await fetch(url, { method: "POST", headers, body: JSON.stringify(input) })).status).toBe(400);+ for (const input of [{ grid: true }, { ...guides, mode: "yolo" }, { ...guides, action: 1 }]) {+ expect(() => images.setGuides(input)).toThrow();expect(images.settings()).toEqual(original);- expect((await fetch(url, { method: "POST", headers: { ...headers, Origin: "https://foreign.example" }, body: JSON.stringify(guides) })).status).toBe(403);- expect(await (await fetch(url, { method: "POST", headers, body: JSON.stringify(guides) })).json()).toMatchObject({ guides, version: 1 });- expect(original.guides).toBeUndefined();- expect((await (await fetch(new URL("state", server.url))).json()).imageSettings.guides).toEqual(guides);- } finally { await server.close(); }+ }+ expect(images.setGuides(guides)).toMatchObject({ guides, version: 1 });+ expect(original.guides).toBeUndefined();+ const copy = images.settings(); copy.guides!.grid = false;+ expect(images.settings().guides).toEqual(guides);});-test("the private hints toggle is same-origin, session-local and does not rewrite old captures", async () => {- const images = new CuaImageSettings(); const trace = new VisionTrace("hints");- const frozen = images.settings(); const bytes = await readFile(new URL("./fixtures/cua-zoom.jpg", import.meta.url));- trace.startAction("first", "computer_observe", {});- trace.finishAction("first", [{ type: "image", data: bytes.toString("base64"), mimeType: "image/jpeg" }],- { status: "observed", agent_image: { configuration: frozen } }, false);- const before = trace.snapshot().captures; const server = await startVisionServer(trace, images);- const url = new URL("hints", server.url); const headers = { Origin: url.origin, "Content-Type": "application/json" };- try {- for (const body of [{ enabled: "false" }, { enabled: false, mode: "yolo" }]) {- expect((await fetch(url, { method: "POST", headers, body: JSON.stringify(body) })).status).toBe(400);- expect(images.settings()).toEqual(frozen);- }- expect((await fetch(url, { method: "POST", headers: { ...headers, Origin: "https://foreign.example" }, body: '{"enabled":false}' })).status).toBe(403);- expect(await (await fetch(url, { method: "POST", headers, body: '{"enabled":false}' })).json()).toMatchObject({ hints: false, version: 1 });- images.update({ kernel: "lanczos3", maxLongEdge: 800, scale: 1 });- expect(images.settings()).toMatchObject({ hints: false, version: 2 });- expect(trace.snapshot().captures).toEqual(before); expect(frozen.hints).toBeUndefined();- expect(trace.asset(trace.snapshot().captures[0].hash)).toEqual(bytes);- } finally { await server.close(); }++test("hints remain session-local and do not rewrite frozen capture settings", () => {+ const images = new CuaImageSettings(); const another = new CuaImageSettings(); const frozen = images.settings();+ for (const value of ["false", 0, undefined]) {+ expect(() => images.setHints(value as unknown as boolean)).toThrow();+ expect(images.settings()).toEqual(frozen);+ }+ expect(images.setHints(false)).toMatchObject({ hints: false, version: 1 });+ images.update({ kernel: "lanczos3", maxLongEdge: 800, scale: 1 });+ expect(images.settings()).toMatchObject({ hints: false, version: 2 });+ expect(frozen.hints).toBeUndefined(); expect(another.settings()).toEqual(frozen);});-test("native settings freeze captures, reject unsupported kernels and reset without originals", () => {++test("native settings freeze captures, reject unsupported tuples and reset", () => {const images = new CuaImageSettings(); const first = images.settings();- expect(first).toMatchObject({ kernel: "lanczos3", maxLongEdge: 1280, scale: 1, version: 0 });+ expect(first).toMatchObject({ kernel: "lanczos3", maxLongEdge: 1280, scale: 1, version: 0, route: "cua_native" });expect(images.update({ kernel: "lanczos3", maxLongEdge: 800, scale: 0.625 })).toMatchObject({ version: 1, scale: 0.625 });expect(first.scale).toBe(1);- for (const input of [{ kernel: "mks2021", maxLongEdge: 500, scale: 1 }, { kernel: "lanczos3", maxLongEdge: 500, scale: 0 }, { kernel: "lanczos3", maxLongEdge: 500, scale: 1, mode: "yolo" }])+ for (const input of [{ kernel: "mks2021", maxLongEdge: 500, scale: 1 }, { kernel: "lanczos3", maxLongEdge: 500, scale: 0 },+ { kernel: "lanczos3", maxLongEdge: 500, scale: 1, mode: "yolo" }, { kernel: "lanczos3", maxLongEdge: 800.5, scale: 1 },+ { kernel: "lanczos3", maxLongEdge: 500 }, null]) {expect(() => images.update(input)).toThrow();- expect(images.settings().version).toBe(1);+ expect(images.settings()).toMatchObject({ version: 1, maxLongEdge: 800, scale: 0.625 });+ }+ expect(nativeImageCap(images.settings())).toBe(500);images.reset(); expect(images.settings()).toEqual(first);});-test("the native debug server reports the active route and refuses the retired trial and kernel controls", async () => {- const images = new CuaImageSettings(); const server = await startVisionServer(new VisionTrace("native"), images);- const settings = new URL("settings", server.url);- const post = (value: unknown) => fetch(settings, { method: "POST", headers: { Origin: settings.origin, "Content-Type": "application/json" }, body: JSON.stringify(value) });- try {- expect((await fetch(new URL("trial", server.url))).status).toBe(410);- expect((await post({ kernel: "mks2021", maxLongEdge: 500, scale: 1 })).status).toBe(400);- expect(await (await post({ kernel: "lanczos3", maxLongEdge: 800, scale: 0.625 })).json()).toMatchObject({ version: 1, route: "cua_native" });- expect((await (await fetch(new URL("state", server.url))).json()).imageSettings).toMatchObject({ route: "cua_native", kernel: "lanczos3", maxLongEdge: 800, scale: 0.625 });- } finally { await server.close(); }+test("concurrent settings stores preserve each capture's configuration", async () => {+ const first = new CuaImageSettings(); const second = new CuaImageSettings(); const frozen = first.settings();+ const replies = await Promise.all([+ Promise.resolve().then(() => first.update({ kernel: "lanczos3", maxLongEdge: 800, scale: 0.5 })),+ Promise.resolve().then(() => first.update({ kernel: "lanczos3", maxLongEdge: 600, scale: 0.75 })),+ ]);+ expect(replies.map(value => value.version)).toEqual([1, 2]);+ expect(first.settings()).toEqual(replies[1]); expect(second.settings()).toEqual(frozen);+ first.update({ kernel: "lanczos3", maxLongEdge: 4096, scale: 1 });+ expect(frozen).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });+ first.reset(); expect(first.settings()).toEqual(frozen); expect(second.settings()).toEqual(frozen);});--test("concurrent native settings stay session-local and preserve a frame's captured configuration", async () => {- const first = new CuaImageSettings(); const second = new CuaImageSettings();- const trace = new VisionTrace("first"); const frozen = first.settings();- const bytes = await readFile(new URL("./fixtures/cua-zoom.jpg", import.meta.url));- trace.startAction("capture", "browser_observe", {});- const a = await startVisionServer(trace, first);- const b = await startVisionServer(new VisionTrace("second"), second);- const post = (base: string, maxLongEdge: number, scale: number) => {- const url = new URL("settings", base);- return fetch(url, { method: "POST", headers: { Origin: url.origin, "Content-Type": "application/json" },- body: JSON.stringify({ kernel: "lanczos3", maxLongEdge, scale }) }).then(response => response.json());- };- try {- const replies = await Promise.all([post(a.url, 800, 0.5), post(a.url, 600, 0.75)]);- expect(replies.map(value => value.version).sort()).toEqual([1, 2]);- const last = replies.find(value => value.version === 2);- expect(first.settings()).toEqual(last);- expect(second.settings()).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });- trace.finishAction("capture", [{ type: "image", data: bytes.toString("base64"), mimeType: "image/jpeg" }],- { status: "observed", agent_image: { configuration: frozen } }, false);- const state = await (await fetch(new URL("state", a.url))).json();- expect(state.imageSettings).toEqual(last);- expect(state.captures[0].configuration.effective.configuration).toEqual(frozen);- const image = await fetch(new URL(`images/${state.captures[0].hash}`, a.url));- expect(Buffer.from(await image.arrayBuffer()).equals(bytes)).toBe(true);- const foreign = new URL(a.url); foreign.host = new URL(b.url).host;- expect((await fetch(new URL("settings", foreign))).status).toBe(404);- const before = structuredClone(state.captures);- await post(a.url, 4096, 1);- expect((await (await fetch(new URL("state", a.url))).json()).captures).toEqual(before);- first.reset();- expect(first.settings()).toEqual(frozen);- expect(second.settings()).toEqual(frozen);- } finally { await Promise.all([a.close(), b.close()]); }-});--test("unsafe or invalid settings writes cannot change any part of the active tuple", async () => {- const images = new CuaImageSettings(); const server = await startVisionServer(new VisionTrace("writes"), images);- const url = new URL("settings", server.url); const initial = images.settings();- const valid = JSON.stringify({ kernel: "lanczos3", maxLongEdge: 800, scale: 0.5 });- const json = { Origin: url.origin, "Content-Type": "application/json" };- try {- for (const [headers, body, status] of [- [{ "Content-Type": "application/json" }, valid, 403],- [{ ...json, Origin: "https://foreign.example" }, valid, 403],- [{ ...json, "Content-Type": "text/plain" }, valid, 403],- [json, valid.replace("0.5", "0"), 400],- [json, valid.replace("800", "800.5"), 400],- [json, valid.slice(0, -1) + ',"mode":"yolo"}', 400],- [json, "{", 400],- [json, " ".repeat(1025), 413],- ] as const) {- expect((await fetch(url, { method: "POST", headers, body })).status).toBe(status);- expect(images.settings()).toEqual(initial);- }- expect((await fetch(new URL("state", server.url), { method: "POST", headers: json, body: valid })).status).toBe(405);- expect(images.settings()).toEqual(initial);- } finally { await server.close(); }-});\ No newline at end of filediff --git a/test/cua-vision-trace.test.ts b/test/cua-vision-trace.test.tsdeleted file mode 100644index f3a71ed..0000000--- a/test/cua-vision-trace.test.ts+++ /dev/null@@ -1,51 +0,0 @@-import { createHash } from "node:crypto";-import { readFile } from "node:fs/promises";-import { expect, test } from "vitest";-import { VisionTrace } from "../src/vision-trace.ts";--test("annotated captures retain exact raw bytes privately without consuming another boundary card", async () => {- const raw = await readFile(new URL("./fixtures/cua-zoom.jpg", import.meta.url));- const marked = Buffer.from("separate marked test bytes");- const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");- const trace = new VisionTrace("hints");- trace.retainOriginal({ type: "image", mimeType: "image/jpeg", data: raw.toString("base64") });- trace.startAction("hints", "computer_observe", {});- trace.finishAction("hints", [{ type: "image", mimeType: "image/png", data: marked.toString("base64") }],- { status: "observed", agent_image: { annotation: { raw: { hash: hash(raw) }, labels: [{ hint: "1:a" }] }, output: { hash: hash(marked) } } }, false);- trace.observePayload({ input: [{ type: "input_image", image_url: `data:image/png;base64,${marked.toString("base64")}` }] }, { provider: "test", api: "test", model: "test" });- const state = trace.snapshot();- expect(state.frames).toHaveLength(1);- expect(state.frames[0]).toMatchObject({ original: "available", originalHash: hash(raw) });- expect(trace.asset(hash(raw))?.equals(raw)).toBe(true);- expect(JSON.stringify(state)).not.toContain(raw.toString("base64"));- trace.reset("another"); expect(trace.asset(hash(raw))).toBeUndefined();-});--test("the common source envelope reaches capture and provider-preview metadata without changing image bytes", async () => {- const bytes = await readFile(new URL("./fixtures/cua-zoom.jpg", import.meta.url));- const data = bytes.toString("base64"); const hash = createHash("sha256").update(bytes).digest("hex");- const source = { kind: "display", domain: "windows-host", identity: { display_id: "primary" }, capabilities: { input: "none" },- image: { scope: "primary_display", coordinate_space: "display_image_pixels", dimensions: { width: 499, height: 333 }, original_dimensions: { width: 3840, height: 2560 } } };- const trace = new VisionTrace("source-proof");- trace.startAction("capture", "computer_observe", { display: "primary" });- trace.finishAction("capture", [{ type: "image", data, mimeType: "image/jpeg" }], { status: "observed", source, image: { type: "image", data, mimeType: "image/jpeg" } }, false);- trace.observePayload({ input: [{ type: "input_image", image_url: `data:image/jpeg;base64,${data}` }] }, { provider: "test", api: "test", model: "test" });- expect(trace.snapshot().captures[0].source).toMatchObject({ observation: source });- expect(trace.snapshot().frames[0].source).toMatchObject({ observation: source });- expect(trace.snapshot().actions[0].adapterStructure).not.toHaveProperty("image");- expect(JSON.stringify(trace.snapshot().actions[0].adapterStructure)).not.toContain(data);- expect(trace.asset(hash)?.equals(bytes)).toBe(true);-});--test("native JPEG preview keeps exact bytes, matching dimensions and its zoom coordinate domain", async () => {- const bytes = await readFile(new URL("./fixtures/cua-zoom.jpg", import.meta.url));- const data = bytes.toString("base64"); const hash = createHash("sha256").update(bytes).digest("hex");- const trace = new VisionTrace("cua-zoom");- const details = { status: "observed", tab: 1, agent_image: { route: "cua_fresh_zoom", coordinate_space: "zoom_image_pixels",- configuration: { kernel: "lanczos3", maxWidth: 500 }, output: { hash, width: 152, height: 120 }, native: { width: 152, height: 120 } } };- trace.startAction("zoom", "browser_observe", { zoom: { x: 24, y: 165, width: 41, height: 32 } });- trace.finishAction("zoom", [{ type: "image", data, mimeType: "image/jpeg" }], details, false);- trace.observePayload({ input: [{ type: "input_image", image_url: `data:image/jpeg;base64,${data}` }] }, { provider: "test", api: "test", model: "test" });- expect(trace.snapshot().frames[0]).toMatchObject({ width: 152, height: 120, original: "unavailable", source: { route: "cua_fresh_zoom", coordinate_space: "zoom_image_pixels" } });- expect(trace.asset(hash)?.equals(bytes)).toBe(true); expect(trace.assetMime(hash)).toBe("image/jpeg");-});diff --git a/test/display-mode.test.ts b/test/display-mode.test.tsindex 73ce2cd..35ef654 100644--- a/test/display-mode.test.ts+++ b/test/display-mode.test.ts@@ -22,7 +22,7 @@ function fixture(remote = false) {throw new Error(`Unexpected native call ${name}`);}), identity: vi.fn(async () => "unused"), close: vi.fn(async () => {}),};- const session = new ComputerSession(driver, () => mode, undefined, undefined, undefined, undefined, new MachineHost({ run: vi.fn(), spawn: vi.fn() }, modes));+ const session = new ComputerSession(driver, () => mode, undefined, undefined, undefined, new MachineHost({ run: vi.fn(), spawn: vi.fn() }, modes));const ref = async () => ((await session.displays()).details.state!.displays as { display_ref: string }[])[0].display_ref;return { session, modes, driver, ref, current: () => current, setMode(value: ComputerMode) { mode = value; } };}diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjsindex 4f33f1b..cc35c99 100644--- a/test/fixtures/agent-cursor-driver.mjs+++ b/test/fixtures/agent-cursor-driver.mjs@@ -8,7 +8,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotconst [log, mode] = process.argv.slice(2);let glide = 0;const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });-server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [] }));+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode === "visuals" ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {appendFileSync(log, JSON.stringify({ name, arguments: args }) + "\n");if (name === "set_agent_cursor_motion") {diff --git a/test/fixtures/image-context-extension.ts b/test/fixtures/image-context-extension.tsindex f8dbd8e..00ba8b2 100644--- a/test/fixtures/image-context-extension.ts+++ b/test/fixtures/image-context-extension.ts@@ -1,9 +1,6 @@-import { writeFile } from "node:fs/promises";-import path from "node:path";import sharp from "sharp";import { Type } from "@earendil-works/pi-ai";import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";-import { registerVisionDebug } from "../../src/vision-debug.ts";export default function imageContextFixture(pi: ExtensionAPI): void {createCodemodeExtension({ mode: "on" })(pi);@@ -12,7 +9,6 @@ export default function imageContextFixture(pi: ExtensionAPI): void {pi.registerProvider("image-loopback", { baseUrl, api: "openai-completions", apiKey: "synthetic",models: [{ id: "fixture", name: "Image context fixture", reasoning: false, input: ["text", "image"], contextWindow: 128000,maxTokens: 4096, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } }] });- const debug = registerVisionDebug(pi);pi.registerTool({ name: "computer_sample", label: "Synthetic frame", description: "Generate one synthetic fixture image, not a desktop capture.",parameters: Type.Object({ n: Type.Integer({ minimum: 0, maximum: 20 }) }),outputSchema: Type.Object({ image: Type.Unknown() }),@@ -22,6 +18,4 @@ export default function imageContextFixture(pi: ExtensionAPI): void {const image = { type: "image" as const, data: bytes.toString("base64"), mimeType: n % 2 ? "image/jpeg" : "image/png" };return { content: [{ type: "text", text: `Frame ${n} ${n % 2 ? "detail" : "overview"}` }, image], details: { n }, structuredContent: { image } };} });- pi.on("session_start", async (_event, ctx) => { await writeFile(path.join(ctx.cwd, "vision-url.txt"), debug.url() ?? ""); });- pi.on("agent_end", async (_event, ctx) => { await writeFile(path.join(ctx.cwd, "vision-trace.json"), JSON.stringify(debug.trace.snapshot())); });}diff --git a/test/fixtures/vision-extension.ts b/test/fixtures/vision-extension.tsdeleted file mode 100644index 551cea2..0000000--- a/test/fixtures/vision-extension.ts+++ /dev/null@@ -1,60 +0,0 @@-import { writeFile } from "node:fs/promises";-import path from "node:path";-import { Type } from "@earendil-works/pi-ai";-import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";-import { registerVisionDebug } from "../../src/vision-debug.ts";--const PIXEL = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aN2kAAAAASUVORK5CYII=";--/** Real-Pi Vision debug fixture: synthetic image tools, no native UI or browser. */-export default function visionFixture(pi: ExtensionAPI): void {- const baseUrl = process.env.VISION_TEST_PROVIDER;- if (baseUrl) {- if (!/^http:\/\/127\.0\.0\.1:\d+\/v1$/.test(baseUrl)) throw new Error("Fixture provider must be loopback.");- pi.registerProvider("vision-loopback", {- baseUrl, api: "openai-completions", apiKey: "synthetic-test-key",- models: [{ id: "fixture", name: "Vision loopback fixture", reasoning: false, input: ["text", "image"],- contextWindow: 128000, maxTokens: 4096, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } }],- });- }- const debug = registerVisionDebug(pi);- createCodemodeExtension({ mode: "on" })(pi);- const image = { type: "image" as const, data: PIXEL, mimeType: "image/png" };- const imageTool = (name: string) => pi.registerTool({- name, label: name, description: "Return one synthetic PNG.",- outputSchema: Type.Object({ text: Type.String(), image: Type.Unknown() }, { additionalProperties: true }),- parameters: Type.Object({}, { additionalProperties: false }),- async execute() {- return { content: [{ type: "text" as const, text: "Synthetic image." }, image], details: { status: "observed" },- structuredContent: { status: "observed", text: "Synthetic image.", image } };- },- });- imageTool("computer_sample_open");- imageTool("computer_sample_observe");- // Stands in for Pi Agent IDE's read, which serves computer: paths alongside ordinary files.- pi.registerTool({- name: "read", label: "read", description: "Read a path; computer: paths return one synthetic PNG.",- parameters: Type.Object({ path: Type.String() }, { additionalProperties: false }),- async execute(_id, params) {- return params.path.startsWith("computer:")- ? { content: [{ type: "text" as const, text: "Synthetic image." }, image], details: { status: "observed" } }- : { content: [{ type: "text" as const, text: "plain file" }], details: { status: "read" } };- },- });- pi.registerTool({- name: "computer_sample_fail", label: "computer_sample_fail", description: "Always refuse.",- parameters: Type.Object({}, { additionalProperties: false }),- async execute() { return { content: [{ type: "text" as const, text: "Refused." }], details: { status: "refused" }, isError: true }; },- });- pi.registerTool({- name: "computer_sample_close", label: "computer_sample_close", description: "Return text only.",- parameters: Type.Object({}, { additionalProperties: false }),- async execute() { return { content: [{ type: "text" as const, text: "Closed." }], details: { status: "closed" } }; },- });- pi.on("session_start", async (_event, ctx) => {- await writeFile(path.join(ctx.cwd, "vision-url.txt"), debug.url() ?? "");- });- pi.on("agent_end", async (_event, ctx) => {- await writeFile(path.join(ctx.cwd, "vision-trace.json"), JSON.stringify(debug.trace.snapshot()));- });-}diff --git a/test/fixtures/window-overlay-peer.mjs b/test/fixtures/window-overlay-peer.mjsdeleted file mode 100644index b98fc20..0000000--- a/test/fixtures/window-overlay-peer.mjs+++ /dev/null@@ -1,10 +0,0 @@-import { appendFileSync, writeFileSync } from "node:fs";-import { createInterface } from "node:readline";-const [record, mode] = process.argv.slice(2);-writeFileSync(record, JSON.stringify({ kind: "started", pid: process.pid }) + "\n");-console.log(JSON.stringify({ status: "ready", generation: 0, protocol_version: mode === "legacy" ? undefined : 2, capture_affinity: mode === "bad-ready" ? 0 : 17, pid: process.pid, overlay_hwnd: 42 }));-createInterface({ input: process.stdin }).on("line", line => {- const request = JSON.parse(line); appendFileSync(record, line + "\n");- if (mode === "bad-reply") console.log("{invalid");- else console.log(JSON.stringify({ generation: request.generation, status: request.kind === "flash" ? "shown" : "hidden" }));-}).on("close", () => { appendFileSync(record, JSON.stringify({ kind: "eof" }) + "\n"); });diff --git a/test/image-context.integration.test.ts b/test/image-context.integration.test.tsindex 5dc59f0..7d85a27 100644--- a/test/image-context.integration.test.ts+++ b/test/image-context.integration.test.ts@@ -1,10 +1,10 @@-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";import { createServer } from "node:http";import path from "node:path";import { getToolExecution, PiIntegrationTest, testArtifactsDir } from "pi-coding-agent-test";import { expect, test } from "vitest";-test("real Pi retains all twelve images on the actual HTTP wire while the debug viewer keeps three cards", async () => {+test("real Pi retains all twelve images on the actual HTTP wire", async () => {await mkdir(".tmp/image-context", { recursive: true }); const cwd = await mkdtemp(path.resolve(".tmp/image-context/pi-"));const bodies: any[] = [];const provider = createServer(async (request, response) => {@@ -34,11 +34,6 @@ test("real Pi retains all twelve images on the actual HTTP wire while the debugexpect(JSON.stringify(bodies.at(-1))).toContain(wireImages[1][0]);for (let n = 0; n < 12; n++) expect(JSON.stringify(bodies.at(-1))).toContain(`frame=${n}`);for (let n = 0; n < 12; n += 2) expect(getToolExecution(run, `frames-${n}`).isError).toBe(false);- const state = JSON.parse(await readFile(path.join(cwd, "vision-trace.json"), "utf8"));- expect(state).not.toHaveProperty("imageContext");- expect(state.requests.at(-1).imageCount).toBe(12);- expect(state.frames).toHaveLength(3);- expect(state.captures).toHaveLength(3);await writeFile(path.join(run.artifacts.directory, "provider-bodies.json"), JSON.stringify(bodies));} finally {await new Promise<void>(resolve => { provider.close(() => resolve()); provider.closeAllConnections(); });diff --git a/test/image-settings-command.test.ts b/test/image-settings-command.test.tsnew file mode 100644index 0000000..f74c7e5--- /dev/null+++ b/test/image-settings-command.test.ts@@ -0,0 +1,43 @@+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";+import { expect, test, vi } from "vitest";+import { CuaImageSettings } from "../src/cua-image-settings.ts";+import { registerImageSettingsCommand } from "../src/image-settings-command.ts";++type Command = { handler(args: string, ctx: ExtensionCommandContext): Promise<void> };+function fixture() {+ const commands = new Map<string, Command>();+ const images = new CuaImageSettings(() => "bilinear");+ const notify = vi.fn();+ registerImageSettingsCommand({ registerCommand: (name: string, command: Command) => commands.set(name, command) } as unknown as ExtensionAPI, images);+ const ctx = { hasUI: true, ui: { notify } } as unknown as ExtensionCommandContext;+ return { images, notify, run: (input: string) => commands.get("computer-images")!.handler(input, ctx), commands };+}++test("the human image command changes future native captures without a filter switch", async () => {+ const f = fixture(); const old = f.images.settings();+ await f.run("cap 800"); await f.run("scale 0.5"); await f.run("hints off"); await f.run("guides all");+ expect(f.images.settings()).toMatchObject({ kernel: "bilinear", maxLongEdge: 800, scale: 0.5, hints: false,+ guides: { grid: true, action: true }, version: 4 });+ expect(old).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });+ expect(old.hints).toBeUndefined(); expect(old.guides).toBeUndefined();+ await f.run("reset"); expect(f.images.settings()).toEqual(old);+});++test("bad image command arguments preserve the full settings tuple", async () => {+ const f = fixture(); await f.run("cap 800"); const saved = f.images.settings();+ for (const input of ["cap 63", "cap 800.5", "cap Infinity", "scale 0", "scale NaN", "hints false", "hints off extra",+ "guides on", "kernel lanczos3", "mode yolo", "reset extra"]) {+ await f.run(input); expect(f.images.settings()).toEqual(saved);+ expect(f.notify.mock.calls.at(-1)?.[1]).toBe("warning");+ }+});++test("showing image settings does not change captures and instances stay independent", async () => {+ const first = fixture(); const second = fixture();+ await first.run("cap 600"); await first.run("guides grid");+ const saved = first.images.settings(); await first.run(""); await first.run("show");+ expect(first.images.settings()).toEqual(saved);+ expect(second.images.settings()).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });+ expect(first.commands.size).toBe(1);+ expect(first.notify.mock.calls.at(-1)?.[0]).toContain("600");+});diff --git a/test/live/remote.ts b/test/live/remote.tsindex 9371d7f..8d0d765 100644--- a/test/live/remote.ts+++ b/test/live/remote.ts@@ -25,7 +25,7 @@ export async function connectLive(cwd = process.cwd()) {let local: ComputerSession | undefined;const resources = new ComputerResources(async () => {if (name !== "local") throw new Error("Live checks use only the remote machine.");- return local ??= new ComputerSession(new McpDriver(config), () => "yolo", () => images.settings(), undefined, undefined,+ return local ??= new ComputerSession(new McpDriver(config), () => "yolo", () => images.settings(), undefined,(options, signal) => WebBrowser.launch({ ...options, interop: config.interop }, signal), new MachineHost(localRunner(config.interop)));}, async () => local ? local.close() : { content: [], details: { status: "closed", mode: "yolo" } },sandboxFactory({ settings: () => images.settings() }), machineFactory({ getMode: () => "yolo", settings: () => images.settings() }));diff --git a/test/native-visuals.test.ts b/test/native-visuals.test.tsnew file mode 100644index 0000000..bc11fda--- /dev/null+++ b/test/native-visuals.test.ts@@ -0,0 +1,80 @@+import { expect, test, vi } from "vitest";+import { NativeVisuals, VisualSettings, notifyHighlight } from "../src/native-visuals.ts";++const frame = { target: { window_id: 101, pid: 42 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`,+ bounds: { x: -100, y: 20, width: 800, height: 600 } };+const tick = async () => { for (let n = 0; n < 8; n++) await Promise.resolve(); };++test("presentation returns immediately while native admission is pending, with only the newest pending image retained", async () => {+ const sent: { name: string; args: Record<string, unknown> }[] = [];+ let release!: () => void;+ const visuals = new NativeVisuals(async (name, args) => { sent.push({ name, args }); await new Promise<void>(r => { release = r; }); return true; });+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" });+ await tick();+ for (let n = 1; n <= 100; n++) await visuals.flash({ ...frame, mark: { kind: "action", point: { x: n, y: 50 } } });+ expect(sent).toHaveLength(1);+ release(); await tick();+ expect(sent).toHaveLength(2);+ expect(sent[1].args).toMatchObject({ generation: 101, point: { x: 100, y: 50 } });+ release(); await tick(); visuals.close();+});++test("previews contain the supplied output image and validated physical source, never a new capture", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ const image = { type: "image" as const, mimeType: "image/png", data: "aGVsbG8=" };+ await notifyHighlight(visuals.flash, { ...frame, image }); await tick();+ expect(send).toHaveBeenCalledExactlyOnceWith("present_agent_activity", {+ generation: 1, kind: "overview", source: { window_id: 101, pid: 42, identity: frame.identity, bounds: frame.bounds },+ image: { data: image.data, mime_type: image.mimeType },+ });+ visuals.close();+});++test("invalid or out-of-source geometry is cleared without sending image content", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ const result = await notifyHighlight(visuals.flash, { ...frame, identity: frame.identity.replace(":42:", ":99:") });+ await tick();+ expect(result?.status).toBe("unavailable");+ expect(send.mock.calls).toEqual([["present_agent_activity", { kind: "clear", generation: 1 }]]);+ visuals.close();+});++test("off discards queued images and old cancellation cannot clear a newer event", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const settings = new VisualSettings(); const visuals = new NativeVisuals(send, settings);+ const abort = new AbortController();+ await visuals.flash(frame, abort.signal); await tick();+ await visuals.flash(frame); await tick();+ abort.abort(); await tick();+ expect(send).toHaveBeenCalledTimes(2);+ settings.update({ preset: "off" }); await tick();+ expect(send.mock.calls.at(-1)).toEqual(["set_agent_visuals", expect.objectContaining({ preset: "off" })]);+ const count = send.mock.calls.length;+ expect(await visuals.flash(frame)).toMatchObject({ status: "skipped" });+ await tick(); expect(send).toHaveBeenCalledTimes(count); visuals.close();+});++test("selected-display regions retain their physical monitor origin", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ await visuals.flash({ display: "DISPLAY2", bounds: { x: -1920, y: 0, width: 1920, height: 1080 }, mark: {+ kind: "detail", region: { x: -1900, y: 40, width: 200, height: 100 },+ } }); await tick();+ expect(send.mock.calls[0][1]).toMatchObject({ source: { display: "DISPLAY2", bounds: { x: -1920, y: 0 } }, region: { x: -1900, y: 40 } });+ visuals.close();+});++test("restoring a branch resets unrelated preferences and rejects prototype keys", () => {+ const settings = new VisualSettings();+ settings.update({ keyboard: false, quality: "high" });+ expect(settings.restore({ preset: "subtle" })).toMatchObject({ keyboard: true, quality: "balanced", previews: false });+ const before = settings.current();+ expect(() => settings.update(JSON.parse('{"constructor":false}'))).toThrow();+ expect(settings.current()).toEqual(before);+});+test("settings validate before mutation and unavailable native presentation does not throw into operations", async () => {+ const settings = new VisualSettings();+ expect(() => settings.update({ opacity: NaN })).toThrow();+ expect(settings.current().opacity).toBe(0.8);+ const visuals = new NativeVisuals(async () => { throw new Error("not available"); }, settings);+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" }); await tick();+ expect(await visuals.flash(frame)).toMatchObject({ status: "unavailable" }); visuals.close();+});diff --git a/test/selected-displays.test.ts b/test/selected-displays.test.tsindex c975711..0b48b75 100644--- a/test/selected-displays.test.ts+++ b/test/selected-displays.test.ts@@ -130,12 +130,14 @@ test("cancellation during native selected capture withholds pixels and does notexpect(result.details).not.toHaveProperty("agent_image");});-test("selected-monitor reads clear a primary-only human overlay rather than marking another monitor", async () => {+test("selected-monitor observations carry the exact physical monitor source and returned image", async () => {const f = await fixture(); const observer = vi.fn(async () => ({ status: "shown" as const }));const session = new ComputerSession(f.driver, undefined, undefined, observer);const listed = await session.displays();const ref = (listed.details.state!.displays as { display_ref: string }[])[1].display_ref;const view = await session.observeDisplay(ref);- expect(view.details.status).toBe("observed"); expect(observer).toHaveBeenCalledWith({ clear: true }, undefined);- expect(view.details.highlight).toMatchObject({ status: "unavailable", reason: "selected_display_overlay_unsupported" });+ expect(view.details.status).toBe("observed");+ const display = left;+ expect(observer).toHaveBeenCalledWith({ display: display.display_id, bounds: display.bounds, image: view.content.find(block => block.type === "image") }, undefined);+ expect(view.details.highlight).toMatchObject({ status: "shown" });});diff --git a/test/sequence-trace.test.ts b/test/sequence-trace.test.tsdeleted file mode 100644index f07c4aa..0000000--- a/test/sequence-trace.test.ts+++ /dev/null@@ -1,28 +0,0 @@-import sharp from "sharp";-import { expect, test } from "vitest";-import { VisionTrace } from "../src/vision-trace.ts";--test("the viewer correlates every selected temporal frame but does not treat selection as forwarding", async () => {- const trace = new VisionTrace("sequence");- const images = await Promise.all(["red", "blue", "green", "yellow"].map(async background => ({ type: "image", mimeType: "image/png",- data: (await sharp({ create: { width: 10, height: 10, channels: 3, background } }).png().toBuffer()).toString("base64") })));- trace.startAction("capture", "computer_sequence", {});- trace.finishAction("capture", [], { status: "observed", sequence: { sequence_id: "seq-one", frames: [{ index: 0 }] } }, false);- expect(trace.snapshot().captures).toHaveLength(0);- trace.startAction("select", "computer_sequence", { sequence_id: "seq-one", selection: "all" });- trace.finishAction("select", images, { status: "observed", selected_frames: images.map((_, index) => ({ index, timestamp_ms: index * 250, source: { kind: "window" } })) }, false);- expect(trace.snapshot().requests).toHaveLength(0);- trace.startAction("repeat", "computer_sequence", { sequence_id: "seq-one", selection: "first" });- trace.finishAction("repeat", [images[0]], { status: "observed", selected_frames: [{ index: 0, timestamp_ms: 0 }] }, false);- trace.observePayload({ images }, { provider: "test", api: "test", model: "test" });- const latest = trace.snapshot().requests.at(-1)!;- expect(latest.sequenceImages).toHaveLength(4);- expect(latest.sequenceImages.every(item => item.matches.length === 1)).toBe(true);- expect(latest.sequenceImages.map(item => item.matches[0])).toEqual(images.map((_, index) => ({ sequence_id: "seq-one", index, timestamp_ms: index * 250 })));- expect(latest.sequenceImages.every(item => !!trace.asset(item.hash))).toBe(true);- expect(trace.snapshot().frames).toHaveLength(3);- trace.startAction("same-pixels", "computer_sequence", { sequence_id: "seq-one", selection: "last" });- trace.finishAction("same-pixels", [images[0]], { status: "observed", selected_frames: [{ index: 4, timestamp_ms: 1000 }] }, false);- trace.observePayload({ images: [images[0]] }, { provider: "test", api: "test", model: "test" });- expect(trace.snapshot().requests.at(-1)!.sequenceImages[0].matches.map(frame => frame.index)).toEqual([0, 4]);-});diff --git a/test/ui-resources.test.ts b/test/ui-resources.test.tsindex b64fdd7..0efc704 100644--- a/test/ui-resources.test.ts+++ b/test/ui-resources.test.ts@@ -29,11 +29,10 @@ async function fixture() {return { content: [], structuredContent: { path: "ax", effect: "confirmed", verified: true } };}),};- const retained: Content[] = [];- const session = new ComputerSession(driver, () => mode, undefined, undefined, image => retained.push(image));+ const session = new ComputerSession(driver, () => mode);const windows = (await session.windows()).details.state!.windows as { window_ref: string }[];const ref = windows[0].window_ref;- return { session, driver, raw, retained, ref, other: windows[1].window_ref, setMode: (value: "safe" | "yolo") => { mode = value; } };+ return { session, driver, raw, ref, other: windows[1].window_ref, setMode: (value: "safe" | "yolo") => { mode = value; } };}function ui(result: Awaited<ReturnType<ComputerSession["observe"]>>) { return result.details.ui!; }@@ -178,10 +177,16 @@ test("local search and scope keep the snapshot without another native walk or inexpect(ui(absent)).toMatchObject({ complete: false, matches: 0 });});-test("hints annotate only one delivered image and retain the exact raw PNG privately", async () => {+test("hints annotate one image while later snapshot inspection can use the private raw PNG", async () => {const f = await fixture(); const view = await f.session.observe(f.ref); const images = view.content.filter(b => b.type === "image");expect(images).toHaveLength(1); expect(images[0].data).not.toBe(f.raw.toString("base64"));- expect(f.retained).toContainEqual({ type: "image", mimeType: "image/png", data: f.raw.toString("base64") });+ const calls = nativeCalls(f.driver);+ const inspected = await f.session.observe(f.ref, undefined, undefined, undefined, {+ operation: "scope", observation_id: view.details.observation_id!, hint: ui(view).elements[3].hint,+ });+ expect(inspected.details.status).toBe("observed");+ expect(inspected.content).toContainEqual({ type: "image", mimeType: "image/png", data: f.raw.toString("base64") });+ expect(nativeCalls(f.driver)).toEqual(calls);expect(JSON.stringify(view.details)).not.toContain(f.raw.toString("base64"));expect(view.details.agent_image).toMatchObject({ route: "cua_native_ui_hints", annotation: { enabled: true, labels: expect.any(Array) } });expect(ui(view).elements[3].hint_display).toBe("no_bounds");diff --git a/test/vision-debug.test.ts b/test/vision-debug.test.tsdeleted file mode 100644index f400485..0000000--- a/test/vision-debug.test.ts+++ /dev/null@@ -1,125 +0,0 @@-import { createHash } from "node:crypto";-import { request } from "node:http";-import { runInNewContext } from "node:vm";-import { visionScript } from "../src/vision-panel.ts";-import { createPanelDom } from "./fixtures/vision-panel-dom.ts";-import { expect, test } from "vitest";-import { VisionTrace } from "../src/vision-trace.ts";-import { startVisionServer } from "../src/vision-server.ts";--const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jC1kAAAAASUVORK5CYII=";-const image = { type: "image", data: png, mimeType: "image/png" };-const hash = createHash("sha256").update(Buffer.from(png, "base64")).digest("hex");-const payload = (count = 1) => ({ input: [{ role: "user", content: Array.from({ length: count }, () => ({ type: "input_image", image_url: `data:image/png;base64,${png}`, detail: "high" })) }], instructions: "not a UI tree" });--test("panel polling keeps expanded action rows and their text scroll position", () => {- const { document, nodes } = createPanelDom();- const trace = new VisionTrace("panel");- trace.startAction("first", "browser_observe", {});- trace.finishAction("first", [], { status: "observed" }, false);- const render = runInNewContext(visionScript.replace(/poll\(\);\s*$/, "") + "\nrender;", { document });- render(trace.snapshot());- const first = nodes.get("actions")!.children[0];- first.open = true;- first.children[1].scrollTop = 80;- trace.startAction("second", "browser_observe", {});- render(trace.snapshot());- expect(nodes.get("actions")!.children[1]).toBe(first);- expect(first.open).toBe(true);- expect(first.children[1].scrollTop).toBe(80);-});--test("links actual boundary image bytes to a nested action without claiming final delivery", () => {- const trace = new VisionTrace("session-one");- trace.startAction("program/1", "browser_observe", {}, "program");- trace.finishAction("program/1", [image, { type: "text", text: "Visible UI" }], { status: "observed", tab: 1, elements: [{ hint: "a", name: "Button" }] }, false);- expect(trace.snapshot().frames).toHaveLength(0);- const boundary = payload();- trace.observePayload(boundary, { provider: "openai-codex", api: "openai-codex-responses", model: "test" });- trace.response(200);- const state = trace.snapshot();- expect(state.frames[0]).toMatchObject({ hash, actionId: "program/1", stage: "prepared", finalDelivery: "unconfirmed", width: 1, height: 1, detail: "high", original: "unavailable" });- expect(state.actions[0]).toMatchObject({ id: "program/1", parentId: "program", status: "observed", isError: false });- expect(state.requests[0]).toMatchObject({ responseStatus: 200, finalPayload: "unconfirmed" });- expect(boundary).toEqual(payload());- expect(trace.asset(hash)?.equals(Buffer.from(png, "base64"))).toBe(true);-});--test("retains only three displayed boundary images and never hides images in the effective context", () => {- const trace = new VisionTrace("bounded");- const body = payload(5);- trace.observePayload(body, { provider: "openai", api: "openai-responses", model: "test" });- expect(trace.snapshot().frames).toHaveLength(3);- expect(trace.snapshot().requests[0]).toMatchObject({ imageCount: 5 });- expect(body.input[0].content).toHaveLength(5);- expect(trace.snapshot().limitations).toContain("The three-card viewer does not limit the model context.");-});--test("keeps captured-only images out of boundary history and reports action errors", () => {- const trace = new VisionTrace("captured-only");- trace.startAction("program/1", "browser_observe", {}, "program");- trace.finishAction("program/1", [image], { status: "observed" }, false);- trace.startAction("program/2", "browser_act", { click: "disabled" }, "program");- trace.finishAction("program/2", [{ type: "text", text: "Observed element is disabled." }], undefined, true);- trace.observePayload({ input: [{ type: "function_call_output", output: "Only text was forwarded" }] }, { provider: "test", api: "test", model: "test" });- expect(trace.snapshot().frames).toHaveLength(0);- expect(trace.snapshot().captures[0]).toMatchObject({ hash, stage: "captured", finalDelivery: "unconfirmed" });- expect(trace.snapshot().actions[1]).toMatchObject({ isError: true, text: "Observed element is disabled." });- expect(trace.snapshot().requests[0].texts[0].text).toBe("Only text was forwarded");-});--test("does not fetch remote images and isolates an old session on reset", () => {- const trace = new VisionTrace("first");- trace.observePayload(payload(), { provider: "test", api: "test", model: "test" });- trace.reset("second");- expect(trace.snapshot()).toMatchObject({ sessionId: "second", frames: [], captures: [], actions: [], requests: [] });- expect(trace.asset(hash)).toBeUndefined();- trace.observePayload({ messages: [{ content: [{ type: "image_url", image_url: { url: "https://private.example/image" } }] }] }, { provider: "test", api: "test", model: "test" });- expect(trace.snapshot().frames).toHaveLength(0);- expect(trace.snapshot().requests[0]).toMatchObject({ imageCount: 1, unavailableImages: 1 });-});--test("does not invent a unique source for identical captures and retains actual tool text", () => {- const trace = new VisionTrace("ambiguous");- for (const id of ["first", "captured-only"]) {- trace.startAction(id, "browser_observe", {});- trace.finishAction(id, [image], { status: "observed" }, false);- }- const block = { type: "input_image", image_url: `data:image/png;base64,${png}` };- trace.observePayload({ messages: [{ role: "tool", content: "Actual forwarded structure" }], input: [{ content: [block, block] }] }, { provider: "test", api: "test", model: "test" });- const state = trace.snapshot();- expect(state.requests[0].imageCount).toBe(2);- expect(state.requests[0].texts[0].text).toBe("Actual forwarded structure");- expect(state.frames[0].actionId).toBeUndefined();- expect(state.frames[0].matchingActionIds).toEqual(["first", "captured-only"]);-});--test("loopback viewer supports reconnect but rejects foreign routes, hosts and origins and closes", async () => {- const trace = new VisionTrace("server-session");- trace.observePayload(payload(), { provider: "test", api: "test", model: "test" });- const server = await startVisionServer(trace);- try {- const page = await fetch(server.url);- expect(page.status).toBe(200);- expect(page.headers.get("content-security-policy")).toContain("default-src 'none'");- expect(await page.text()).toContain("Final delivery is unconfirmed");- const state = new URL("state", server.url);- expect((await (await fetch(state)).json()).sessionId).toBe("server-session");- expect((await (await fetch(state)).json()).frames[0].hash).toBe(hash);- expect((await fetch(new URL(`images/${hash}`, server.url))).headers.get("content-type")).toBe("image/png");- expect((await fetch(new URL("/state", server.url))).status).toBe(404);- expect((await fetch(state, { headers: { Origin: "https://foreign.example" } })).status).toBe(403);- const foreignHostStatus = await new Promise<number | undefined>((resolve, reject) => {- const probe = request(state, { headers: { Host: "foreign.example" } }, response => {- response.resume(); resolve(response.statusCode);- });- probe.on("error", reject); probe.end();- });- expect(foreignHostStatus).toBe(403);- expect((await fetch(state, { method: "POST" })).status).toBe(405);- trace.reset("new-session");- expect((await (await fetch(state)).json()).frames).toHaveLength(0);- } finally { await server.close(); }- await expect(fetch(server.url)).rejects.toThrow();- await server.close();-});diff --git a/test/vision-extension.integration.test.ts b/test/vision-extension.integration.test.tsdeleted file mode 100644index 804ad5b..0000000--- a/test/vision-extension.integration.test.ts+++ /dev/null@@ -1,80 +0,0 @@-import { createHash } from "node:crypto";-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";-import { createServer } from "node:http";-import path from "node:path";-import { getToolExecution, PiIntegrationTest, testArtifactsDir } from "pi-coding-agent-test";-import { expect, test } from "vitest";--test("real Pi mirrors native provider image bytes and traces nested and captured-only calls", async () => {- await mkdir(".tmp", { recursive: true });- const cwd = await mkdtemp(path.resolve(".tmp", "vision-contract-"));- const programs = [- { id: "program", name: "codemode", args: { code: `-const view = await tools.computer_sample_open({});-image(view.image); text(view.text);-` } },- { id: "captured-only", name: "codemode", args: { code: `-await tools.computer_sample_observe({});-text("Capture deliberately not forwarded.");-` } },- { id: "failed", name: "codemode", args: { code: "await tools.computer_sample_fail({});" } },- { id: "ide-form", name: "codemode", args: { code: `-await tools.read({ path: "computer:window/fixture" });-await tools.read({ path: "note.txt" });-text("IDE-form computer read and an ordinary read.");-` } },- { id: "close", name: "computer_sample_close", args: {} },- ];- const bodies: Record<string, unknown>[] = [];- const provider = createServer(async (request, response) => {- let input = "";- for await (const chunk of request) input += chunk;- bodies.push(JSON.parse(input));- const call = programs[bodies.length - 1];- const delta = call- ? { tool_calls: [{ index: 0, id: call.id, type: "function", function: { name: call.name, arguments: JSON.stringify(call.args) } }] }- : { content: "The debug trace is complete." };- response.writeHead(200, { "Content-Type": "text/event-stream" });- response.write("data: " + JSON.stringify({ id: "fixture-" + bodies.length, object: "chat.completion.chunk", model: "fixture",- choices: [{ index: 0, delta, finish_reason: call ? "tool_calls" : "stop" }] }) + "\n\n");- response.end("data: [DONE]\n\n");- });- await new Promise<void>(resolve => provider.listen(0, "127.0.0.1", resolve));- const address = provider.address();- if (!address || typeof address === "string") throw new Error("Missing fixture port.");- try {- const run = await new PiIntegrationTest({- testName: "vision-debug-native-provider", artifactsDir: testArtifactsDir(import.meta.filename), cwd,- extensions: [path.resolve("test/fixtures/vision-extension.ts")],- tools: ["codemode", "read", "computer_sample_open", "computer_sample_observe", "computer_sample_fail", "computer_sample_close"],- providerMode: "user", model: "vision-loopback/fixture", rawMode: true,- environment: { VISION_TEST_PROVIDER: `http://127.0.0.1:${address.port}/v1` },- isolateUserResources: true,- }).run("Exercise only the synthetic image tools through Codemode, including captured-only output and a refusal.");- const state = JSON.parse(await readFile(path.join(cwd, "vision-trace.json"), "utf8"));- await mkdir(run.artifacts.directory, { recursive: true });- await writeFile(path.join(run.artifacts.directory, "vision-trace.json"), JSON.stringify(state, null, 2));- await writeFile(path.join(run.artifacts.directory, "provider-bodies.json"), JSON.stringify(bodies, null, 2));- expect(state.actions).toEqual(expect.arrayContaining([- expect.objectContaining({ id: "program/1", parentId: "program", tool: "computer_sample_open", status: "observed" }),- expect.objectContaining({ id: "captured-only/1", parentId: "captured-only", tool: "computer_sample_observe" }),- expect.objectContaining({ id: "failed/1", isError: true }),- expect.objectContaining({ id: "ide-form/1", parentId: "ide-form", tool: "read", status: "observed" }),- ]));- expect(state.actions.map((action: { id: string }) => action.id)).not.toContain("ide-form/2");- expect(state.captures).toHaveLength(3);- expect(state.frames.length).toBeGreaterThan(0);- expect(state.frames.every((frame: { finalDelivery: string }) => frame.finalDelivery === "unconfirmed")).toBe(true);- expect(state.requests).toHaveLength(bodies.length);- const encoded = JSON.stringify(bodies.at(-1)).match(/data:image\/png;base64,([A-Za-z0-9+/=]+)/)?.[1];- expect(encoded).toBeDefined();- expect(state.frames.at(-1).hash).toBe(createHash("sha256").update(Buffer.from(encoded!, "base64")).digest("hex"));- expect(state.requests.at(-1).responseStatus).toBe(200);- expect(getToolExecution(run, "failed").isError).toBe(true);- const url = await readFile(path.join(cwd, "vision-url.txt"), "utf8");- await expect(fetch(url)).rejects.toThrow();- } finally {- await new Promise<void>(resolve => { provider.close(() => resolve()); provider.closeAllConnections(); });- await rm(cwd, { recursive: true, force: true });- }-});diff --git a/test/vision-panel.test.ts b/test/vision-panel.test.tsdeleted file mode 100644index d480e78..0000000--- a/test/vision-panel.test.ts+++ /dev/null@@ -1,54 +0,0 @@-import { createContext, runInContext } from "node:vm";-import { expect, test } from "vitest";-import { visionScript } from "../src/vision-panel.ts";-import { createPanelDom } from "./fixtures/vision-panel-dom.ts";--function fixture() {- const { document } = createPanelDom();- const context = createContext({ document, fetch: () => new Promise(() => {}), setTimeout: () => {} });- runInContext(visionScript, context);- return {- frames: document.getElementById("frames"),- render(frames: object[], revision = 1) {- context.state = { sessionId: "test", revision, frames, requests: [], actions: [], captures: [], limitations: [] };- runInContext("render(state)", context);- },- };-}-const frame = (id: string, hash = "sent") => ({ id, hash, originalHash: "raw", original: "available", stage: "prepared", time: "now", bytes: 10, width: 20, height: 20, requestId: "request-1" });--test("temporal dropdown shows every boundary frame in payload order and selects exact bytes", () => {- const { document } = createPanelDom();- const context = createContext({ document, fetch: () => new Promise(() => {}), setTimeout: () => {} });- runInContext(visionScript, context);- context.state = { sessionId: "test", revision: 1, frames: [], actions: [], captures: [], limitations: [], requests: [{ id: "r", texts: [],- sequenceImages: [0, 1, 2, 3].map(index => ({ hash: `frame-${index}`, available: true, path: `payload[${index}]`,- matches: [{ sequence_id: "seq", index, timestamp_ms: index * 100 }] })) }] };- runInContext("render(state)", context);- const section = document.getElementById("sequence-images"); const select = section.querySelector("select")!;- expect(select.children).toHaveLength(4); expect(select.children[3].textContent).toContain("frame 3 @ 300.0 ms");- select.value = "3"; select.handlers.change();- expect(section.querySelector("img")!.src).toBe("images/frame-3");- expect(section.querySelector("pre")!.textContent).toContain("payload[3]");-});--test("trace updates preserve the comparison node and selected raw image while metadata updates", () => {- const f = fixture(); f.render([frame("one")]);- const card = f.frames.children[0], select = card.querySelector("select")!, image = card.querySelector("img")!;- select.value = "raw"; select.handlers.change();- f.render([{ ...frame("two"), requestId: "request-2" }], 2);- expect(f.frames.children[0]).toBe(card); expect(card.querySelector("select")).toBe(select);- expect(image.src).toBe("images/raw"); expect(select.value).toBe("raw");- expect(select.attributes["aria-label"]).toBe("Image comparison two");- expect(card.querySelectorAll("p")[1].textContent).toContain("request-2");-});--test("identical image cards remain distinct and only replaced images lose their controls", () => {- const f = fixture(); f.render([frame("one"), frame("two"), frame("three")]);- const cards = [...f.frames.children]; expect(new Set(cards).size).toBe(3);- f.render([frame("four"), frame("five"), frame("six")], 2);- expect(f.frames.children).toEqual(cards);- f.render([frame("seven", "different")], 3);- expect(f.frames.children).toHaveLength(1); expect(cards.every(card => card.parent === null)).toBe(true);- expect(f.frames.children[0].querySelector("img")!.src).toBe("images/different");-});diff --git a/test/window-highlights.test.ts b/test/window-highlights.test.tsdeleted file mode 100644index df73e1f..0000000--- a/test/window-highlights.test.ts+++ /dev/null@@ -1,34 +0,0 @@-import { expect, test, vi } from "vitest";-import { WindowHighlights } from "../src/window-highlights.ts";-import type { OverlayPeer } from "../src/window-overlay.ts";--const frame = { target: { pid: 42, window_id: 123 }, identity: `PiComputerUse.${"a".repeat(32)}:42:7`,- bounds: { x: 10, y: 20, width: 900, height: 600 } };-function peer(): OverlayPeer {- return { send: vi.fn(async command => ({ status: command.kind === "flash" ? "shown" : "hidden", generation: command.generation })), close: vi.fn(async () => {}) };-}--test("a new helper can start after confirmed close, never during shutdown", async () => {- const first = peer(); const second = peer(); let stopped!: () => void;- first.close = vi.fn(() => new Promise<void>(resolve => { stopped = resolve; }));- const create = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(second);- const highlights = new WindowHighlights(create);- await highlights.flash(frame); const closing = highlights.close();- expect(await highlights.flash(frame)).toMatchObject({ status: "skipped" });- await Promise.resolve(); stopped(); await closing;- expect(await highlights.flash(frame)).toMatchObject({ status: "shown" });- expect(create).toHaveBeenCalledTimes(2); await highlights.close();-});--test("uncertain helper shutdown is reported and cannot launch another helper", async () => {- const native = peer(); native.close = vi.fn(async () => { throw new Error("unknown shutdown"); });- const create = vi.fn(async () => native); const highlights = new WindowHighlights(create);- await highlights.flash(frame); await expect(highlights.close()).rejects.toThrow("unknown shutdown");- expect(await highlights.flash(frame)).toMatchObject({ status: "skipped" });- expect(create).toHaveBeenCalledTimes(1);-});--test("permission clear and unused close do not start a helper", async () => {- const create = vi.fn(async () => peer()); const highlights = new WindowHighlights(create);- await highlights.clear(); await highlights.close(); expect(create).not.toHaveBeenCalled();-});diff --git a/test/window-observer.test.ts b/test/window-observer.test.tsindex 742613a..6540cbc 100644--- a/test/window-observer.test.ts+++ b/test/window-observer.test.ts@@ -1,7 +1,7 @@import { expect, test, vi } from "vitest";import sharp from "sharp";import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";-import type { WindowObserver } from "../src/window-overlay.ts";+import type { WindowObserver } from "../src/native-visuals.ts";async function fixture(observer: WindowObserver) {let mode: ComputerMode = "yolo"; let marker = 1;@@ -39,13 +39,15 @@ test("overview, requested window zoom and primary-display crop flash their own pconst observer = vi.fn<WindowObserver>(async () => ({ status: "shown" })); const f = await fixture(observer);f.setMode("safe"); const opened = await f.session.open("Calculator");expect(opened.details).toMatchObject({ status: "observed", highlight: { status: "shown" } });- expect(observer).toHaveBeenCalledWith({ target: { pid: 42, window_id: 101 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`, bounds: f.bounds }, undefined);+ expect(observer).toHaveBeenCalledWith({ target: { pid: 42, window_id: 101 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`, bounds: f.bounds, image: opened.content.find(block => block.type === "image") }, undefined);const overview = await f.session.observe(opened.details.window_ref!);const detail = await f.session.observe(opened.details.window_ref!, undefined, undefined, {image_id: overview.details.agent_image!.image_id as string, x: 10, y: 10, width: 100, height: 70,});expect(detail.details.status).toBe("observed"); expect(detail.content).toContainEqual(f.jpeg);- expect(detail.content[0]).toMatchObject({type:"text",text:expect.stringContaining("Human mark: detail shown; screen region 51,61 200x140.")});+ expect(detail.details.highlight).toMatchObject({ kind: "detail", status: "shown", region: { x: 51, y: 61, width: 200, height: 140 } });+ expect(observer.mock.calls[1][0]).toHaveProperty("image", overview.content.find(block => block.type === "image"));+ expect(observer.mock.calls[2][0]).toHaveProperty("image", f.jpeg);expect(observer).toHaveBeenCalledTimes(3);expect(observer.mock.calls[2][0]).toMatchObject({ bounds: f.bounds, mark: { kind: "detail", region: { x: 51, y: 61, width: 200, height: 140 } } });f.setMode("yolo"); const display = await f.session.observeDisplay("primary");@@ -63,9 +65,10 @@ test("action marks AX before input and the resolved pixel after dispatch; staleconst read = await f.session.observe(ref); observer.mockClear(); events.length = 0;const action = { window_ref: ref, observation_id: read.details.observation_id!, action: "click" as const, element_token: "button" };expect((await f.session.act(action)).details.status).toBe("delivered");- expect(events).toEqual(["mark", "input"]);+ expect(events).toEqual(["mark", "input", "mark"]);+ expect(observer.mock.calls[1][0]).toMatchObject({ outcome: "delivered" });expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "action", region: { x: 230, y: 240, width: 120, height: 60 } } });- expect((await f.session.act(action)).details.status).toBe("refused"); expect(observer).toHaveBeenCalledTimes(1);+ expect((await f.session.act(action)).details.status).toBe("refused"); expect(observer).toHaveBeenCalledTimes(2);const next = await f.session.observe(ref); observer.mockClear();await f.session.act({ ...action, observation_id: next.details.observation_id!, element_token: undefined, x: 100, y: 100 });expect(observer.mock.calls.at(-1)![0]).toMatchObject({ mark: { kind: "action", point: { x: 231, y: 241 } } });@@ -77,7 +80,10 @@ test("an ungrounded keyboard action does not invent a focused element, and rendeobserver.mockClear();const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "return" });expect(result.details.status).toBe("delivered"); expect(result.details.highlight).toMatchObject({ status: "unavailable" });- expect(result.content[0]).toMatchObject({type:"text",text:expect.stringContaining("Human mark: target unavailable (native_geometry_unavailable).")});+ expect(result.details.highlight).toMatchObject({ reason: "renderer_unavailable" });+ expect(observer.mock.calls.at(-1)?.[0]).toMatchObject({ mark: { kind: "action" }, outcome: "delivered" });+ expect(observer.mock.calls.at(-1)?.[0]).not.toHaveProperty("mark.point");+ expect(observer.mock.calls.at(-1)?.[0]).not.toHaveProperty("mark.region");expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "press_key")).toHaveLength(1);});test("permission loss or cancellation during an action mark prevents the single native dispatch", async () => {@@ -98,7 +104,9 @@ test("a refused pixel dispatch cannot turn a plausible native coordinate into an} : original(name, args, signal));const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "click", x: 100, y: 100 });expect(result.details.status).toBe("refused");- expect(observer.mock.calls.every(([frame]) => "clear" in frame)).toBe(true);+ expect(observer.mock.calls[0][0]).toEqual({ clear: true });+ expect(observer.mock.calls.at(-1)?.[0]).toMatchObject({ mark: { kind: "action" }, outcome: "refused" });+ expect(observer.mock.calls.at(-1)?.[0]).not.toHaveProperty("mark.point");expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "click")).toHaveLength(1);});test("Safe denial, pre-cancellation and recycled HWNDs never reach the frame observer", async () => {@@ -111,6 +119,23 @@ test("Safe denial, pre-cancellation and recycled HWNDs never reach the frame obsexpect(observer).not.toHaveBeenCalled();});+test("snapshot refinement previews the final annotated image and verification emits one completion, not every sample", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" }));+ const f = await fixture(observer); const ref = await outside(f.session);+ const read = await f.session.observe(ref); observer.mockClear();+ const refined = await f.session.observe(ref, undefined, undefined, undefined, {+ observation_id: read.details.observation_id!, operation: "highlight", hint: read.details.ui!.elements[1].hint,+ });+ expect(refined.details.status).toBe("observed");+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "inspect", region: { x: 230, y: 240, width: 120, height: 60 } }, image: refined.content.find(block => block.type === "image") });+ observer.mockClear();+ const result = await f.session.waitFor(ref, f.session.waitSpec(ref, { action: "verify", conditions: [{ target: { role: "Button" }, is: "visible" }] }));+ expect(result.details.verification?.samples).toBe(1);+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "verify" } });+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image");+});test("renderer failure does not corrupt or replay a valid desktop capture", async () => {const observer = vi.fn<WindowObserver>(async () => { throw new Error("render failure"); }); const f = await fixture(observer);const read = await f.session.observe(await outside(f.session), undefined, undefined, undefined, undefined, { hints: false });diff --git a/test/window-overlay-peer.test.ts b/test/window-overlay-peer.test.tsdeleted file mode 100644index a26e3a4..0000000--- a/test/window-overlay-peer.test.ts+++ /dev/null@@ -1,24 +0,0 @@-import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";-import path from "node:path";-import { expect, test } from "vitest";-import { startOverlayPeer } from "../src/window-overlay-peer.ts";--for (const mode of ["normal", "bad-ready", "bad-reply", "legacy"]) test(`rendering pipe ${mode}: exact replies or fail-soft EOF, never replay`, async () => {- await mkdir(".tmp/qt-overlay", { recursive: true });- const cwd = await mkdtemp(path.resolve(".tmp/qt-overlay/peer-")); const record = path.join(cwd, "calls.jsonl");- try {- const start = () => startOverlayPeer(process.execPath, [path.resolve("test/fixtures/window-overlay-peer.mjs"), record, mode]);- if (mode === "bad-ready" || mode === "legacy") await expect(start()).rejects.toThrow("protocol");- else {- const peer = await start();- const hide = peer.send({ kind: "hide", generation: 1 });- if (mode === "bad-reply") await expect(hide).rejects.toThrow("protocol");- else expect(await hide).toEqual({ status: "hidden", generation: 1 });- await peer.close(); await peer.close();- }- const calls = (await readFile(record, "utf8")).trim().split("\n").map(line => JSON.parse(line));- expect(calls.filter(c => c.kind === "hide")).toHaveLength(["bad-ready", "legacy"].includes(mode) ? 0 : 1);- expect(calls.at(-1)).toEqual({ kind: "eof" });- expect(() => process.kill(calls[0].pid, 0)).toThrow();- } finally { await rm(cwd, { recursive: true, force: true }); }-});diff --git a/test/window-overlay.test.ts b/test/window-overlay.test.tsdeleted file mode 100644index c00e3a3..0000000--- a/test/window-overlay.test.ts+++ /dev/null@@ -1,74 +0,0 @@-import { expect, test, vi } from "vitest";-import { WindowOverlay, type OverlayPeer } from "../src/window-overlay.ts";--const frame = { target: { pid: 42, window_id: 123 }, identity: `PiComputerUse.${"a".repeat(32)}:42:7`,- bounds: { x: -100, y: 20, width: 900, height: 600 } };-function peer(): OverlayPeer {- return { send: vi.fn(async command => ({ status: command.kind === "flash" ? "shown" : "hidden", generation: command.generation })),- close: vi.fn(async () => {}) };-}--test("registration and clearing an unused frame never start a native helper", async () => {- const start = vi.fn(async () => peer()); const overlay = new WindowOverlay(start);- await overlay.clear(); await overlay.close(); expect(start).not.toHaveBeenCalled();-});--test("a flash uses the exact lifetime, physical bounds and a two second expiry", async () => {- const native = peer(); const start = vi.fn(async () => native); const overlay = new WindowOverlay(start);- expect(await overlay.flash(frame)).toMatchObject({ status: "shown" });- expect(native.send).toHaveBeenCalledWith({ kind: "flash", generation: 1, ...frame, duration_ms: 2000 });- expect(await overlay.flash({ ...frame, target: { pid: 42, window_id: 456 } })).toMatchObject({ status: "shown" });- expect(start).toHaveBeenCalledTimes(1);- await overlay.close(); expect(native.close).toHaveBeenCalledTimes(1);-});--test("a cancelled read or mismatched lifetime cannot launch or flash", async () => {- const start = vi.fn(async () => peer()); const overlay = new WindowOverlay(start);- const aborted = new AbortController(); aborted.abort();- expect(await overlay.flash(frame, aborted.signal)).toMatchObject({ status: "skipped" });- expect(await overlay.flash({ ...frame, identity: frame.identity.replace(":42:", ":99:") })).toMatchObject({ status: "unavailable" });- expect(await overlay.flash({ ...frame, bounds: { ...frame.bounds, width: NaN } })).toMatchObject({ status: "unavailable" });- expect(start).not.toHaveBeenCalled();-});--test("a permission clear while the helper starts suppresses the late flash", async () => {- let ready!: (value: OverlayPeer) => void; const native = peer();- const overlay = new WindowOverlay(() => new Promise(resolve => { ready = resolve; }));- const showing = overlay.flash(frame); const clearing = overlay.clear(); await Promise.resolve(); ready(native);- expect(await showing).toMatchObject({ status: "skipped" }); await clearing;- expect(native.send).toHaveBeenCalledExactlyOnceWith({ kind: "hide", generation: 2 }); await overlay.close();-});--test("aborting an older read cannot hide a newer window flash", async () => {- const native = peer(); const overlay = new WindowOverlay(async () => native); const abort = new AbortController();- await overlay.flash(frame, abort.signal); await overlay.flash(frame); abort.abort();- expect(vi.mocked(native.send).mock.calls.map(([command]) => command.kind)).toEqual(["flash", "flash"]);- await overlay.close();-});--test("display, detail and action events replace each other; a scope-escaping primitive hides the previous mark", async () => {- const native = peer(); const overlay = new WindowOverlay(async () => native);- const display = { display: "primary" as const, bounds: { x: 0, y: 0, width: 1920, height: 1080 } };- await overlay.flash(display);- await overlay.flash({ ...display, mark: { kind: "detail", region: { x: 200, y: 100, width: 600, height: 400 } } });- await overlay.flash({ ...frame, mark: { kind: "action", point: { x: 100, y: 100 } } });- expect(vi.mocked(native.send).mock.calls.map(([c]) => c.kind)).toEqual(["flash", "flash", "flash"]);- expect(await overlay.flash({ ...frame, mark: { kind: "detail", region: { x: -101, y: 20, width: 50, height: 50 } } })).toMatchObject({ status: "unavailable" });- expect(native.send).toHaveBeenLastCalledWith({ kind: "hide", generation: 4 });- await overlay.close();-});--test("aborting a replaced display detail cannot hide a newer action marker", async () => {- const native = peer(); const overlay = new WindowOverlay(async () => native); const abort = new AbortController();- await overlay.flash({ display: "primary", bounds: { x: 0, y: 0, width: 1920, height: 1080 },- mark: { kind: "detail", region: { x: 20, y: 20, width: 100, height: 80 } } }, abort.signal);- await overlay.flash({ ...frame, mark: { kind: "action", point: { x: 100, y: 100 } } });- abort.abort(); expect(vi.mocked(native.send).mock.calls.map(([c]) => c.kind)).toEqual(["flash", "flash"]);- await overlay.close();-});-test("helper startup failure is reported without retrying a capture or claiming a frame", async () => {- const start = vi.fn(async () => { throw new Error("native startup failed"); }); const overlay = new WindowOverlay(start);- expect(await overlay.flash(frame)).toMatchObject({ status: "unavailable" });- expect(await overlay.flash(frame)).toMatchObject({ status: "unavailable" });- expect(start).toHaveBeenCalledTimes(1); await overlay.close();-});