The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(os): computer use for agents on Harness OS (Playwright and the desktop), off until turned on (#1959)
Bubu-Dudu-52hzbb83530d3057
diff --git a/cli/src/agentClient/computerUse.spec.ts b/cli/src/agentClient/computerUse.spec.tsnew file mode 100644index 000000000..22fbc6b28--- /dev/null+++ b/cli/src/agentClient/computerUse.spec.ts@@ -0,0 +1,130 @@+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { machineMcpArgs, readMachineMcp } from '../lib/machineMcp.js'+import { applyOpencode, CHROMIUM, DESKTOP_MCP, PLAYWRIGHT_MCP, setComputerUse, computerUseAvailable, computerUseCommand, computerUseServers, computerUseOn } from './computerUse.js'++describe('harness computer-use', () => {+ let dataDir: string+ const lines: { out: string[]; err: string[] } = { out: [], err: [] }+ const onOs = () => true+ const noOpencode = { path: '/nowhere/opencode.json', major: () => null, reload: () => {} }+ const run = (argv: string[], over: Parameters<typeof computerUseCommand>[1] = {}) => computerUseCommand(argv, {+ dataDir, home: '/home/me', exists: onOs, env: {}, inPane: false, opencode: noOpencode,+ out: (line) => lines.out.push(line), err: (line) => lines.err.push(line), ...over,+ })++ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'computer-use-')); lines.out = []; lines.err = [] })+ afterEach(() => rmSync(dataDir, { recursive: true, force: true }))++ it('is off until the person turns it on', () => {+ expect(run([])).toBe(0)+ expect(lines.out.join('\n')).toContain('Computer use is off')+ expect(computerUseOn(readMachineMcp(dataDir).servers)).toBe(false)+ })++ it('attaches Playwright on the OS\'s Chromium and the desktop to every agent, and takes them off again', () => {+ expect(run(['on'])).toBe(0)+ const { servers } = readMachineMcp(dataDir)+ expect(servers.playwright).toEqual({+ command: '/usr/bin/node',+ args: [PLAYWRIGHT_MCP, '--executable-path', CHROMIUM, '--isolated', '--output-dir', '/home/me/.cache/harness-os/playwright-mcp'],+ engines: [], projects: [],+ })+ expect(servers.desktop).toEqual({ command: '/usr/bin/python3', args: [DESKTOP_MCP], engines: [], projects: [] })+ expect(lines.out.at(-1)).toContain('Restart a running harness')+ // What a Claude Code launch then carries.+ expect(machineMcpArgs('claude', servers).join(' ')).toContain(PLAYWRIGHT_MCP)+ expect(run(['off'])).toBe(0)+ expect(readMachineMcp(dataDir).servers).toEqual({})+ })++ it('keeps the person\'s own servers when it turns on and off', () => {+ writeFileSync(join(dataDir, 'mcp-servers.json'), JSON.stringify({ servers: { lamp: { command: 'lamp-mcp' } } }), { mode: 0o600 })+ setComputerUse(true, dataDir, '/home/me', noOpencode)+ expect(Object.keys(readMachineMcp(dataDir).servers).sort()).toEqual(['desktop', 'lamp', 'playwright'])+ setComputerUse(false, dataDir, '/home/me', noOpencode)+ expect(Object.keys(readMachineMcp(dataDir).servers)).toEqual(['lamp'])+ })++ it('refuses an agent, and one in a Harness pane, and changes nothing', () => {+ expect(run(['on'], { env: { CLAUDECODE: '1' } })).toBe(5)+ expect(run(['on'], { inPane: true })).toBe(5)+ expect(lines.err.join('\n')).toContain('phone')+ expect(readMachineMcp(dataDir).servers).toEqual({})+ // Asking how they are is anyone's.+ expect(run([], { env: { CLAUDECODE: '1' } })).toBe(0)+ })++ it('says a computer without them has none', () => {+ expect(run(['on'], { exists: () => false })).toBe(1)+ expect(lines.err.join('\n')).toContain('Harness OS')+ expect(computerUseAvailable((path) => path !== CHROMIUM)).toBe(false)+ })++ it('does not write over a file it cannot read', () => {+ writeFileSync(join(dataDir, 'mcp-servers.json'), 'not json', { mode: 0o600 })+ expect(run(['on'])).toBe(1)+ expect(readFileSync(join(dataDir, 'mcp-servers.json'), 'utf8')).toBe('not json')+ })++ describe('OpenCode, through its own config', () => {+ const opencode = (major: number | null) => {+ const reload = vi.fn()+ return { path: join(dataDir, 'opencode', 'opencode.json'), major: () => major, reload }+ }+ const read = (path: string) => JSON.parse(readFileSync(path, 'utf8'))++ it('puts them under mcp.servers for OpenCode 2, as `opencode mcp add` does, reloads it, and takes them out', () => {+ const oc = opencode(2)+ mkdirSync(join(dataDir, 'opencode'))+ writeFileSync(oc.path, JSON.stringify({ model: 'opencode/free', mcp: { servers: { mine: { type: 'local', command: ['mine'] } } } }))+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true })+ const servers = read(oc.path).mcp.servers+ expect(servers.desktop).toEqual({ type: 'local', command: ['/usr/bin/python3', DESKTOP_MCP] })+ expect(servers.playwright.command.slice(0, 2)).toEqual(['/usr/bin/node', PLAYWRIGHT_MCP])+ expect(oc.reload).toHaveBeenCalledTimes(1)+ setComputerUse(false, dataDir, '/home/me', oc)+ expect(read(oc.path)).toEqual({ model: 'opencode/free', mcp: { servers: { mine: { type: 'local', command: ['mine'] } } } })+ expect(oc.reload).toHaveBeenCalledTimes(2)+ })++ it('puts them directly under mcp, enabled, for OpenCode 1, and leaves no empty mcp behind', () => {+ const oc = opencode(1)+ setComputerUse(true, dataDir, '/home/me', oc)+ expect(read(oc.path).mcp.desktop).toEqual({ type: 'local', command: ['/usr/bin/python3', DESKTOP_MCP], enabled: true })+ setComputerUse(false, dataDir, '/home/me', oc)+ expect(read(oc.path)).toEqual({})+ })++ it('keeps a server of the person\'s own under the same name, and says so', () => {+ const oc = opencode(2)+ mkdirSync(join(dataDir, 'opencode'))+ writeFileSync(oc.path, JSON.stringify({ mcp: { servers: { playwright: { type: 'local', command: ['npx', '@playwright/mcp'] } } } }))+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true, note: expect.stringContaining('its own playwright') })+ expect(read(oc.path).mcp.servers.playwright.command).toEqual(['npx', '@playwright/mcp'])+ setComputerUse(false, dataDir, '/home/me', oc)+ expect(read(oc.path).mcp.servers).toEqual({ playwright: { type: 'local', command: ['npx', '@playwright/mcp'] } })+ })++ it('leaves a config that is not plain JSON, and one OpenCode is not installed for, alone', () => {+ const oc = opencode(2)+ mkdirSync(join(dataDir, 'opencode'))+ writeFileSync(oc.path, '{ // mine\n}')+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true, note: expect.stringContaining('not plain JSON') })+ expect(readFileSync(oc.path, 'utf8')).toBe('{ // mine\n}')+ expect(oc.reload).not.toHaveBeenCalled()+ const absent = opencode(null)+ expect(applyOpencode(true, absent, '/home/me')).toBeNull()+ expect(existsSync(absent.path) && readFileSync(absent.path, 'utf8')).toContain('mine')+ })+ })++ it('answers usage for anything else', () => {+ expect(run(['maybe'])).toBe(2)+ expect(run(['on', 'now'])).toBe(2)+ expect(run(['--help'])).toBe(0)+ expect(Object.keys(computerUseServers())).toEqual(['playwright', 'desktop'])+ })+})diff --git a/cli/src/agentClient/computerUse.ts b/cli/src/agentClient/computerUse.tsnew file mode 100644index 000000000..07032cd22--- /dev/null+++ b/cli/src/agentClient/computerUse.ts@@ -0,0 +1,211 @@+/**+ * `harness computer-use`: Harness OS's tools for an agent to try what it builds as a person would, off until the person+ * turns them on. Two machine MCP servers (lib/machineMcp.ts, #1872), both bundled in the OS:+ *+ * - `playwright`: Playwright's MCP server (os/packaging/playwright-mcp) driving the OS's Chromium, for a website: it+ * opens pages, clicks, types and reads them and their console. A window of its own, on the screen.+ * - `desktop`: the desktop (os/desktop_mcp.py), for a desktop app: screenshots, and the mouse and keys on them.+ *+ * harness computer-use on or off+ * harness computer-use on attached to every agent started from now on (Claude Code, Codex, OpenCode)+ * harness computer-use off taken off again+ *+ * Turning them on or off writes this machine's mcp-servers.json, read at each Claude Code and Codex start, and+ * OpenCode's own config (below): a harness running already gets them when it restarts. The person's phone does the same (services/osDevice.ts `computer_use`). An agent may+ * not: a server runs inside every later agent.+ */+import { execFile, execFileSync } from 'node:child_process'+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'+import { homedir } from 'node:os'+import { dirname, join } from 'node:path'+import { env } from '../config/env.js'+import { MACHINE_MCP_MOST_SERVERS, readMachineMcp, writeMachineMcp, type MachineMcpServer, type MachineMcpServers } from '../lib/machineMcp.js'+import { outsideCaller } from './model.js'++export const PLAYWRIGHT_MCP = '/usr/lib/harness-os/playwright-mcp/node_modules/@playwright/mcp/cli.js'+export const DESKTOP_MCP = '/usr/lib/harness-os/desktop_mcp.py'+export const CHROMIUM = '/usr/bin/chromium'+const NODE = '/usr/bin/node'+const PYTHON = '/usr/bin/python3'+/** The machine MCP servers' names: the agents' tools are `mcp__playwright__…` and `mcp__desktop__…`. */+export const COMPUTER_USE = ['playwright', 'desktop'] as const++/** A refusal, as the agent verbs exit (commands.ts `EXIT.refused`): not a failure to retry. */+const REFUSED = 5++export const COMPUTER_USE_USAGE = `Usage: harness computer-use [on|off]++Tools for the agents Harness starts on this Harness OS computer to try what they build as a person would, off until+you turn them on:+ playwright a website, in a Chromium window of its own: open it, click, type, read it and its console+ desktop a desktop app: screenshots of the screen, and the mouse and keyboard on them+On, Claude Code and Codex agents Harness starts from now on get them, and OpenCode (its own config); a harness+running already gets them when you restart it. Each agent asks you before it uses them. Only you turn them on or off:+an agent is refused.`++/** The servers' entries; [home] places Playwright's screenshots and snapshots out of the project's folder. */+export function computerUseServers(home: string = homedir()): Record<(typeof COMPUTER_USE)[number], MachineMcpServer> {+ return {+ playwright: {+ command: NODE,+ args: [PLAYWRIGHT_MCP, '--executable-path', CHROMIUM, '--isolated', '--output-dir', join(home, '.cache', 'harness-os', 'playwright-mcp')],+ engines: [],+ projects: [],+ },+ desktop: { command: PYTHON, args: [DESKTOP_MCP], engines: [], projects: [] },+ }+}++/** Whether this computer carries them: Harness OS from the release that bundles them. */+export const computerUseAvailable = (exists: (path: string) => boolean = existsSync): boolean =>+ [PLAYWRIGHT_MCP, DESKTOP_MCP, CHROMIUM].every((path) => exists(path))++/** On: both attached. */+export const computerUseOn = (servers: MachineMcpServers): boolean => COMPUTER_USE.every((name) => !!servers[name])++/**+ * OpenCode, whose launch takes no MCP server: its own global config, as Connections writes it+ * (lib/connectors/agents.ts), read by its background service on `opencode reload`. OpenCode 2 (Harness OS has+ * 2.0.21) keeps them under `mcp.servers`, a local one as `{ type: 'local', command }`, as `opencode mcp add --global`+ * writes them; OpenCode 1 directly under `mcp`, with `enabled`. Only an entry running one of these servers is+ * Harness's to change: one of the person's under the same name stays.+ */+export interface Opencode {+ /** Its global opencode.json. */+ path: string+ /** Its major version, or null when it is not installed. */+ major: () => number | null+ /** Asks its service to read the config again. */+ reload: () => void+}++export function machineOpencode(environment: NodeJS.ProcessEnv = process.env, home: string = homedir()): Opencode {+ return {+ path: join(environment.XDG_CONFIG_HOME || join(home, '.config'), 'opencode', 'opencode.json'),+ major: () => {+ try {+ const out = execFileSync('opencode', ['--version'], { encoding: 'utf8', timeout: 15_000, stdio: ['ignore', 'pipe', 'ignore'] })+ const major = /(\d+)\.\d+\.\d+/.exec(out)?.[1]+ return major ? Number(major) : null+ } catch { return null }+ },+ reload: () => {+ // Its service picks the change up now; a failure only leaves it to the next start.+ execFile('opencode', ['reload'], { timeout: 30_000 }, () => {})+ },+ }+}++const ours = (entry: unknown): boolean => {+ const command = (entry as { command?: unknown } | null)?.command+ return Array.isArray(command) && command.some((part) => part === PLAYWRIGHT_MCP || part === DESKTOP_MCP)+}++/** Put the computer-use tools into OpenCode's config, or take them out: what it could not do, in words, or null. */+export function applyOpencode(on: boolean, opencode: Opencode, home: string = homedir()): string | null {+ const major = opencode.major()+ if (major === null) return null+ let data: Record<string, unknown> = {}+ if (existsSync(opencode.path)) {+ try {+ const parsed: unknown = JSON.parse(readFileSync(opencode.path, 'utf8'))+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object')+ data = parsed as Record<string, unknown>+ } catch {+ return `OpenCode's ${opencode.path} is not plain JSON, so it was left as it is: OpenCode does not get them.`+ }+ } else if (!on) return null+ const record = (value: unknown): Record<string, unknown> =>+ value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}+ const mcp = record(data.mcp)+ const servers = major >= 2 ? record(mcp.servers) : mcp+ const skipped: string[] = []+ for (const [name, server] of Object.entries(computerUseServers(home))) {+ if (servers[name] !== undefined && !ours(servers[name])) { if (on) skipped.push(name); continue }+ if (!on) { delete servers[name]; continue }+ const command = [server.command, ...server.args]+ servers[name] = major >= 2 ? { type: 'local', command } : { type: 'local', command, enabled: true }+ }+ if (major >= 2) {+ if (Object.keys(servers).length) mcp.servers = servers+ else delete mcp.servers+ }+ if (Object.keys(mcp).length) data.mcp = mcp+ else delete data.mcp+ mkdirSync(dirname(opencode.path), { recursive: true })+ const temporary = `${opencode.path}.${process.pid}.tmp`+ writeFileSync(temporary, JSON.stringify(data, null, 2) + '\n')+ renameSync(temporary, opencode.path)+ opencode.reload()+ return skipped.length ? `OpenCode already has its own ${skipped.join(' and ')}, so it was left as it is.` : null+}++/** Turn them on or off in this machine's mcp-servers.json (Claude Code, Codex) and OpenCode's config: whether they+ * are on afterwards and what OpenCode could not take, or why it could not. */+export function setComputerUse(+ on: boolean, dataDir: string = env.ADAPTER_DATA_DIR, home: string = homedir(), opencode: Opencode = machineOpencode(process.env, home),+): { on: boolean; note?: string } | { error: string } {+ const { servers, why } = readMachineMcp(dataDir)+ if (why) return { error: `Fix or remove it first: ${why}` }+ if (on) {+ const added = COMPUTER_USE.filter((name) => !servers[name]).length+ if (Object.keys(servers).length + added > MACHINE_MCP_MOST_SERVERS) {+ return { error: `At most ${MACHINE_MCP_MOST_SERVERS} machine MCP servers: remove one with harness mcp-servers remove first.` }+ }+ Object.assign(servers, computerUseServers(home))+ } else {+ for (const name of COMPUTER_USE) delete servers[name]+ }+ writeMachineMcp(servers, dataDir)+ const note = applyOpencode(on, opencode, home)+ return note ? { on, note } : { on }+}++export interface ComputerUseDeps {+ dataDir?: string+ home?: string+ exists?: (path: string) => boolean+ opencode?: Opencode+ /** Who runs it, by what its shell says (model.ts `outsideCaller`): an agent may ask, never change. */+ env?: NodeJS.ProcessEnv+ /** Whether it runs in a pane a Harness daemon made (cli.ts, as for `harness mcp-servers`): refused alike. */+ inPane?: boolean+ out?: (line: string) => void+ err?: (line: string) => void+}++export function computerUseCommand(argv: readonly string[], deps: ComputerUseDeps = {}): number {+ const out = deps.out ?? console.log+ const err = deps.err ?? console.error+ const dataDir = deps.dataDir ?? env.ADAPTER_DATA_DIR+ if (argv.includes('--help') || argv.includes('-h')) { out(COMPUTER_USE_USAGE); return 0 }+ const [verb = 'status', ...rest] = argv+ if (rest.length || !['status', 'on', 'off'].includes(verb)) { err(COMPUTER_USE_USAGE); return 2 }+ if (!computerUseAvailable(deps.exists)) {+ err('Computer use comes with Harness OS (Playwright and the desktop MCP): this computer has no such tools.')+ return 1+ }+ if (verb === 'status') {+ const { servers, why } = readMachineMcp(dataDir)+ if (why) { err(`Not read, so no agent gets them: ${why}`); return 1 }+ out(computerUseOn(servers)+ ? 'Computer use is on: agents Harness starts get playwright and desktop. harness computer-use off takes them off.'+ : 'Computer use is off. harness computer-use on gives the agents Harness starts playwright and desktop.')+ return 0+ }+ const agent = outsideCaller(deps.env ?? process.env, '')+ if (agent || deps.inPane) {+ err(`${agent ? agent.name : 'An agent in a Harness pane'} may not turn computer use on or off: its tools run inside every `+ + 'later agent here. Ask the person to turn them on from their phone (the computer\'s Computer use), or to run '+ + `harness computer-use ${verb} in a terminal outside Harness.`)+ return REFUSED+ }+ const home = deps.home ?? homedir()+ const result = setComputerUse(verb === 'on', dataDir, home, deps.opencode ?? machineOpencode(process.env, home))+ if ('error' in result) { err(result.error); return 1 }+ out(result.on+ ? 'Computer use on: agents started from now on get playwright and desktop. Restart a running harness to give it them.'+ : 'Computer use off: agents started from now on don\'t get them. Restart a running harness to take them away.')+ if (result.note) err(result.note)+ return 0+}diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 3109791cd..cd2e5a6de 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -181,6 +181,7 @@ Machine:harness help agents the agent commands, states, exit codes and rules on one screenharness settings [<name> on|off] what agents may do (agents-start-agents, agents-tell-agents, pull-request-line, agent-messages): your account's experiments in Settings ▸ Experimental, each off by defaultharness mcp-servers [add|remove] local MCP servers (a paired lamp's lamp-mcp) for the agents Harness starts here: yours alone to change+ harness computer-use [on|off] Harness OS: browser (Playwright) and desktop tools for agents to try what they build; off until you turn them onharness machines delete <id> remove ANOTHER machine (refuses this one; use \`harness logout\`)harness machines engines <machine> [--engine E]… [--json]ask another of your machines now which engines it has: installed, signed in, first prompt@@ -2718,6 +2719,7 @@ import { openUrlCommand } from './agentClient/openUrl.js'import { harnessRunner, mcpCommand } from './agentClient/mcp.js'import { settingsCommand } from './agentClient/settings.js'import { mcpServersCommand } from './agentClient/mcpServers.js'+import { computerUseCommand } from './agentClient/computerUse.js'// ── arg parse ──────────────────────────────────────────────────────────────────────────────────// `hn` is the terminal client's short name (like tmux, fzf): the same CLI, entered at `tui`. A call@@ -2779,6 +2781,12 @@ switch (cmd) {process.exitCode = mcpServersCommand(rest, { inPane: !!pane && (!pane.startsWith('%') || !!paneOwner(pane)) })break}+ case 'computer-use': {+ // Machine MCP servers too, refused in a Harness pane alike.+ const pane = sendPane(process.env)+ process.exitCode = computerUseCommand(rest, { inPane: !!pane && (!pane.startsWith('%') || !!paneOwner(pane)) })+ break+ }case 'mcp': {const pane = process.env.TMUX_PANEmcpCommand({ run: harnessRunner(SCRIPT_PATH), inPane: !!pane && !!paneOwner(pane) }).then(code => { process.exitCode = code }).catch(onError)diff --git a/cli/src/services/osDevice.spec.ts b/cli/src/services/osDevice.spec.tsindex 67bd50543..7b31476c5 100644--- a/cli/src/services/osDevice.spec.ts+++ b/cli/src/services/osDevice.spec.ts@@ -191,6 +191,41 @@ describe('the Harness OS computer, managed from the phone', () => {expect(await ask({ action: 'update' })).toEqual({ started: true })})+ describe('computer use, turned on and off from the phone', () => {+ function tools(available = true) {+ const state = { on: false }+ const computerUse = {+ available: () => available,+ on: () => state.on,+ set: vi.fn((on: boolean) => { state.on = on; return { on } }),+ }+ return { ...service({ computerUse }), computerUse }+ }++ it('names them among its features only on a computer that carries them', async () => {+ expect(await tools().ask({ action: 'status' })).toMatchObject({ features: expect.arrayContaining(['computer_use']) })+ expect((await tools(false).ask({ action: 'status' }) as { features: string[] }).features).not.toContain('computer_use')+ })++ it('says how they are, and turns them on and off', async () => {+ vi.spyOn(console, 'log').mockImplementation(() => {})+ const { ask, computerUse } = tools()+ expect(await ask({ action: 'computer_use' })).toEqual({ on: false })+ expect(await ask({ action: 'computer_use', state: 'on' })).toEqual({ on: true })+ expect(await ask({ action: 'computer_use' })).toEqual({ on: true })+ expect(await ask({ action: 'computer_use', state: 'off' })).toEqual({ on: false })+ expect(computerUse.set.mock.calls).toEqual([[true], [false]])+ })++ it('answers why it could not, and nothing on a computer without them', async () => {+ const { ask, computerUse } = tools()+ computerUse.set.mockReturnValueOnce({ error: 'Fix or remove it first: not JSON' } as never)+ expect(await ask({ action: 'computer_use', state: 'on' })).toEqual({ error: 'FAILED', message: 'Fix or remove it first: not JSON' })+ expect(await tools(false).ask({ action: 'computer_use', state: 'on' })).toEqual({ error: 'UNSUPPORTED_OS' })+ expect(await tools().ask({ action: 'computer_use', state: 'on' }, { local: false, owner: false })).toEqual({ error: 'OWNER_REQUIRED' })+ })+ })+describe('checking for updates now, from the phone', () => {/** A computer whose check is held until [done] settles it. */function holding() {diff --git a/cli/src/services/osDevice.ts b/cli/src/services/osDevice.tsindex db0ccb8c1..99b629933 100644--- a/cli/src/services/osDevice.ts+++ b/cli/src/services/osDevice.ts@@ -15,6 +15,8 @@ import { existsSync } from 'node:fs'import { readFile } from 'node:fs/promises'import { basename } from 'node:path'import type { CoreApi, ServiceRequests } from '../core/api.js'+import { setComputerUse, computerUseAvailable, computerUseOn } from '../agentClient/computerUse.js'+import { readMachineMcp } from '../lib/machineMcp.js'/** The requests this service answers for the apps, declared in core/api.ts for the core to route. */export { OS_DEVICE_REQUESTS } from '../core/api.js'@@ -49,7 +51,7 @@ export const GH_TIMEOUT_MS = 15_000export const GH_INSTALL_TIMEOUT_MS = 300_000/** How long a GitHub sign-in waits for its code to be entered: GitHub's device codes last 15 minutes. */export const GITHUB_SIGN_IN_MS = 15 * 60_000-const ACTIONS = ['update', 'check_update', 'wifi_scan','wifi_join', 'wifi_result', 'sign_out', 'password', 'ssh', 'github']+const ACTIONS = ['update', 'check_update', 'wifi_scan','wifi_join', 'wifi_result', 'sign_out', 'password', 'ssh', 'github', 'computer_use']/** The most output a command run here may print before it is stopped. */export const MAX_OUTPUT = 1024 * 1024@@ -67,9 +69,24 @@ export interface OsDeviceDeps {spawnDetached?: (file: string, args: readonly string[], input?: string) => void/** Start `gh auth login`'s device flow (a code to enter at github.com/login/device). */githubSignIn?: () => GithubSignIn+ /** Computer use (agentClient/computerUse.ts): whether this computer has them, whether they are on, turning them. */+ computerUse?: ComputerUsenow?: () => number}+export interface ComputerUse {+ available: () => boolean+ on: () => boolean+ set: (on: boolean) => { on: boolean; note?: string } | { error: string }+}++/** This machine's mcp-servers.json, where `harness computer-use` turns them on and off too. */+const machineComputerUse: ComputerUse = {+ available: () => computerUseAvailable(),+ on: () => computerUseOn(readMachineMcp().servers),+ set: (on) => setComputerUse(on),+}+/** A `gh auth login` under way: the code it shows, and it ending once the code was entered (or refused). */export interface GithubSignIn {code: Promise<{ code: string; url: string }>@@ -239,6 +256,7 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceRconst detached = deps.spawnDetached ?? spawnDetachedconst startSignIn = deps.githubSignIn ?? githubDeviceSignInconst now = deps.now ?? Date.now+ const computerUse = deps.computerUse ?? machineComputerUselet updateStartedAt: number | null = null/** The GitHub sign-in waiting for its code, and why the last one ended without signing in. */let signingIn: { run: GithubSignIn; code: string; url: string } | null = null@@ -275,10 +293,20 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceR// features: what this daemon does for a phone beyond the update and the Wi-Fi.return {harnessOs: true, update, wifi: { ssid }, hotspot, encrypted: /(^|\s)rd\.luks\.name=/.test(cmdline),- features: ['sign_out', 'password', 'ssh', 'github', 'check_update'],+ // computer_use on a computer that carries them (Harness OS from the release that bundles them).+ features: ['sign_out', 'password', 'ssh', 'github', 'check_update', ...(computerUse.available() ? ['computer_use'] : [])],}}+ /** Computer use, on, off or how they are: `{ on }`, for the phone to say a running harness gets them on restart. */+ const setTools = (payload: Payload): Payload => {+ if (!computerUse.available()) return { error: 'UNSUPPORTED_OS' }+ if (payload.state !== 'on' && payload.state !== 'off') return { on: computerUse.on() }+ console.log(`[os-device] computer use ${payload.state}`)+ const done = computerUse.set(payload.state === 'on')+ return 'error' in done ? { error: 'FAILED', message: done.error } : done+ }+/** Check for updates now, as the timer does every 15 minutes, the OS release too: answered at once,* `update.checking` in the status until it is done, for the phone to follow. One at a time. */const checkUpdate = (): Payload => {@@ -449,6 +477,7 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceRtry {if (action === 'update') return update(payload)if (action === 'check_update') return checkUpdate()+ if (action === 'computer_use') return setTools(payload)if (action === 'wifi_scan') return await scan()if (action === 'wifi_join') return join(payload)if (action === 'sign_out') return signOut()diff --git a/os/desktop_mcp.py b/os/desktop_mcp.pynew file mode 100755index 000000000..2539fb2fb--- /dev/null+++ b/os/desktop_mcp.py@@ -0,0 +1,448 @@+#!/usr/bin/python3+"""Harness OS's desktop as tools for an agent: what is on the screen, and the mouse and keyboard on it.++A stdio MCP server (JSON-RPC 2.0, a message a line) for an agent to try the desktop app it builds as a person+would: `screenshot`, then `click`, `drag`, `scroll`, `type` and `key` where it sees things, each answered with+the screen as it is afterwards. Off until the person turns it on, with Playwright's, from their phone or with+`harness computer-use on` (cli agentClient/computerUse.ts), never by an agent.++Input goes through the compositor's own virtual devices (labwc: wlr-virtual-pointer for the mouse,+virtual-keyboard for the keys), so it stays inside this session: no /dev/uinput. While the screen is locked+it refuses everything, the screenshot too.++The keyboard is one of its own for as long as the server runs, with the compositor's default keymap: keys reach+an app as a real keyboard's do. wtype, a keyboard with a keymap of just the characters it types, is used only+for text that keymap cannot type (accented letters): GTK 3 ignores wtype's Return, BackSpace and shortcuts.++Coordinates are the screenshot's pixels. A screen wider than WIDTH is shown scaled down to it, and a point on the+screenshot is scaled back up to the screen.+"""+import base64+import ctypes+import importlib+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import time++NAME = 'desktop'+VERSION = '0.1.0'+PROTOCOL = '2025-06-18'+# The widest screenshot an agent is shown: wide enough to read a window's text, small enough to send each turn.+WIDTH = 1280+# How long the screen gets to change before the screenshot that answers an action.+SETTLE = 0.4+# A wtype is a keyboard of its own, its keymap sent to the app in front as it starts: keys from another+# keyboard right after it were dropped (labwc, Chromium, 0.4 s apart).+KEY_GAP = 1.0+POINTER_XML = Path('/usr/share/wlr-protocols/unstable/wlr-virtual-pointer-unstable-v1.xml')+# Not in a package of Arch's: Harness OS's own copy of wlroots' (MIT).+KEYBOARD_XML = Path('/usr/share/harness-os/virtual-keyboard-unstable-v1.xml')+CACHE = Path(os.environ.get('XDG_CACHE_HOME') or Path.home() / '.cache') / 'harness-os' / 'desktop-mcp'+# Linux input event codes (linux/input-event-codes.h).+BUTTONS = {'left': 0x110, 'right': 0x111, 'middle': 0x112}+# Friendly key names to XKB keysym names; anything else is looked up as it is (`F5`, `a`, `Page_Up`).+KEYS = {'enter': 'Return', 'return': 'Return', 'esc': 'Escape', 'escape': 'Escape', 'tab': 'Tab',+ 'backspace': 'BackSpace', 'delete': 'Delete', 'del': 'Delete', 'space': 'space', 'up': 'Up',+ 'down': 'Down', 'left': 'Left', 'right': 'Right', 'home': 'Home', 'end': 'End',+ 'pageup': 'Page_Up', 'pagedown': 'Page_Down', 'insert': 'Insert',+ # A modifier pressed on its own.+ 'shift': 'Shift_L', 'ctrl': 'Control_L', 'control': 'Control_L', 'alt': 'Alt_L', 'super': 'Super_L'}+MODIFIERS = {'ctrl': 'Control_L', 'control': 'Control_L', 'shift': 'Shift_L', 'alt': 'Alt_L',+ 'altgr': 'ISO_Level3_Shift', 'super': 'Super_L', 'logo': 'Super_L', 'win': 'Super_L',+ 'meta': 'Super_L', 'cmd': 'Super_L'}+# Each modifier key's XKB modifier: the virtual keyboard's modifiers are said, not worked out from its keys+# (on labwc, Shift and Control held by key alone changed nothing).+MODIFIER_NAMES = {'Shift_L': 'Shift', 'Control_L': 'Control', 'Alt_L': 'Mod1', 'Super_L': 'Mod4',+ 'ISO_Level3_Shift': 'Mod5'}+++class Refused(Exception):+ """An action this server will not take, said to the agent as a tool error."""+++def environment():+ """The session's display, for a server started from an SSH pane or a service with none of its own."""+ env = dict(os.environ)+ runtime = env.setdefault('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}')+ if not env.get('WAYLAND_DISPLAY'):+ sockets = sorted(p.name for p in Path(runtime).glob('wayland-*') if not p.name.endswith('.lock'))+ if not sockets:+ raise Refused('No desktop is running on this computer.')+ env['WAYLAND_DISPLAY'] = sockets[0]+ return env+++def locked():+ return subprocess.run(['pgrep', '-x', 'gtklock'], capture_output=True).returncode == 0+++def png_size(data):+ if data[:8] != b'\x89PNG\r\n\x1a\n':+ raise Refused('The screenshot is not a PNG.')+ return int.from_bytes(data[16:20], 'big'), int.from_bytes(data[20:24], 'big')+++def bindings(xml):+ """pywayland's module for the protocol in [xml], made by its scanner once per pywayland version.++ The scanner writes one file a protocol, importing the core one relatively (`from .wayland import WlSeat`):+ so a package of its own, with a `wayland` that is pywayland's.+ """+ import pywayland+ from pywayland.scanner import Protocol+ package = 'harness_desktop_protocols'+ folder = CACHE / pywayland.__version__+ protocol = Protocol.parse_file(str(xml))+ module = folder / package / f'{protocol.name}.py'+ if not module.is_file():+ imports = {interface.name: protocol.name for interface in protocol.interface}+ imports.update({name: 'wayland' for name in ('wl_seat', 'wl_output', 'wl_surface')})+ (folder / package).mkdir(parents=True, exist_ok=True)+ (folder / package / '__init__.py').touch()+ (folder / package / 'wayland.py').write_text('from pywayland.protocol.wayland import * # noqa: F403\n')+ with tempfile.TemporaryDirectory(dir=folder / package) as temp:+ protocol.output(temp, imports)+ os.replace(Path(temp) / module.name, module)+ if str(folder) not in sys.path:+ sys.path.insert(0, str(folder))+ return importlib.import_module(f'{package}.{protocol.name}')+++class Keymap:+ """The compositor's default keymap (XKB's defaults, as labwc's), compiled by libxkbcommon: its text for the+ virtual keyboard, and where each keysym is, as an evdev key and whether Shift reaches it."""++ def __init__(self):+ xkb = ctypes.CDLL('libxkbcommon.so.0')+ xkb.xkb_context_new.restype = ctypes.c_void_p+ xkb.xkb_keymap_new_from_names.restype = ctypes.c_void_p+ xkb.xkb_keymap_new_from_names.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]+ xkb.xkb_keymap_get_as_string.restype = ctypes.c_void_p+ xkb.xkb_keymap_get_as_string.argtypes = [ctypes.c_void_p, ctypes.c_int]+ for name in ('xkb_keymap_min_keycode', 'xkb_keymap_max_keycode'):+ getattr(xkb, name).restype = ctypes.c_uint32+ getattr(xkb, name).argtypes = [ctypes.c_void_p]+ xkb.xkb_keymap_key_get_syms_by_level.argtypes = [+ ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.POINTER(ctypes.c_uint32))]+ xkb.xkb_keysym_from_name.restype = ctypes.c_uint32+ xkb.xkb_keysym_from_name.argtypes = [ctypes.c_char_p, ctypes.c_int]+ xkb.xkb_utf32_to_keysym.restype = ctypes.c_uint32+ xkb.xkb_utf32_to_keysym.argtypes = [ctypes.c_uint32]+ xkb.xkb_keymap_mod_get_index.restype = ctypes.c_uint32+ xkb.xkb_keymap_mod_get_index.argtypes = [ctypes.c_void_p, ctypes.c_char_p]+ libc = ctypes.CDLL(None)+ libc.free.argtypes = [ctypes.c_void_p]+ self.xkb = xkb+ keymap = xkb.xkb_keymap_new_from_names(xkb.xkb_context_new(0), None, 0)+ if not keymap:+ raise Refused('Could not make a keymap for the virtual keyboard.')+ text = xkb.xkb_keymap_get_as_string(keymap, 1) # XKB_KEYMAP_FORMAT_TEXT_V1+ self.text = ctypes.string_at(text) + b'\0'+ libc.free(text)+ self.where = {}+ syms = ctypes.POINTER(ctypes.c_uint32)()+ for keycode in range(xkb.xkb_keymap_min_keycode(keymap), xkb.xkb_keymap_max_keycode(keymap) + 1):+ for level in (0, 1):+ for i in range(xkb.xkb_keymap_key_get_syms_by_level(keymap, keycode, 0, level, ctypes.byref(syms))):+ # The first, unshifted, place a keysym has wins: `1` is 2's key, not the keypad's.+ self.where.setdefault(syms[i], (keycode - 8, level == 1))+ # A modifier key's evdev code to its mask.+ self.masks = {}+ for sym_name, mod_name in MODIFIER_NAMES.items():+ index = xkb.xkb_keymap_mod_get_index(keymap, mod_name.encode())+ sym = xkb.xkb_keysym_from_name(sym_name.encode(), 0)+ if index != 0xFFFFFFFF and sym in self.where:+ self.masks[self.where[sym][0]] = 1 << index++ def named(self, name):+ sym = self.xkb.xkb_keysym_from_name(name.encode(), 0) or self.xkb.xkb_keysym_from_name(name.encode(), 1)+ if not sym or sym not in self.where:+ raise Refused(f'Not a key on this keyboard: {name!r}. Use names like "enter", "ctrl+s", "F5".')+ return self.where[sym]++ def character(self, char):+ """Where [char] is, or None for one this keymap cannot type."""+ if char == '\n':+ return self.where.get(0xff0d) # Return+ if char == '\t':+ return self.where.get(0xff09) # Tab+ return self.where.get(self.xkb.xkb_utf32_to_keysym(ord(char)))+++class Devices:+ """This session's own mouse and keyboard, made by the compositor (one Wayland connection, kept)."""++ def __init__(self, env):+ from pywayland.client import Display+ from pywayland.protocol.wayland import WlSeat+ pointer, keyboard = bindings(POINTER_XML), bindings(KEYBOARD_XML)+ self.display = Display(env['WAYLAND_DISPLAY'])+ self.display.connect()+ found = {}++ def announced(registry, name, interface, version):+ if interface == 'wl_seat' and 'seat' not in found:+ found['seat'] = registry.bind(name, WlSeat, 1)+ elif interface == 'zwlr_virtual_pointer_manager_v1':+ found['pointers'] = registry.bind(name, pointer.ZwlrVirtualPointerManagerV1, 1)+ elif interface == 'zwp_virtual_keyboard_manager_v1':+ found['keyboards'] = registry.bind(name, keyboard.ZwpVirtualKeyboardManagerV1, 1)++ registry = self.display.get_registry()+ registry.dispatcher['global'] = announced+ self.display.roundtrip()+ missing = {'seat', 'pointers', 'keyboards'} - set(found)+ if missing:+ self.display.disconnect()+ raise Refused('This desktop has no virtual mouse or keyboard (' + ', '.join(sorted(missing)) + ').')+ self.pointer = found['pointers'].create_virtual_pointer(found['seat'])+ self.keymap = Keymap()+ self.keyboard = found['keyboards'].create_virtual_keyboard(found['seat'])+ fd = os.memfd_create('harness-desktop-keymap')+ os.write(fd, self.keymap.text)+ self.keyboard.keymap(1, fd, len(self.keymap.text)) # WL_KEYBOARD_KEYMAP_FORMAT_XKB_V1+ self.display.roundtrip()+ os.close(fd)++ @staticmethod+ def now():+ return int(time.monotonic() * 1000) & 0xFFFFFFFF++ def frame(self):+ self.pointer.frame()+ self.display.roundtrip()++ def move(self, x, y, size):+ self.pointer.motion_absolute(self.now(), x, y, size[0], size[1])+ self.frame()++ def button(self, code, pressed):+ self.pointer.button(self.now(), code, 1 if pressed else 0)+ self.frame()++ def scroll(self, steps):+ # A wheel's notches: 15 units each, as a mouse wheel reports them. Positive scrolls down.+ self.pointer.axis_source(0)+ self.pointer.axis_discrete(self.now(), 0, 15.0 * steps, steps)+ self.frame()++ def key(self, code, pressed):+ self.keyboard.key(self.now(), code, 1 if pressed else 0)+ self.display.roundtrip()++ def press(self, codes):+ """Hold [codes] down in order, then let them go in reverse: a shortcut, or a key with Shift. The+ modifiers among them are said as they go down and up."""+ held = 0+ for code in codes:+ self.key(code, True)+ if code in self.keymap.masks:+ held |= self.keymap.masks[code]+ self.keyboard.modifiers(held, 0, 0, 0)+ for code in reversed(codes):+ self.key(code, False)+ if code in self.keymap.masks:+ held &= ~self.keymap.masks[code]+ self.keyboard.modifiers(held, 0, 0, 0)+ self.display.roundtrip()+ time.sleep(0.01)+++class Screen:+ def __init__(self):+ self.env = environment()+ self.size = None+ self.devices = None+ self.wtyped = 0.0++ def grab(self, scale=None):+ args = ['grim', '-t', 'png'] + (['-s', f'{scale:.4f}'] if scale else []) + ['-']+ done = subprocess.run(args, capture_output=True, env=self.env, timeout=15)+ if done.returncode:+ raise Refused('Could not take a screenshot: ' + done.stderr.decode(errors='replace').strip())+ return done.stdout++ def scale(self):+ """The screenshot's size over the screen's: 1 for a screen no wider than WIDTH."""+ if self.size is None:+ self.size = png_size(self.grab())+ return min(1.0, WIDTH / self.size[0])++ def screenshot(self):+ if locked():+ raise Refused('The screen is locked. Ask the person to unlock it.')+ scale = self.scale()+ data = self.grab(scale if scale < 1 else None)+ return data, png_size(data)++ def point(self, x, y):+ """A screenshot's pixel on the screen."""+ scale = self.scale()+ width, height = self.size+ x, y = round(float(x) / scale), round(float(y) / scale)+ if not (0 <= x < width and 0 <= y < height):+ raise Refused(f'({x}, {y}) is outside the screen ({width}x{height} on the screen).')+ return x, y++ def input(self):+ if self.devices is None:+ self.devices = Devices(self.env)+ return self.devices++ def after_wtype(self):+ time.sleep(max(0.0, self.wtyped + KEY_GAP - time.monotonic()))++ def type(self, text):+ devices = self.input()+ places = [devices.keymap.character(char) for char in text]+ if all(places):+ self.after_wtype()+ shift = devices.keymap.named('Shift_L')[0]+ for code, shifted in places:+ devices.press([shift, code] if shifted else [code])+ return+ # Accented letters and the like: wtype, with a keymap of its own made of them.+ self.after_wtype()+ done = subprocess.run(['wtype', '--', text], capture_output=True, env=self.env, timeout=60)+ self.wtyped = time.monotonic()+ if done.returncode:+ raise Refused('Could not type: ' + done.stderr.decode(errors='replace').strip())++ def key(self, keys):+ """`ctrl+shift+t`: the modifiers held around the key, then let go."""+ parts = [part.strip() for part in str(keys).split('+')]+ if not parts or not all(parts):+ raise Refused(f'Not a key: {keys!r}. Use names like "enter", "ctrl+s" or "super+b".')+ devices = self.input()+ *modifiers, key = parts+ codes = []+ for modifier in modifiers:+ if modifier.lower() not in MODIFIERS:+ raise Refused(f'Not a modifier: {modifier!r}. Use ctrl, shift, alt, altgr or super.')+ codes.append(devices.keymap.named(MODIFIERS[modifier.lower()])[0])+ code, shifted = devices.keymap.named(KEYS.get(key.lower(), key))+ shift = devices.keymap.named('Shift_L')[0]+ if shifted and shift not in codes:+ codes.append(shift)+ self.after_wtype()+ devices.press(codes + [code])+++POINT = {'x': {'type': 'number', 'description': 'Pixels from the left of the screenshot.'},+ 'y': {'type': 'number', 'description': 'Pixels from the top of the screenshot.'}}+TOOLS = [+ {'name': 'screenshot', 'description': 'What is on the screen now, as an image. Coordinates for the other '+ 'tools are this image\'s pixels.', 'inputSchema': {'type': 'object', 'properties': {}}},+ {'name': 'click', 'description': 'Move the mouse to a point and click there. Answers with the screen after.',+ 'inputSchema': {'type': 'object', 'required': ['x', 'y'], 'properties': {+ **POINT, 'button': {'type': 'string', 'enum': list(BUTTONS), 'default': 'left'},+ 'count': {'type': 'integer', 'minimum': 1, 'maximum': 3, 'default': 1,+ 'description': '2 for a double click.'}}}},+ {'name': 'move', 'description': 'Move the mouse to a point without clicking (to hover). Answers with the '+ 'screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y'], 'properties': POINT}},+ {'name': 'drag', 'description': 'Press the left button at one point, move to another and let go. Answers '+ 'with the screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y', 'to_x', 'to_y'],+ 'properties': {**POINT, 'to_x': {'type': 'number'},+ 'to_y': {'type': 'number'}}}},+ {'name': 'scroll', 'description': 'Turn the mouse wheel over a point: positive steps scroll down, negative '+ 'up. Answers with the screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y', 'steps'],+ 'properties': {**POINT, 'steps': {'type': 'integer'}}}},+ {'name': 'type', 'description': 'Type text into whatever has the keyboard focus, as a person would ("\\n" '+ 'is Enter). Answers with the screen after.',+ 'inputSchema': {'type': 'object', 'required': ['text'], 'properties': {'text': {'type': 'string'}}}},+ {'name': 'key', 'description': 'Press a key or a combination: "enter", "esc", "tab", "backspace", "ctrl+s", '+ '"ctrl+shift+t", "super+b", "F5". Answers with the screen after.',+ 'inputSchema': {'type': 'object', 'required': ['keys'], 'properties': {'keys': {'type': 'string'}}}},+]+++def run_tool(screen, name, args):+ """Do [name] with [args]; the screen afterwards, as the answer's content."""+ if name != 'screenshot':+ if locked():+ raise Refused('The screen is locked. Ask the person to unlock it.')+ if name in ('click', 'move', 'drag', 'scroll'):+ x, y = screen.point(args['x'], args['y'])+ devices = screen.input()+ devices.move(x, y, screen.size)+ if name == 'click':+ code = BUTTONS.get(args.get('button', 'left'))+ if code is None:+ raise Refused('button is left, right or middle.')+ for _ in range(max(1, min(3, int(args.get('count', 1))))):+ devices.button(code, True)+ devices.button(code, False)+ elif name == 'drag':+ to = screen.point(args['to_x'], args['to_y'])+ devices.button(BUTTONS['left'], True)+ for step in range(1, 11):+ devices.move(round(x + (to[0] - x) * step / 10), round(y + (to[1] - y) * step / 10), screen.size)+ time.sleep(0.02)+ devices.button(BUTTONS['left'], False)+ elif name == 'scroll':+ devices.scroll(int(args['steps']))+ elif name == 'type':+ screen.type(str(args['text']))+ elif name == 'key':+ screen.key(args['keys'])+ else:+ raise Refused(f'No tool {name!r}.')+ time.sleep(SETTLE)+ data, (width, height) = screen.screenshot()+ return [{'type': 'text', 'text': f'The screen now, {width}x{height}.'},+ {'type': 'image', 'data': base64.b64encode(data).decode(), 'mimeType': 'image/png'}]+++def answer(request, screen):+ method, ident = request.get('method'), request.get('id')+ if method == 'initialize':+ asked = (request.get('params') or {}).get('protocolVersion')+ result = {'protocolVersion': asked or PROTOCOL, 'capabilities': {'tools': {}},+ 'serverInfo': {'name': NAME, 'version': VERSION}}+ elif method == 'tools/list':+ result = {'tools': TOOLS}+ elif method == 'tools/call':+ params = request.get('params') or {}+ try:+ result = {'content': run_tool(screen(), params.get('name'), params.get('arguments') or {})}+ except (Refused, KeyError, ValueError, TypeError, OSError, subprocess.SubprocessError) as error:+ text = str(error) if isinstance(error, Refused) else f'{type(error).__name__}: {error}'+ result = {'content': [{'type': 'text', 'text': text}], 'isError': True}+ elif method == 'ping':+ result = {}+ elif ident is None:+ return None # A notification (initialized, cancelled): nothing to answer.+ else:+ return {'jsonrpc': '2.0', 'id': ident, 'error': {'code': -32601, 'message': f'No method {method}'}}+ return {'jsonrpc': '2.0', 'id': ident, 'result': result}+++def main():+ held = {}++ def screen():+ # Made at the first tool call, so the server starts (and lists its tools) without a desktop.+ if 'screen' not in held:+ held['screen'] = Screen()+ return held['screen']++ for line in sys.stdin:+ if not line.strip():+ continue+ try:+ request = json.loads(line)+ except json.JSONDecodeError:+ reply = {'jsonrpc': '2.0', 'id': None, 'error': {'code': -32700, 'message': 'Not JSON'}}+ else:+ reply = answer(request, screen)+ if reply is not None:+ sys.stdout.write(json.dumps(reply) + '\n')+ sys.stdout.flush()+++if __name__ == '__main__':+ main()diff --git a/os/packages.x86_64 b/os/packages.x86_64index 77d8914b0..89eb2dd6e 100644--- a/os/packages.x86_64+++ b/os/packages.x86_64@@ -41,6 +41,10 @@ intel-media-driverwl-clipboardgrimslurp+# The desktop MCP (os/desktop_mcp.py): an agent's own keyboard and mouse on the screen it sees.+wtype+python-pywayland+wlr-protocolsxdg-utilsxdg-desktop-portal-wlrpipewirediff --git a/os/packaging/playwright-mcp/package-lock.json b/os/packaging/playwright-mcp/package-lock.jsonnew file mode 100644index 000000000..78a8b13a9--- /dev/null+++ b/os/packaging/playwright-mcp/package-lock.json@@ -0,0 +1,57 @@+{+ "name": "harness-os-playwright-mcp",+ "lockfileVersion": 3,+ "requires": true,+ "packages": {+ "": {+ "name": "harness-os-playwright-mcp",+ "license": "MIT",+ "dependencies": {+ "@playwright/mcp": "0.0.83"+ }+ },+ "node_modules/@playwright/mcp": {+ "version": "0.0.83",+ "resolved": "https://registry.npmjs.org/@playwright/mcp/-/mcp-0.0.83.tgz",+ "integrity": "sha512-oNcl+Ae2/IAjhfPeP46BfIkSakfmprY+aOtkv5MjrQ4lPav4/yNtPhL0iq8SlIM90oApWgBDUxaNKvktazUKOg==",+ "license": "Apache-2.0",+ "dependencies": {+ "playwright": "1.64.0-alpha-1790635538000",+ "playwright-core": "1.64.0-alpha-1790635538000"+ },+ "bin": {+ "playwright-mcp": "cli.js"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/playwright": {+ "version": "1.64.0-alpha-1790635538000",+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0-alpha-1790635538000.tgz",+ "integrity": "sha512-/5XDUMxpOd/9AojJtWDFIRV3EJZNQ0AwvK69wgt3CPzy8wTgiBG9pX45VfxtqtvIRGwfWzMhrvawNKreG69Snw==",+ "license": "Apache-2.0",+ "dependencies": {+ "playwright-core": "1.64.0-alpha-1790635538000"+ },+ "bin": {+ "playwright": "cli.js"+ },+ "engines": {+ "node": ">=20"+ }+ },+ "node_modules/playwright-core": {+ "version": "1.64.0-alpha-1790635538000",+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0-alpha-1790635538000.tgz",+ "integrity": "sha512-pNwaXirhXMRLaRQs4NQ18EpTdtDoFwyPH9FOaaDC7YXG4hpwE2xmwCic/1srbuExVQC7zT9LJSJpDBAru+Vo9A==",+ "license": "Apache-2.0",+ "bin": {+ "playwright-core": "cli.js"+ },+ "engines": {+ "node": ">=20"+ }+ }+ }+}diff --git a/os/packaging/playwright-mcp/package.json b/os/packaging/playwright-mcp/package.jsonnew file mode 100644index 000000000..0de729181--- /dev/null+++ b/os/packaging/playwright-mcp/package.json@@ -0,0 +1,9 @@+{+ "name": "harness-os-playwright-mcp",+ "private": true,+ "description": "Playwright's MCP server, bundled in Harness OS for the agents' browser tools (harness computer-use).",+ "license": "MIT",+ "dependencies": {+ "@playwright/mcp": "0.0.83"+ }+}diff --git a/os/root/usr/share/harness-os/AGENTS.md b/os/root/usr/share/harness-os/AGENTS.mdindex 220f15ba9..9926fb64c 100644--- a/os/root/usr/share/harness-os/AGENTS.md+++ b/os/root/usr/share/harness-os/AGENTS.md@@ -116,6 +116,14 @@ is required.- Print/`Super+p` saves a full screenshot and Shift+Print/`Super+r` a region to`~/Pictures/Screenshots`, also copied to the clipboard. When the user mentions"the screenshot", read the newest file there. `grim` and `slurp` are installed.+- To try what you build as a person would, use computer use when you have it:+ `playwright` for a website (it opens it in a Chromium window of its own, clicks,+ types, and reads the page and its console) and `desktop` for a desktop app+ (`screenshot`, `click`, `drag`, `scroll`, `type`, `key`, each answered with the+ screen afterwards; refused while the screen is locked). They are off until the+ person turns them on, from their phone (the computer's Computer use) or with+ `harness computer-use on` in a terminal outside Harness, and a running harness gets+ them when it restarts. Never turn them on yourself: suggest it instead.- On the PC image, the packages for supported NVIDIA Turing and newer GPUs are`nvidia-open-lts nvidia-utils`, including RTX 4090/5090 and RTX 6000 generations.New USB images carrying the NVIDIA bundle install them offline when the exactdiff --git a/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xml b/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xmlnew file mode 100644index 000000000..583ff6e3a--- /dev/null+++ b/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xml@@ -0,0 +1,65 @@+<?xml version="1.0" encoding="UTF-8"?>+<protocol name="virtual_keyboard_unstable_v1">+ <copyright>+ Copyright © 2008-2011 Kristian Høgsberg+ Copyright © 2010-2013 Intel Corporation+ Copyright © 2012-2013 Collabora, Ltd.+ Copyright © 2018 Purism SPC++ Permission is hereby granted, free of charge, to any person obtaining a+ copy of this software and associated documentation files (the "Software"),+ to deal in the Software without restriction, including without limitation+ the rights to use, copy, modify, merge, publish, distribute, sublicense,+ and/or sell copies of the Software, and to permit persons to whom the+ Software is furnished to do so, subject to the following conditions:++ The above copyright notice and this permission notice (including the next+ paragraph) shall be included in all copies or substantial portions of the+ Software.++ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+ DEALINGS IN THE SOFTWARE.+ </copyright>++ <interface name="zwp_virtual_keyboard_v1" version="1">+ <description summary="virtual keyboard">+ The virtual keyboard provides an application with requests which emulate+ the behaviour of a physical keyboard.+ </description>+ <request name="keymap">+ <arg name="format" type="uint" summary="keymap format, one of wl_keyboard.keymap_format"/>+ <arg name="fd" type="fd" summary="keymap file descriptor"/>+ <arg name="size" type="uint" summary="keymap size, in bytes"/>+ </request>+ <enum name="error">+ <entry name="no_keymap" value="0" summary="No keymap was set"/>+ </enum>+ <request name="key">+ <arg name="time" type="uint" summary="timestamp with millisecond granularity"/>+ <arg name="key" type="uint" summary="key that produced the event"/>+ <arg name="state" type="uint" summary="physical state of the key"/>+ </request>+ <request name="modifiers">+ <arg name="mods_depressed" type="uint"/>+ <arg name="mods_latched" type="uint"/>+ <arg name="mods_locked" type="uint"/>+ <arg name="group" type="uint"/>+ </request>+ <request name="destroy" type="destructor" since="1"/>+ </interface>++ <interface name="zwp_virtual_keyboard_manager_v1" version="1">+ <enum name="error">+ <entry name="unauthorized" value="0" summary="client not authorized to use the interface"/>+ </enum>+ <request name="create_virtual_keyboard">+ <arg name="seat" type="object" interface="wl_seat"/>+ <arg name="id" type="new_id" interface="zwp_virtual_keyboard_v1"/>+ </request>+ </interface>+</protocol>diff --git a/os/tests/test_desktop_mcp.py b/os/tests/test_desktop_mcp.pynew file mode 100644index 000000000..b13f5b307--- /dev/null+++ b/os/tests/test_desktop_mcp.py@@ -0,0 +1,149 @@+import importlib.util+import json+from pathlib import Path+import subprocess+import tempfile+import unittest+from unittest.mock import MagicMock, patch++spec = importlib.util.spec_from_file_location('desktop_mcp', Path(__file__).resolve().parents[1] / 'desktop_mcp.py')+desktop = importlib.util.module_from_spec(spec)+spec.loader.exec_module(desktop)++PNG = b'\x89PNG\r\n\x1a\n' + b'\0\0\0\rIHDR' + (1280).to_bytes(4, 'big') + (720).to_bytes(4, 'big') + b'rest'+++def screen(width=1920, height=1080):+ """A Screen on a [width]x[height] display, its devices a fake whose keymap has a few keys."""+ with patch.object(desktop, 'environment', return_value={'WAYLAND_DISPLAY': 'wayland-0'}):+ made = desktop.Screen()+ made.size = (width, height)+ places = {'Control_L': (29, False), 'Shift_L': (42, False), 'Super_L': (125, False), 'Return': (28, False),+ 't': (20, False), 'b': (48, False), 'question': (53, True)}+ keymap = MagicMock()+ keymap.named.side_effect = lambda name: places[name] if name in places else (_ for _ in ()).throw(+ desktop.Refused(f'Not a key on this keyboard: {name!r}.'))+ keymap.character.side_effect = lambda char: {'h': (35, False), 'i': (23, False), '!': (2, True)}.get(char)+ made.devices = MagicMock(keymap=keymap)+ return made+++class Protocol(unittest.TestCase):+ def ask(self, request, made=None):+ return desktop.answer(request, lambda: made or screen())++ def test_it_introduces_itself_in_the_clients_protocol_version(self):+ reply = self.ask({'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {'protocolVersion': '2025-03-26'}})+ self.assertEqual(reply['result']['protocolVersion'], '2025-03-26')+ self.assertEqual(reply['result']['serverInfo']['name'], 'desktop')+ self.assertIn('tools', reply['result']['capabilities'])++ def test_it_lists_its_tools(self):+ tools = self.ask({'jsonrpc': '2.0', 'id': 2, 'method': 'tools/list'})['result']['tools']+ self.assertEqual([tool['name'] for tool in tools], ['screenshot', 'click', 'move', 'drag', 'scroll', 'type', 'key'])+ json.dumps(tools)++ def test_it_answers_no_notification_and_refuses_an_unknown_method(self):+ self.assertIsNone(self.ask({'jsonrpc': '2.0', 'method': 'notifications/initialized'}))+ self.assertEqual(self.ask({'jsonrpc': '2.0', 'id': 3, 'method': 'resources/list'})['error']['code'], -32601)++ def test_a_tool_that_cannot_act_answers_an_error_in_words(self):+ with patch.object(desktop, 'locked', return_value=True):+ reply = self.ask({'jsonrpc': '2.0', 'id': 4, 'method': 'tools/call',+ 'params': {'name': 'click', 'arguments': {'x': 1, 'y': 1}}})+ self.assertTrue(reply['result']['isError'])+ self.assertIn('locked', reply['result']['content'][0]['text'])+++class Screen(unittest.TestCase):+ def test_a_screenshots_pixel_is_scaled_back_to_the_screen(self):+ made = screen()+ self.assertEqual(made.scale(), 1280 / 1920)+ self.assertEqual(made.point(640, 360), (960, 540))+ with self.assertRaisesRegex(desktop.Refused, 'outside the screen'):+ made.point(1280, 10)++ def test_a_narrow_screen_is_shown_as_it_is(self):+ made = screen(1024, 768)+ self.assertEqual(made.scale(), 1.0)+ self.assertEqual(made.point(10, 20), (10, 20))++ def test_the_screenshot_is_refused_while_locked(self):+ with patch.object(desktop, 'locked', return_value=True), self.assertRaisesRegex(desktop.Refused, 'locked'):+ screen().screenshot()++ def test_the_screenshot_is_taken_scaled_down_to_the_width_an_agent_sees(self):+ made = screen()+ with patch.object(desktop, 'locked', return_value=False), \+ patch.object(desktop.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, PNG, b'')) as run:+ data, size = made.screenshot()+ self.assertEqual((data, size), (PNG, (1280, 720)))+ self.assertEqual(run.call_args.args[0], ['grim', '-t', 'png', '-s', '0.6667', '-'])++ def test_an_action_answers_with_the_screen_after_it(self):+ made = screen()+ with patch.object(desktop, 'locked', return_value=False), patch.object(desktop.time, 'sleep'), \+ patch.object(made, 'screenshot', return_value=(PNG, (1280, 720))):+ content = desktop.run_tool(made, 'click', {'x': 640, 'y': 360, 'count': 2})+ made.devices.move.assert_called_with(960, 540, (1920, 1080))+ self.assertEqual(made.devices.button.call_count, 4)+ self.assertEqual([part['type'] for part in content], ['text', 'image'])+ self.assertEqual(content[1]['mimeType'], 'image/png')+++class Keys(unittest.TestCase):+ def test_a_combination_holds_its_modifiers_around_the_key(self):+ made = screen()+ made.key('ctrl+shift+t')+ made.devices.press.assert_called_with([29, 42, 20])+ made.key('super+b')+ made.devices.press.assert_called_with([125, 48])+ made.key('enter')+ made.devices.press.assert_called_with([28])++ def test_a_key_shift_reaches_gets_shift(self):+ made = screen()+ made.key('ctrl+question')+ made.devices.press.assert_called_with([29, 42, 53])++ def test_an_unknown_modifier_or_key_is_refused(self):+ made = screen()+ with self.assertRaisesRegex(desktop.Refused, 'Not a modifier'):+ made.key('hyper+t')+ with self.assertRaisesRegex(desktop.Refused, 'Not a key'):+ made.key('ctrl+')+ with self.assertRaisesRegex(desktop.Refused, 'Not a key on this keyboard'):+ made.key('nosuchkey')++ def test_text_the_keymap_types_goes_through_its_own_keyboard(self):+ made = screen()+ with patch.object(desktop.subprocess, 'run') as run:+ made.type('hi!')+ run.assert_not_called()+ self.assertEqual([call.args[0] for call in made.devices.press.call_args_list], [[35], [23], [42, 2]])++ def test_accented_text_goes_through_wtype(self):+ made = screen()+ with patch.object(desktop.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, b'', b'')) as run:+ made.type('chào')+ self.assertEqual(run.call_args.args[0], ['wtype', '--', 'chào'])+ made.devices.press.assert_not_called()+++class Environment(unittest.TestCase):+ def test_a_server_with_no_display_finds_the_sessions(self):+ with tempfile.TemporaryDirectory() as runtime:+ (Path(runtime) / 'wayland-1').touch()+ (Path(runtime) / 'wayland-1.lock').touch()+ with patch.dict(desktop.os.environ, {'XDG_RUNTIME_DIR': runtime}, clear=True):+ self.assertEqual(desktop.environment()['WAYLAND_DISPLAY'], 'wayland-1')++ def test_no_desktop_is_said(self):+ with tempfile.TemporaryDirectory() as runtime, \+ patch.dict(desktop.os.environ, {'XDG_RUNTIME_DIR': runtime}, clear=True), \+ self.assertRaisesRegex(desktop.Refused, 'No desktop'):+ desktop.environment()+++if __name__ == '__main__':+ unittest.main()diff --git a/os/tools/build-package.py b/os/tools/build-package.pyindex 32f78df03..aba1c2ee2 100644--- a/os/tools/build-package.py+++ b/os/tools/build-package.py@@ -16,7 +16,9 @@ import tempfile# dnsmasq: NetworkManager's shared mode hands the setup hotspot's phone an address with it.# github-cli (gh) and sound-theme-freedesktop (foot.ini's bell): for a computer installed before they were# in the image (packages.x86_64) to get them with an update.-DEPENDENCIES = ('nodejs-lts-jod', 'tmux', 'foot', 'gtklock', 'grim', 'slurp', 'dnsmasq', 'github-cli', 'sound-theme-freedesktop')+# wtype, python-pywayland and wlr-protocols: the desktop MCP's keys and mouse (os/desktop_mcp.py).+DEPENDENCIES = ('nodejs-lts-jod', 'tmux', 'foot', 'gtklock', 'grim', 'slurp', 'dnsmasq', 'github-cli', 'sound-theme-freedesktop',+ 'wtype', 'python-pywayland', 'wlr-protocols')def package_info(version, timestamp, size, compositor_dependencies=()):@@ -83,6 +85,7 @@ def stage(source, runtime, destination, commit):'os/hardware.py': 'usr/lib/harness-os/hardware.py','os/gpu_health.py': 'usr/lib/harness-os/gpu_health.py','os/gpu_probe.py': 'usr/lib/harness-os/gpu_probe.py',+ 'os/desktop_mcp.py': 'usr/lib/harness-os/desktop_mcp.py','os/tools/hn-os': 'usr/bin/hn-os','os/lock.json': 'usr/share/harness-os/lock.json','tui/README.md': 'usr/share/harness-os/guide/tui.md',@@ -103,6 +106,12 @@ def stage(source, runtime, destination, commit):shutil.copyfile(runtime / local, library / target)(library / target).chmod(0o755 if local == 'harness-tui' else 0o644)(library / 'hn').symlink_to('harness-tui')+ # Playwright's MCP server (build-runtime.sh), for `harness computer-use`. Without npm's .bin links: a link out of+ # its directory is refused by the updater, and the CLI runs the server's cli.js with node.+ playwright = runtime / 'playwright-mcp'+ if playwright.is_dir():+ shutil.copytree(playwright, destination / 'usr/lib/harness-os/playwright-mcp',+ ignore=shutil.ignore_patterns('.bin', '.package-lock.json'))for directory in ['usr/bin', 'usr/lib/harness-os']:for path in (destination / directory).rglob('*'):if path.is_file() and not path.is_symlink():diff --git a/os/tools/build-runtime.sh b/os/tools/build-runtime.shindex e10cd89ef..e35cad265 100644--- a/os/tools/build-runtime.sh+++ b/os/tools/build-runtime.sh@@ -51,7 +51,16 @@ data['files'] = {path.name: {'sha256': hashlib.sha256(path.read_bytes()).hexdigefor path in sorted(folder.iterdir())}(folder / 'source.json').write_text(json.dumps(data, indent=2) + '\n')PY+# Playwright's MCP server, from its lock and without install scripts, for `harness computer-use` (cli+# agentClient/computerUse.ts): in the package only, never in a fast runtime update.+playwright_stage=$(mktemp -d "${TMPDIR:-/tmp}/harness-playwright-mcp.XXXXXX")+trap 'rm -rf -- "$runtime_stage" "$playwright_stage"' EXIT+cp os/packaging/playwright-mcp/package.json os/packaging/playwright-mcp/package-lock.json "$playwright_stage/"+npm ci --prefix "$playwright_stage" --omit=dev --ignore-scripts --no-audit --no-fund+node "$playwright_stage/node_modules/@playwright/mcp/cli.js" --version >/dev/nullmkdir -p -- "$(dirname -- "$runtime_output")"mkdir -- "$runtime_output"cp "$runtime_stage/harness-tui" "$runtime_stage/cli.js" "$runtime_stage/notify.mjs" "$runtime_output/"cp "$runtime_stage/source.json" "$runtime_output/"+mkdir -- "$runtime_output/playwright-mcp"+cp -R "$playwright_stage/package.json" "$playwright_stage/node_modules" "$runtime_output/playwright-mcp/"