Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(os): computer use for agents on Harness OS (Playwright and the desktop), off until turned on (#1959)

Bubu-Dudu-52hzbb83530d3057
diff --git a/cli/src/agentClient/computerUse.spec.ts b/cli/src/agentClient/computerUse.spec.ts
new file mode 100644
index 000000000..22fbc6b28
--- /dev/null
+++ b/cli/src/agentClient/computerUse.spec.ts
@@ -0,0 +1,130 @@
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { machineMcpArgs, readMachineMcp } from '../lib/machineMcp.js'
+import { applyOpencode, CHROMIUM, DESKTOP_MCP, PLAYWRIGHT_MCP, setComputerUse, computerUseAvailable, computerUseCommand, computerUseServers, computerUseOn } from './computerUse.js'
+
+describe('harness computer-use', () => {
+ let dataDir: string
+ const lines: { out: string[]; err: string[] } = { out: [], err: [] }
+ const onOs = () => true
+ const noOpencode = { path: '/nowhere/opencode.json', major: () => null, reload: () => {} }
+ const run = (argv: string[], over: Parameters<typeof computerUseCommand>[1] = {}) => computerUseCommand(argv, {
+ dataDir, home: '/home/me', exists: onOs, env: {}, inPane: false, opencode: noOpencode,
+ out: (line) => lines.out.push(line), err: (line) => lines.err.push(line), ...over,
+ })
+
+ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'computer-use-')); lines.out = []; lines.err = [] })
+ afterEach(() => rmSync(dataDir, { recursive: true, force: true }))
+
+ it('is off until the person turns it on', () => {
+ expect(run([])).toBe(0)
+ expect(lines.out.join('\n')).toContain('Computer use is off')
+ expect(computerUseOn(readMachineMcp(dataDir).servers)).toBe(false)
+ })
+
+ it('attaches Playwright on the OS\'s Chromium and the desktop to every agent, and takes them off again', () => {
+ expect(run(['on'])).toBe(0)
+ const { servers } = readMachineMcp(dataDir)
+ expect(servers.playwright).toEqual({
+ command: '/usr/bin/node',
+ args: [PLAYWRIGHT_MCP, '--executable-path', CHROMIUM, '--isolated', '--output-dir', '/home/me/.cache/harness-os/playwright-mcp'],
+ engines: [], projects: [],
+ })
+ expect(servers.desktop).toEqual({ command: '/usr/bin/python3', args: [DESKTOP_MCP], engines: [], projects: [] })
+ expect(lines.out.at(-1)).toContain('Restart a running harness')
+ // What a Claude Code launch then carries.
+ expect(machineMcpArgs('claude', servers).join(' ')).toContain(PLAYWRIGHT_MCP)
+ expect(run(['off'])).toBe(0)
+ expect(readMachineMcp(dataDir).servers).toEqual({})
+ })
+
+ it('keeps the person\'s own servers when it turns on and off', () => {
+ writeFileSync(join(dataDir, 'mcp-servers.json'), JSON.stringify({ servers: { lamp: { command: 'lamp-mcp' } } }), { mode: 0o600 })
+ setComputerUse(true, dataDir, '/home/me', noOpencode)
+ expect(Object.keys(readMachineMcp(dataDir).servers).sort()).toEqual(['desktop', 'lamp', 'playwright'])
+ setComputerUse(false, dataDir, '/home/me', noOpencode)
+ expect(Object.keys(readMachineMcp(dataDir).servers)).toEqual(['lamp'])
+ })
+
+ it('refuses an agent, and one in a Harness pane, and changes nothing', () => {
+ expect(run(['on'], { env: { CLAUDECODE: '1' } })).toBe(5)
+ expect(run(['on'], { inPane: true })).toBe(5)
+ expect(lines.err.join('\n')).toContain('phone')
+ expect(readMachineMcp(dataDir).servers).toEqual({})
+ // Asking how they are is anyone's.
+ expect(run([], { env: { CLAUDECODE: '1' } })).toBe(0)
+ })
+
+ it('says a computer without them has none', () => {
+ expect(run(['on'], { exists: () => false })).toBe(1)
+ expect(lines.err.join('\n')).toContain('Harness OS')
+ expect(computerUseAvailable((path) => path !== CHROMIUM)).toBe(false)
+ })
+
+ it('does not write over a file it cannot read', () => {
+ writeFileSync(join(dataDir, 'mcp-servers.json'), 'not json', { mode: 0o600 })
+ expect(run(['on'])).toBe(1)
+ expect(readFileSync(join(dataDir, 'mcp-servers.json'), 'utf8')).toBe('not json')
+ })
+
+ describe('OpenCode, through its own config', () => {
+ const opencode = (major: number | null) => {
+ const reload = vi.fn()
+ return { path: join(dataDir, 'opencode', 'opencode.json'), major: () => major, reload }
+ }
+ const read = (path: string) => JSON.parse(readFileSync(path, 'utf8'))
+
+ it('puts them under mcp.servers for OpenCode 2, as `opencode mcp add` does, reloads it, and takes them out', () => {
+ const oc = opencode(2)
+ mkdirSync(join(dataDir, 'opencode'))
+ writeFileSync(oc.path, JSON.stringify({ model: 'opencode/free', mcp: { servers: { mine: { type: 'local', command: ['mine'] } } } }))
+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true })
+ const servers = read(oc.path).mcp.servers
+ expect(servers.desktop).toEqual({ type: 'local', command: ['/usr/bin/python3', DESKTOP_MCP] })
+ expect(servers.playwright.command.slice(0, 2)).toEqual(['/usr/bin/node', PLAYWRIGHT_MCP])
+ expect(oc.reload).toHaveBeenCalledTimes(1)
+ setComputerUse(false, dataDir, '/home/me', oc)
+ expect(read(oc.path)).toEqual({ model: 'opencode/free', mcp: { servers: { mine: { type: 'local', command: ['mine'] } } } })
+ expect(oc.reload).toHaveBeenCalledTimes(2)
+ })
+
+ it('puts them directly under mcp, enabled, for OpenCode 1, and leaves no empty mcp behind', () => {
+ const oc = opencode(1)
+ setComputerUse(true, dataDir, '/home/me', oc)
+ expect(read(oc.path).mcp.desktop).toEqual({ type: 'local', command: ['/usr/bin/python3', DESKTOP_MCP], enabled: true })
+ setComputerUse(false, dataDir, '/home/me', oc)
+ expect(read(oc.path)).toEqual({})
+ })
+
+ it('keeps a server of the person\'s own under the same name, and says so', () => {
+ const oc = opencode(2)
+ mkdirSync(join(dataDir, 'opencode'))
+ writeFileSync(oc.path, JSON.stringify({ mcp: { servers: { playwright: { type: 'local', command: ['npx', '@playwright/mcp'] } } } }))
+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true, note: expect.stringContaining('its own playwright') })
+ expect(read(oc.path).mcp.servers.playwright.command).toEqual(['npx', '@playwright/mcp'])
+ setComputerUse(false, dataDir, '/home/me', oc)
+ expect(read(oc.path).mcp.servers).toEqual({ playwright: { type: 'local', command: ['npx', '@playwright/mcp'] } })
+ })
+
+ it('leaves a config that is not plain JSON, and one OpenCode is not installed for, alone', () => {
+ const oc = opencode(2)
+ mkdirSync(join(dataDir, 'opencode'))
+ writeFileSync(oc.path, '{ // mine\n}')
+ expect(setComputerUse(true, dataDir, '/home/me', oc)).toEqual({ on: true, note: expect.stringContaining('not plain JSON') })
+ expect(readFileSync(oc.path, 'utf8')).toBe('{ // mine\n}')
+ expect(oc.reload).not.toHaveBeenCalled()
+ const absent = opencode(null)
+ expect(applyOpencode(true, absent, '/home/me')).toBeNull()
+ expect(existsSync(absent.path) && readFileSync(absent.path, 'utf8')).toContain('mine')
+ })
+ })
+
+ it('answers usage for anything else', () => {
+ expect(run(['maybe'])).toBe(2)
+ expect(run(['on', 'now'])).toBe(2)
+ expect(run(['--help'])).toBe(0)
+ expect(Object.keys(computerUseServers())).toEqual(['playwright', 'desktop'])
+ })
+})
diff --git a/cli/src/agentClient/computerUse.ts b/cli/src/agentClient/computerUse.ts
new file mode 100644
index 000000000..07032cd22
--- /dev/null
+++ b/cli/src/agentClient/computerUse.ts
@@ -0,0 +1,211 @@
+/**
+ * `harness computer-use`: Harness OS's tools for an agent to try what it builds as a person would, off until the person
+ * turns them on. Two machine MCP servers (lib/machineMcp.ts, #1872), both bundled in the OS:
+ *
+ * - `playwright`: Playwright's MCP server (os/packaging/playwright-mcp) driving the OS's Chromium, for a website: it
+ * opens pages, clicks, types and reads them and their console. A window of its own, on the screen.
+ * - `desktop`: the desktop (os/desktop_mcp.py), for a desktop app: screenshots, and the mouse and keys on them.
+ *
+ * harness computer-use on or off
+ * harness computer-use on attached to every agent started from now on (Claude Code, Codex, OpenCode)
+ * harness computer-use off taken off again
+ *
+ * Turning them on or off writes this machine's mcp-servers.json, read at each Claude Code and Codex start, and
+ * OpenCode's own config (below): a harness running already gets them when it restarts. The person's phone does the same (services/osDevice.ts `computer_use`). An agent may
+ * not: a server runs inside every later agent.
+ */
+import { execFile, execFileSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { env } from '../config/env.js'
+import { MACHINE_MCP_MOST_SERVERS, readMachineMcp, writeMachineMcp, type MachineMcpServer, type MachineMcpServers } from '../lib/machineMcp.js'
+import { outsideCaller } from './model.js'
+
+export const PLAYWRIGHT_MCP = '/usr/lib/harness-os/playwright-mcp/node_modules/@playwright/mcp/cli.js'
+export const DESKTOP_MCP = '/usr/lib/harness-os/desktop_mcp.py'
+export const CHROMIUM = '/usr/bin/chromium'
+const NODE = '/usr/bin/node'
+const PYTHON = '/usr/bin/python3'
+/** The machine MCP servers' names: the agents' tools are `mcp__playwright__…` and `mcp__desktop__…`. */
+export const COMPUTER_USE = ['playwright', 'desktop'] as const
+
+/** A refusal, as the agent verbs exit (commands.ts `EXIT.refused`): not a failure to retry. */
+const REFUSED = 5
+
+export const COMPUTER_USE_USAGE = `Usage: harness computer-use [on|off]
+
+Tools for the agents Harness starts on this Harness OS computer to try what they build as a person would, off until
+you turn them on:
+ playwright a website, in a Chromium window of its own: open it, click, type, read it and its console
+ desktop a desktop app: screenshots of the screen, and the mouse and keyboard on them
+On, Claude Code and Codex agents Harness starts from now on get them, and OpenCode (its own config); a harness
+running already gets them when you restart it. Each agent asks you before it uses them. Only you turn them on or off:
+an agent is refused.`
+
+/** The servers' entries; [home] places Playwright's screenshots and snapshots out of the project's folder. */
+export function computerUseServers(home: string = homedir()): Record<(typeof COMPUTER_USE)[number], MachineMcpServer> {
+ return {
+ playwright: {
+ command: NODE,
+ args: [PLAYWRIGHT_MCP, '--executable-path', CHROMIUM, '--isolated', '--output-dir', join(home, '.cache', 'harness-os', 'playwright-mcp')],
+ engines: [],
+ projects: [],
+ },
+ desktop: { command: PYTHON, args: [DESKTOP_MCP], engines: [], projects: [] },
+ }
+}
+
+/** Whether this computer carries them: Harness OS from the release that bundles them. */
+export const computerUseAvailable = (exists: (path: string) => boolean = existsSync): boolean =>
+ [PLAYWRIGHT_MCP, DESKTOP_MCP, CHROMIUM].every((path) => exists(path))
+
+/** On: both attached. */
+export const computerUseOn = (servers: MachineMcpServers): boolean => COMPUTER_USE.every((name) => !!servers[name])
+
+/**
+ * OpenCode, whose launch takes no MCP server: its own global config, as Connections writes it
+ * (lib/connectors/agents.ts), read by its background service on `opencode reload`. OpenCode 2 (Harness OS has
+ * 2.0.21) keeps them under `mcp.servers`, a local one as `{ type: 'local', command }`, as `opencode mcp add --global`
+ * writes them; OpenCode 1 directly under `mcp`, with `enabled`. Only an entry running one of these servers is
+ * Harness's to change: one of the person's under the same name stays.
+ */
+export interface Opencode {
+ /** Its global opencode.json. */
+ path: string
+ /** Its major version, or null when it is not installed. */
+ major: () => number | null
+ /** Asks its service to read the config again. */
+ reload: () => void
+}
+
+export function machineOpencode(environment: NodeJS.ProcessEnv = process.env, home: string = homedir()): Opencode {
+ return {
+ path: join(environment.XDG_CONFIG_HOME || join(home, '.config'), 'opencode', 'opencode.json'),
+ major: () => {
+ try {
+ const out = execFileSync('opencode', ['--version'], { encoding: 'utf8', timeout: 15_000, stdio: ['ignore', 'pipe', 'ignore'] })
+ const major = /(\d+)\.\d+\.\d+/.exec(out)?.[1]
+ return major ? Number(major) : null
+ } catch { return null }
+ },
+ reload: () => {
+ // Its service picks the change up now; a failure only leaves it to the next start.
+ execFile('opencode', ['reload'], { timeout: 30_000 }, () => {})
+ },
+ }
+}
+
+const ours = (entry: unknown): boolean => {
+ const command = (entry as { command?: unknown } | null)?.command
+ return Array.isArray(command) && command.some((part) => part === PLAYWRIGHT_MCP || part === DESKTOP_MCP)
+}
+
+/** Put the computer-use tools into OpenCode's config, or take them out: what it could not do, in words, or null. */
+export function applyOpencode(on: boolean, opencode: Opencode, home: string = homedir()): string | null {
+ const major = opencode.major()
+ if (major === null) return null
+ let data: Record<string, unknown> = {}
+ if (existsSync(opencode.path)) {
+ try {
+ const parsed: unknown = JSON.parse(readFileSync(opencode.path, 'utf8'))
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object')
+ data = parsed as Record<string, unknown>
+ } catch {
+ return `OpenCode's ${opencode.path} is not plain JSON, so it was left as it is: OpenCode does not get them.`
+ }
+ } else if (!on) return null
+ const record = (value: unknown): Record<string, unknown> =>
+ value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
+ const mcp = record(data.mcp)
+ const servers = major >= 2 ? record(mcp.servers) : mcp
+ const skipped: string[] = []
+ for (const [name, server] of Object.entries(computerUseServers(home))) {
+ if (servers[name] !== undefined && !ours(servers[name])) { if (on) skipped.push(name); continue }
+ if (!on) { delete servers[name]; continue }
+ const command = [server.command, ...server.args]
+ servers[name] = major >= 2 ? { type: 'local', command } : { type: 'local', command, enabled: true }
+ }
+ if (major >= 2) {
+ if (Object.keys(servers).length) mcp.servers = servers
+ else delete mcp.servers
+ }
+ if (Object.keys(mcp).length) data.mcp = mcp
+ else delete data.mcp
+ mkdirSync(dirname(opencode.path), { recursive: true })
+ const temporary = `${opencode.path}.${process.pid}.tmp`
+ writeFileSync(temporary, JSON.stringify(data, null, 2) + '\n')
+ renameSync(temporary, opencode.path)
+ opencode.reload()
+ return skipped.length ? `OpenCode already has its own ${skipped.join(' and ')}, so it was left as it is.` : null
+}
+
+/** Turn them on or off in this machine's mcp-servers.json (Claude Code, Codex) and OpenCode's config: whether they
+ * are on afterwards and what OpenCode could not take, or why it could not. */
+export function setComputerUse(
+ on: boolean, dataDir: string = env.ADAPTER_DATA_DIR, home: string = homedir(), opencode: Opencode = machineOpencode(process.env, home),
+): { on: boolean; note?: string } | { error: string } {
+ const { servers, why } = readMachineMcp(dataDir)
+ if (why) return { error: `Fix or remove it first: ${why}` }
+ if (on) {
+ const added = COMPUTER_USE.filter((name) => !servers[name]).length
+ if (Object.keys(servers).length + added > MACHINE_MCP_MOST_SERVERS) {
+ return { error: `At most ${MACHINE_MCP_MOST_SERVERS} machine MCP servers: remove one with harness mcp-servers remove first.` }
+ }
+ Object.assign(servers, computerUseServers(home))
+ } else {
+ for (const name of COMPUTER_USE) delete servers[name]
+ }
+ writeMachineMcp(servers, dataDir)
+ const note = applyOpencode(on, opencode, home)
+ return note ? { on, note } : { on }
+}
+
+export interface ComputerUseDeps {
+ dataDir?: string
+ home?: string
+ exists?: (path: string) => boolean
+ opencode?: Opencode
+ /** Who runs it, by what its shell says (model.ts `outsideCaller`): an agent may ask, never change. */
+ env?: NodeJS.ProcessEnv
+ /** Whether it runs in a pane a Harness daemon made (cli.ts, as for `harness mcp-servers`): refused alike. */
+ inPane?: boolean
+ out?: (line: string) => void
+ err?: (line: string) => void
+}
+
+export function computerUseCommand(argv: readonly string[], deps: ComputerUseDeps = {}): number {
+ const out = deps.out ?? console.log
+ const err = deps.err ?? console.error
+ const dataDir = deps.dataDir ?? env.ADAPTER_DATA_DIR
+ if (argv.includes('--help') || argv.includes('-h')) { out(COMPUTER_USE_USAGE); return 0 }
+ const [verb = 'status', ...rest] = argv
+ if (rest.length || !['status', 'on', 'off'].includes(verb)) { err(COMPUTER_USE_USAGE); return 2 }
+ if (!computerUseAvailable(deps.exists)) {
+ err('Computer use comes with Harness OS (Playwright and the desktop MCP): this computer has no such tools.')
+ return 1
+ }
+ if (verb === 'status') {
+ const { servers, why } = readMachineMcp(dataDir)
+ if (why) { err(`Not read, so no agent gets them: ${why}`); return 1 }
+ out(computerUseOn(servers)
+ ? 'Computer use is on: agents Harness starts get playwright and desktop. harness computer-use off takes them off.'
+ : 'Computer use is off. harness computer-use on gives the agents Harness starts playwright and desktop.')
+ return 0
+ }
+ const agent = outsideCaller(deps.env ?? process.env, '')
+ if (agent || deps.inPane) {
+ err(`${agent ? agent.name : 'An agent in a Harness pane'} may not turn computer use on or off: its tools run inside every `
+ + 'later agent here. Ask the person to turn them on from their phone (the computer\'s Computer use), or to run '
+ + `harness computer-use ${verb} in a terminal outside Harness.`)
+ return REFUSED
+ }
+ const home = deps.home ?? homedir()
+ const result = setComputerUse(verb === 'on', dataDir, home, deps.opencode ?? machineOpencode(process.env, home))
+ if ('error' in result) { err(result.error); return 1 }
+ out(result.on
+ ? 'Computer use on: agents started from now on get playwright and desktop. Restart a running harness to give it them.'
+ : 'Computer use off: agents started from now on don\'t get them. Restart a running harness to take them away.')
+ if (result.note) err(result.note)
+ return 0
+}
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 3109791cd..cd2e5a6de 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -181,6 +181,7 @@ Machine:
harness help agents the agent commands, states, exit codes and rules on one screen
harness settings [<name> on|off] what agents may do (agents-start-agents, agents-tell-agents, pull-request-line, agent-messages): your account's experiments in Settings ▸ Experimental, each off by default
harness mcp-servers [add|remove] local MCP servers (a paired lamp's lamp-mcp) for the agents Harness starts here: yours alone to change
+ harness computer-use [on|off] Harness OS: browser (Playwright) and desktop tools for agents to try what they build; off until you turn them on
harness machines delete <id> remove ANOTHER machine (refuses this one; use \`harness logout\`)
harness machines engines <machine> [--engine E]… [--json]
ask another of your machines now which engines it has: installed, signed in, first prompt
@@ -2718,6 +2719,7 @@ import { openUrlCommand } from './agentClient/openUrl.js'
import { harnessRunner, mcpCommand } from './agentClient/mcp.js'
import { settingsCommand } from './agentClient/settings.js'
import { mcpServersCommand } from './agentClient/mcpServers.js'
+import { computerUseCommand } from './agentClient/computerUse.js'
// ── arg parse ──────────────────────────────────────────────────────────────────────────────────
// `hn` is the terminal client's short name (like tmux, fzf): the same CLI, entered at `tui`. A call
@@ -2779,6 +2781,12 @@ switch (cmd) {
process.exitCode = mcpServersCommand(rest, { inPane: !!pane && (!pane.startsWith('%') || !!paneOwner(pane)) })
break
}
+ case 'computer-use': {
+ // Machine MCP servers too, refused in a Harness pane alike.
+ const pane = sendPane(process.env)
+ process.exitCode = computerUseCommand(rest, { inPane: !!pane && (!pane.startsWith('%') || !!paneOwner(pane)) })
+ break
+ }
case 'mcp': {
const pane = process.env.TMUX_PANE
mcpCommand({ run: harnessRunner(SCRIPT_PATH), inPane: !!pane && !!paneOwner(pane) }).then(code => { process.exitCode = code }).catch(onError)
diff --git a/cli/src/services/osDevice.spec.ts b/cli/src/services/osDevice.spec.ts
index 67bd50543..7b31476c5 100644
--- a/cli/src/services/osDevice.spec.ts
+++ b/cli/src/services/osDevice.spec.ts
@@ -191,6 +191,41 @@ describe('the Harness OS computer, managed from the phone', () => {
expect(await ask({ action: 'update' })).toEqual({ started: true })
})
+ describe('computer use, turned on and off from the phone', () => {
+ function tools(available = true) {
+ const state = { on: false }
+ const computerUse = {
+ available: () => available,
+ on: () => state.on,
+ set: vi.fn((on: boolean) => { state.on = on; return { on } }),
+ }
+ return { ...service({ computerUse }), computerUse }
+ }
+
+ it('names them among its features only on a computer that carries them', async () => {
+ expect(await tools().ask({ action: 'status' })).toMatchObject({ features: expect.arrayContaining(['computer_use']) })
+ expect((await tools(false).ask({ action: 'status' }) as { features: string[] }).features).not.toContain('computer_use')
+ })
+
+ it('says how they are, and turns them on and off', async () => {
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ const { ask, computerUse } = tools()
+ expect(await ask({ action: 'computer_use' })).toEqual({ on: false })
+ expect(await ask({ action: 'computer_use', state: 'on' })).toEqual({ on: true })
+ expect(await ask({ action: 'computer_use' })).toEqual({ on: true })
+ expect(await ask({ action: 'computer_use', state: 'off' })).toEqual({ on: false })
+ expect(computerUse.set.mock.calls).toEqual([[true], [false]])
+ })
+
+ it('answers why it could not, and nothing on a computer without them', async () => {
+ const { ask, computerUse } = tools()
+ computerUse.set.mockReturnValueOnce({ error: 'Fix or remove it first: not JSON' } as never)
+ expect(await ask({ action: 'computer_use', state: 'on' })).toEqual({ error: 'FAILED', message: 'Fix or remove it first: not JSON' })
+ expect(await tools(false).ask({ action: 'computer_use', state: 'on' })).toEqual({ error: 'UNSUPPORTED_OS' })
+ expect(await tools().ask({ action: 'computer_use', state: 'on' }, { local: false, owner: false })).toEqual({ error: 'OWNER_REQUIRED' })
+ })
+ })
+
describe('checking for updates now, from the phone', () => {
/** A computer whose check is held until [done] settles it. */
function holding() {
diff --git a/cli/src/services/osDevice.ts b/cli/src/services/osDevice.ts
index db0ccb8c1..99b629933 100644
--- a/cli/src/services/osDevice.ts
+++ b/cli/src/services/osDevice.ts
@@ -15,6 +15,8 @@ import { existsSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { basename } from 'node:path'
import type { CoreApi, ServiceRequests } from '../core/api.js'
+import { setComputerUse, computerUseAvailable, computerUseOn } from '../agentClient/computerUse.js'
+import { readMachineMcp } from '../lib/machineMcp.js'
/** The requests this service answers for the apps, declared in core/api.ts for the core to route. */
export { OS_DEVICE_REQUESTS } from '../core/api.js'
@@ -49,7 +51,7 @@ export const GH_TIMEOUT_MS = 15_000
export const GH_INSTALL_TIMEOUT_MS = 300_000
/** How long a GitHub sign-in waits for its code to be entered: GitHub's device codes last 15 minutes. */
export const GITHUB_SIGN_IN_MS = 15 * 60_000
-const ACTIONS = ['update', 'check_update', 'wifi_scan','wifi_join', 'wifi_result', 'sign_out', 'password', 'ssh', 'github']
+const ACTIONS = ['update', 'check_update', 'wifi_scan','wifi_join', 'wifi_result', 'sign_out', 'password', 'ssh', 'github', 'computer_use']
/** The most output a command run here may print before it is stopped. */
export const MAX_OUTPUT = 1024 * 1024
@@ -67,9 +69,24 @@ export interface OsDeviceDeps {
spawnDetached?: (file: string, args: readonly string[], input?: string) => void
/** Start `gh auth login`'s device flow (a code to enter at github.com/login/device). */
githubSignIn?: () => GithubSignIn
+ /** Computer use (agentClient/computerUse.ts): whether this computer has them, whether they are on, turning them. */
+ computerUse?: ComputerUse
now?: () => number
}
+export interface ComputerUse {
+ available: () => boolean
+ on: () => boolean
+ set: (on: boolean) => { on: boolean; note?: string } | { error: string }
+}
+
+/** This machine's mcp-servers.json, where `harness computer-use` turns them on and off too. */
+const machineComputerUse: ComputerUse = {
+ available: () => computerUseAvailable(),
+ on: () => computerUseOn(readMachineMcp().servers),
+ set: (on) => setComputerUse(on),
+}
+
/** A `gh auth login` under way: the code it shows, and it ending once the code was entered (or refused). */
export interface GithubSignIn {
code: Promise<{ code: string; url: string }>
@@ -239,6 +256,7 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceR
const detached = deps.spawnDetached ?? spawnDetached
const startSignIn = deps.githubSignIn ?? githubDeviceSignIn
const now = deps.now ?? Date.now
+ const computerUse = deps.computerUse ?? machineComputerUse
let updateStartedAt: number | null = null
/** The GitHub sign-in waiting for its code, and why the last one ended without signing in. */
let signingIn: { run: GithubSignIn; code: string; url: string } | null = null
@@ -275,10 +293,20 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceR
// features: what this daemon does for a phone beyond the update and the Wi-Fi.
return {
harnessOs: true, update, wifi: { ssid }, hotspot, encrypted: /(^|\s)rd\.luks\.name=/.test(cmdline),
- features: ['sign_out', 'password', 'ssh', 'github', 'check_update'],
+ // computer_use on a computer that carries them (Harness OS from the release that bundles them).
+ features: ['sign_out', 'password', 'ssh', 'github', 'check_update', ...(computerUse.available() ? ['computer_use'] : [])],
}
}
+ /** Computer use, on, off or how they are: `{ on }`, for the phone to say a running harness gets them on restart. */
+ const setTools = (payload: Payload): Payload => {
+ if (!computerUse.available()) return { error: 'UNSUPPORTED_OS' }
+ if (payload.state !== 'on' && payload.state !== 'off') return { on: computerUse.on() }
+ console.log(`[os-device] computer use ${payload.state}`)
+ const done = computerUse.set(payload.state === 'on')
+ return 'error' in done ? { error: 'FAILED', message: done.error } : done
+ }
+
/** Check for updates now, as the timer does every 15 minutes, the OS release too: answered at once,
* `update.checking` in the status until it is done, for the phone to follow. One at a time. */
const checkUpdate = (): Payload => {
@@ -449,6 +477,7 @@ export function startOsDevice(_core: CoreApi, deps: OsDeviceDeps = {}): ServiceR
try {
if (action === 'update') return update(payload)
if (action === 'check_update') return checkUpdate()
+ if (action === 'computer_use') return setTools(payload)
if (action === 'wifi_scan') return await scan()
if (action === 'wifi_join') return join(payload)
if (action === 'sign_out') return signOut()
diff --git a/os/desktop_mcp.py b/os/desktop_mcp.py
new file mode 100755
index 000000000..2539fb2fb
--- /dev/null
+++ b/os/desktop_mcp.py
@@ -0,0 +1,448 @@
+#!/usr/bin/python3
+"""Harness OS's desktop as tools for an agent: what is on the screen, and the mouse and keyboard on it.
+
+A stdio MCP server (JSON-RPC 2.0, a message a line) for an agent to try the desktop app it builds as a person
+would: `screenshot`, then `click`, `drag`, `scroll`, `type` and `key` where it sees things, each answered with
+the screen as it is afterwards. Off until the person turns it on, with Playwright's, from their phone or with
+`harness computer-use on` (cli agentClient/computerUse.ts), never by an agent.
+
+Input goes through the compositor's own virtual devices (labwc: wlr-virtual-pointer for the mouse,
+virtual-keyboard for the keys), so it stays inside this session: no /dev/uinput. While the screen is locked
+it refuses everything, the screenshot too.
+
+The keyboard is one of its own for as long as the server runs, with the compositor's default keymap: keys reach
+an app as a real keyboard's do. wtype, a keyboard with a keymap of just the characters it types, is used only
+for text that keymap cannot type (accented letters): GTK 3 ignores wtype's Return, BackSpace and shortcuts.
+
+Coordinates are the screenshot's pixels. A screen wider than WIDTH is shown scaled down to it, and a point on the
+screenshot is scaled back up to the screen.
+"""
+import base64
+import ctypes
+import importlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+
+NAME = 'desktop'
+VERSION = '0.1.0'
+PROTOCOL = '2025-06-18'
+# The widest screenshot an agent is shown: wide enough to read a window's text, small enough to send each turn.
+WIDTH = 1280
+# How long the screen gets to change before the screenshot that answers an action.
+SETTLE = 0.4
+# A wtype is a keyboard of its own, its keymap sent to the app in front as it starts: keys from another
+# keyboard right after it were dropped (labwc, Chromium, 0.4 s apart).
+KEY_GAP = 1.0
+POINTER_XML = Path('/usr/share/wlr-protocols/unstable/wlr-virtual-pointer-unstable-v1.xml')
+# Not in a package of Arch's: Harness OS's own copy of wlroots' (MIT).
+KEYBOARD_XML = Path('/usr/share/harness-os/virtual-keyboard-unstable-v1.xml')
+CACHE = Path(os.environ.get('XDG_CACHE_HOME') or Path.home() / '.cache') / 'harness-os' / 'desktop-mcp'
+# Linux input event codes (linux/input-event-codes.h).
+BUTTONS = {'left': 0x110, 'right': 0x111, 'middle': 0x112}
+# Friendly key names to XKB keysym names; anything else is looked up as it is (`F5`, `a`, `Page_Up`).
+KEYS = {'enter': 'Return', 'return': 'Return', 'esc': 'Escape', 'escape': 'Escape', 'tab': 'Tab',
+ 'backspace': 'BackSpace', 'delete': 'Delete', 'del': 'Delete', 'space': 'space', 'up': 'Up',
+ 'down': 'Down', 'left': 'Left', 'right': 'Right', 'home': 'Home', 'end': 'End',
+ 'pageup': 'Page_Up', 'pagedown': 'Page_Down', 'insert': 'Insert',
+ # A modifier pressed on its own.
+ 'shift': 'Shift_L', 'ctrl': 'Control_L', 'control': 'Control_L', 'alt': 'Alt_L', 'super': 'Super_L'}
+MODIFIERS = {'ctrl': 'Control_L', 'control': 'Control_L', 'shift': 'Shift_L', 'alt': 'Alt_L',
+ 'altgr': 'ISO_Level3_Shift', 'super': 'Super_L', 'logo': 'Super_L', 'win': 'Super_L',
+ 'meta': 'Super_L', 'cmd': 'Super_L'}
+# Each modifier key's XKB modifier: the virtual keyboard's modifiers are said, not worked out from its keys
+# (on labwc, Shift and Control held by key alone changed nothing).
+MODIFIER_NAMES = {'Shift_L': 'Shift', 'Control_L': 'Control', 'Alt_L': 'Mod1', 'Super_L': 'Mod4',
+ 'ISO_Level3_Shift': 'Mod5'}
+
+
+class Refused(Exception):
+ """An action this server will not take, said to the agent as a tool error."""
+
+
+def environment():
+ """The session's display, for a server started from an SSH pane or a service with none of its own."""
+ env = dict(os.environ)
+ runtime = env.setdefault('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}')
+ if not env.get('WAYLAND_DISPLAY'):
+ sockets = sorted(p.name for p in Path(runtime).glob('wayland-*') if not p.name.endswith('.lock'))
+ if not sockets:
+ raise Refused('No desktop is running on this computer.')
+ env['WAYLAND_DISPLAY'] = sockets[0]
+ return env
+
+
+def locked():
+ return subprocess.run(['pgrep', '-x', 'gtklock'], capture_output=True).returncode == 0
+
+
+def png_size(data):
+ if data[:8] != b'\x89PNG\r\n\x1a\n':
+ raise Refused('The screenshot is not a PNG.')
+ return int.from_bytes(data[16:20], 'big'), int.from_bytes(data[20:24], 'big')
+
+
+def bindings(xml):
+ """pywayland's module for the protocol in [xml], made by its scanner once per pywayland version.
+
+ The scanner writes one file a protocol, importing the core one relatively (`from .wayland import WlSeat`):
+ so a package of its own, with a `wayland` that is pywayland's.
+ """
+ import pywayland
+ from pywayland.scanner import Protocol
+ package = 'harness_desktop_protocols'
+ folder = CACHE / pywayland.__version__
+ protocol = Protocol.parse_file(str(xml))
+ module = folder / package / f'{protocol.name}.py'
+ if not module.is_file():
+ imports = {interface.name: protocol.name for interface in protocol.interface}
+ imports.update({name: 'wayland' for name in ('wl_seat', 'wl_output', 'wl_surface')})
+ (folder / package).mkdir(parents=True, exist_ok=True)
+ (folder / package / '__init__.py').touch()
+ (folder / package / 'wayland.py').write_text('from pywayland.protocol.wayland import * # noqa: F403\n')
+ with tempfile.TemporaryDirectory(dir=folder / package) as temp:
+ protocol.output(temp, imports)
+ os.replace(Path(temp) / module.name, module)
+ if str(folder) not in sys.path:
+ sys.path.insert(0, str(folder))
+ return importlib.import_module(f'{package}.{protocol.name}')
+
+
+class Keymap:
+ """The compositor's default keymap (XKB's defaults, as labwc's), compiled by libxkbcommon: its text for the
+ virtual keyboard, and where each keysym is, as an evdev key and whether Shift reaches it."""
+
+ def __init__(self):
+ xkb = ctypes.CDLL('libxkbcommon.so.0')
+ xkb.xkb_context_new.restype = ctypes.c_void_p
+ xkb.xkb_keymap_new_from_names.restype = ctypes.c_void_p
+ xkb.xkb_keymap_new_from_names.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
+ xkb.xkb_keymap_get_as_string.restype = ctypes.c_void_p
+ xkb.xkb_keymap_get_as_string.argtypes = [ctypes.c_void_p, ctypes.c_int]
+ for name in ('xkb_keymap_min_keycode', 'xkb_keymap_max_keycode'):
+ getattr(xkb, name).restype = ctypes.c_uint32
+ getattr(xkb, name).argtypes = [ctypes.c_void_p]
+ xkb.xkb_keymap_key_get_syms_by_level.argtypes = [
+ ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.POINTER(ctypes.c_uint32))]
+ xkb.xkb_keysym_from_name.restype = ctypes.c_uint32
+ xkb.xkb_keysym_from_name.argtypes = [ctypes.c_char_p, ctypes.c_int]
+ xkb.xkb_utf32_to_keysym.restype = ctypes.c_uint32
+ xkb.xkb_utf32_to_keysym.argtypes = [ctypes.c_uint32]
+ xkb.xkb_keymap_mod_get_index.restype = ctypes.c_uint32
+ xkb.xkb_keymap_mod_get_index.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+ libc = ctypes.CDLL(None)
+ libc.free.argtypes = [ctypes.c_void_p]
+ self.xkb = xkb
+ keymap = xkb.xkb_keymap_new_from_names(xkb.xkb_context_new(0), None, 0)
+ if not keymap:
+ raise Refused('Could not make a keymap for the virtual keyboard.')
+ text = xkb.xkb_keymap_get_as_string(keymap, 1) # XKB_KEYMAP_FORMAT_TEXT_V1
+ self.text = ctypes.string_at(text) + b'\0'
+ libc.free(text)
+ self.where = {}
+ syms = ctypes.POINTER(ctypes.c_uint32)()
+ for keycode in range(xkb.xkb_keymap_min_keycode(keymap), xkb.xkb_keymap_max_keycode(keymap) + 1):
+ for level in (0, 1):
+ for i in range(xkb.xkb_keymap_key_get_syms_by_level(keymap, keycode, 0, level, ctypes.byref(syms))):
+ # The first, unshifted, place a keysym has wins: `1` is 2's key, not the keypad's.
+ self.where.setdefault(syms[i], (keycode - 8, level == 1))
+ # A modifier key's evdev code to its mask.
+ self.masks = {}
+ for sym_name, mod_name in MODIFIER_NAMES.items():
+ index = xkb.xkb_keymap_mod_get_index(keymap, mod_name.encode())
+ sym = xkb.xkb_keysym_from_name(sym_name.encode(), 0)
+ if index != 0xFFFFFFFF and sym in self.where:
+ self.masks[self.where[sym][0]] = 1 << index
+
+ def named(self, name):
+ sym = self.xkb.xkb_keysym_from_name(name.encode(), 0) or self.xkb.xkb_keysym_from_name(name.encode(), 1)
+ if not sym or sym not in self.where:
+ raise Refused(f'Not a key on this keyboard: {name!r}. Use names like "enter", "ctrl+s", "F5".')
+ return self.where[sym]
+
+ def character(self, char):
+ """Where [char] is, or None for one this keymap cannot type."""
+ if char == '\n':
+ return self.where.get(0xff0d) # Return
+ if char == '\t':
+ return self.where.get(0xff09) # Tab
+ return self.where.get(self.xkb.xkb_utf32_to_keysym(ord(char)))
+
+
+class Devices:
+ """This session's own mouse and keyboard, made by the compositor (one Wayland connection, kept)."""
+
+ def __init__(self, env):
+ from pywayland.client import Display
+ from pywayland.protocol.wayland import WlSeat
+ pointer, keyboard = bindings(POINTER_XML), bindings(KEYBOARD_XML)
+ self.display = Display(env['WAYLAND_DISPLAY'])
+ self.display.connect()
+ found = {}
+
+ def announced(registry, name, interface, version):
+ if interface == 'wl_seat' and 'seat' not in found:
+ found['seat'] = registry.bind(name, WlSeat, 1)
+ elif interface == 'zwlr_virtual_pointer_manager_v1':
+ found['pointers'] = registry.bind(name, pointer.ZwlrVirtualPointerManagerV1, 1)
+ elif interface == 'zwp_virtual_keyboard_manager_v1':
+ found['keyboards'] = registry.bind(name, keyboard.ZwpVirtualKeyboardManagerV1, 1)
+
+ registry = self.display.get_registry()
+ registry.dispatcher['global'] = announced
+ self.display.roundtrip()
+ missing = {'seat', 'pointers', 'keyboards'} - set(found)
+ if missing:
+ self.display.disconnect()
+ raise Refused('This desktop has no virtual mouse or keyboard (' + ', '.join(sorted(missing)) + ').')
+ self.pointer = found['pointers'].create_virtual_pointer(found['seat'])
+ self.keymap = Keymap()
+ self.keyboard = found['keyboards'].create_virtual_keyboard(found['seat'])
+ fd = os.memfd_create('harness-desktop-keymap')
+ os.write(fd, self.keymap.text)
+ self.keyboard.keymap(1, fd, len(self.keymap.text)) # WL_KEYBOARD_KEYMAP_FORMAT_XKB_V1
+ self.display.roundtrip()
+ os.close(fd)
+
+ @staticmethod
+ def now():
+ return int(time.monotonic() * 1000) & 0xFFFFFFFF
+
+ def frame(self):
+ self.pointer.frame()
+ self.display.roundtrip()
+
+ def move(self, x, y, size):
+ self.pointer.motion_absolute(self.now(), x, y, size[0], size[1])
+ self.frame()
+
+ def button(self, code, pressed):
+ self.pointer.button(self.now(), code, 1 if pressed else 0)
+ self.frame()
+
+ def scroll(self, steps):
+ # A wheel's notches: 15 units each, as a mouse wheel reports them. Positive scrolls down.
+ self.pointer.axis_source(0)
+ self.pointer.axis_discrete(self.now(), 0, 15.0 * steps, steps)
+ self.frame()
+
+ def key(self, code, pressed):
+ self.keyboard.key(self.now(), code, 1 if pressed else 0)
+ self.display.roundtrip()
+
+ def press(self, codes):
+ """Hold [codes] down in order, then let them go in reverse: a shortcut, or a key with Shift. The
+ modifiers among them are said as they go down and up."""
+ held = 0
+ for code in codes:
+ self.key(code, True)
+ if code in self.keymap.masks:
+ held |= self.keymap.masks[code]
+ self.keyboard.modifiers(held, 0, 0, 0)
+ for code in reversed(codes):
+ self.key(code, False)
+ if code in self.keymap.masks:
+ held &= ~self.keymap.masks[code]
+ self.keyboard.modifiers(held, 0, 0, 0)
+ self.display.roundtrip()
+ time.sleep(0.01)
+
+
+class Screen:
+ def __init__(self):
+ self.env = environment()
+ self.size = None
+ self.devices = None
+ self.wtyped = 0.0
+
+ def grab(self, scale=None):
+ args = ['grim', '-t', 'png'] + (['-s', f'{scale:.4f}'] if scale else []) + ['-']
+ done = subprocess.run(args, capture_output=True, env=self.env, timeout=15)
+ if done.returncode:
+ raise Refused('Could not take a screenshot: ' + done.stderr.decode(errors='replace').strip())
+ return done.stdout
+
+ def scale(self):
+ """The screenshot's size over the screen's: 1 for a screen no wider than WIDTH."""
+ if self.size is None:
+ self.size = png_size(self.grab())
+ return min(1.0, WIDTH / self.size[0])
+
+ def screenshot(self):
+ if locked():
+ raise Refused('The screen is locked. Ask the person to unlock it.')
+ scale = self.scale()
+ data = self.grab(scale if scale < 1 else None)
+ return data, png_size(data)
+
+ def point(self, x, y):
+ """A screenshot's pixel on the screen."""
+ scale = self.scale()
+ width, height = self.size
+ x, y = round(float(x) / scale), round(float(y) / scale)
+ if not (0 <= x < width and 0 <= y < height):
+ raise Refused(f'({x}, {y}) is outside the screen ({width}x{height} on the screen).')
+ return x, y
+
+ def input(self):
+ if self.devices is None:
+ self.devices = Devices(self.env)
+ return self.devices
+
+ def after_wtype(self):
+ time.sleep(max(0.0, self.wtyped + KEY_GAP - time.monotonic()))
+
+ def type(self, text):
+ devices = self.input()
+ places = [devices.keymap.character(char) for char in text]
+ if all(places):
+ self.after_wtype()
+ shift = devices.keymap.named('Shift_L')[0]
+ for code, shifted in places:
+ devices.press([shift, code] if shifted else [code])
+ return
+ # Accented letters and the like: wtype, with a keymap of its own made of them.
+ self.after_wtype()
+ done = subprocess.run(['wtype', '--', text], capture_output=True, env=self.env, timeout=60)
+ self.wtyped = time.monotonic()
+ if done.returncode:
+ raise Refused('Could not type: ' + done.stderr.decode(errors='replace').strip())
+
+ def key(self, keys):
+ """`ctrl+shift+t`: the modifiers held around the key, then let go."""
+ parts = [part.strip() for part in str(keys).split('+')]
+ if not parts or not all(parts):
+ raise Refused(f'Not a key: {keys!r}. Use names like "enter", "ctrl+s" or "super+b".')
+ devices = self.input()
+ *modifiers, key = parts
+ codes = []
+ for modifier in modifiers:
+ if modifier.lower() not in MODIFIERS:
+ raise Refused(f'Not a modifier: {modifier!r}. Use ctrl, shift, alt, altgr or super.')
+ codes.append(devices.keymap.named(MODIFIERS[modifier.lower()])[0])
+ code, shifted = devices.keymap.named(KEYS.get(key.lower(), key))
+ shift = devices.keymap.named('Shift_L')[0]
+ if shifted and shift not in codes:
+ codes.append(shift)
+ self.after_wtype()
+ devices.press(codes + [code])
+
+
+POINT = {'x': {'type': 'number', 'description': 'Pixels from the left of the screenshot.'},
+ 'y': {'type': 'number', 'description': 'Pixels from the top of the screenshot.'}}
+TOOLS = [
+ {'name': 'screenshot', 'description': 'What is on the screen now, as an image. Coordinates for the other '
+ 'tools are this image\'s pixels.', 'inputSchema': {'type': 'object', 'properties': {}}},
+ {'name': 'click', 'description': 'Move the mouse to a point and click there. Answers with the screen after.',
+ 'inputSchema': {'type': 'object', 'required': ['x', 'y'], 'properties': {
+ **POINT, 'button': {'type': 'string', 'enum': list(BUTTONS), 'default': 'left'},
+ 'count': {'type': 'integer', 'minimum': 1, 'maximum': 3, 'default': 1,
+ 'description': '2 for a double click.'}}}},
+ {'name': 'move', 'description': 'Move the mouse to a point without clicking (to hover). Answers with the '
+ 'screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y'], 'properties': POINT}},
+ {'name': 'drag', 'description': 'Press the left button at one point, move to another and let go. Answers '
+ 'with the screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y', 'to_x', 'to_y'],
+ 'properties': {**POINT, 'to_x': {'type': 'number'},
+ 'to_y': {'type': 'number'}}}},
+ {'name': 'scroll', 'description': 'Turn the mouse wheel over a point: positive steps scroll down, negative '
+ 'up. Answers with the screen after.', 'inputSchema': {'type': 'object', 'required': ['x', 'y', 'steps'],
+ 'properties': {**POINT, 'steps': {'type': 'integer'}}}},
+ {'name': 'type', 'description': 'Type text into whatever has the keyboard focus, as a person would ("\\n" '
+ 'is Enter). Answers with the screen after.',
+ 'inputSchema': {'type': 'object', 'required': ['text'], 'properties': {'text': {'type': 'string'}}}},
+ {'name': 'key', 'description': 'Press a key or a combination: "enter", "esc", "tab", "backspace", "ctrl+s", '
+ '"ctrl+shift+t", "super+b", "F5". Answers with the screen after.',
+ 'inputSchema': {'type': 'object', 'required': ['keys'], 'properties': {'keys': {'type': 'string'}}}},
+]
+
+
+def run_tool(screen, name, args):
+ """Do [name] with [args]; the screen afterwards, as the answer's content."""
+ if name != 'screenshot':
+ if locked():
+ raise Refused('The screen is locked. Ask the person to unlock it.')
+ if name in ('click', 'move', 'drag', 'scroll'):
+ x, y = screen.point(args['x'], args['y'])
+ devices = screen.input()
+ devices.move(x, y, screen.size)
+ if name == 'click':
+ code = BUTTONS.get(args.get('button', 'left'))
+ if code is None:
+ raise Refused('button is left, right or middle.')
+ for _ in range(max(1, min(3, int(args.get('count', 1))))):
+ devices.button(code, True)
+ devices.button(code, False)
+ elif name == 'drag':
+ to = screen.point(args['to_x'], args['to_y'])
+ devices.button(BUTTONS['left'], True)
+ for step in range(1, 11):
+ devices.move(round(x + (to[0] - x) * step / 10), round(y + (to[1] - y) * step / 10), screen.size)
+ time.sleep(0.02)
+ devices.button(BUTTONS['left'], False)
+ elif name == 'scroll':
+ devices.scroll(int(args['steps']))
+ elif name == 'type':
+ screen.type(str(args['text']))
+ elif name == 'key':
+ screen.key(args['keys'])
+ else:
+ raise Refused(f'No tool {name!r}.')
+ time.sleep(SETTLE)
+ data, (width, height) = screen.screenshot()
+ return [{'type': 'text', 'text': f'The screen now, {width}x{height}.'},
+ {'type': 'image', 'data': base64.b64encode(data).decode(), 'mimeType': 'image/png'}]
+
+
+def answer(request, screen):
+ method, ident = request.get('method'), request.get('id')
+ if method == 'initialize':
+ asked = (request.get('params') or {}).get('protocolVersion')
+ result = {'protocolVersion': asked or PROTOCOL, 'capabilities': {'tools': {}},
+ 'serverInfo': {'name': NAME, 'version': VERSION}}
+ elif method == 'tools/list':
+ result = {'tools': TOOLS}
+ elif method == 'tools/call':
+ params = request.get('params') or {}
+ try:
+ result = {'content': run_tool(screen(), params.get('name'), params.get('arguments') or {})}
+ except (Refused, KeyError, ValueError, TypeError, OSError, subprocess.SubprocessError) as error:
+ text = str(error) if isinstance(error, Refused) else f'{type(error).__name__}: {error}'
+ result = {'content': [{'type': 'text', 'text': text}], 'isError': True}
+ elif method == 'ping':
+ result = {}
+ elif ident is None:
+ return None # A notification (initialized, cancelled): nothing to answer.
+ else:
+ return {'jsonrpc': '2.0', 'id': ident, 'error': {'code': -32601, 'message': f'No method {method}'}}
+ return {'jsonrpc': '2.0', 'id': ident, 'result': result}
+
+
+def main():
+ held = {}
+
+ def screen():
+ # Made at the first tool call, so the server starts (and lists its tools) without a desktop.
+ if 'screen' not in held:
+ held['screen'] = Screen()
+ return held['screen']
+
+ for line in sys.stdin:
+ if not line.strip():
+ continue
+ try:
+ request = json.loads(line)
+ except json.JSONDecodeError:
+ reply = {'jsonrpc': '2.0', 'id': None, 'error': {'code': -32700, 'message': 'Not JSON'}}
+ else:
+ reply = answer(request, screen)
+ if reply is not None:
+ sys.stdout.write(json.dumps(reply) + '\n')
+ sys.stdout.flush()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/os/packages.x86_64 b/os/packages.x86_64
index 77d8914b0..89eb2dd6e 100644
--- a/os/packages.x86_64
+++ b/os/packages.x86_64
@@ -41,6 +41,10 @@ intel-media-driver
wl-clipboard
grim
slurp
+# The desktop MCP (os/desktop_mcp.py): an agent's own keyboard and mouse on the screen it sees.
+wtype
+python-pywayland
+wlr-protocols
xdg-utils
xdg-desktop-portal-wlr
pipewire
diff --git a/os/packaging/playwright-mcp/package-lock.json b/os/packaging/playwright-mcp/package-lock.json
new file mode 100644
index 000000000..78a8b13a9
--- /dev/null
+++ b/os/packaging/playwright-mcp/package-lock.json
@@ -0,0 +1,57 @@
+{
+ "name": "harness-os-playwright-mcp",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "harness-os-playwright-mcp",
+ "license": "MIT",
+ "dependencies": {
+ "@playwright/mcp": "0.0.83"
+ }
+ },
+ "node_modules/@playwright/mcp": {
+ "version": "0.0.83",
+ "resolved": "https://registry.npmjs.org/@playwright/mcp/-/mcp-0.0.83.tgz",
+ "integrity": "sha512-oNcl+Ae2/IAjhfPeP46BfIkSakfmprY+aOtkv5MjrQ4lPav4/yNtPhL0iq8SlIM90oApWgBDUxaNKvktazUKOg==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright": "1.64.0-alpha-1790635538000",
+ "playwright-core": "1.64.0-alpha-1790635538000"
+ },
+ "bin": {
+ "playwright-mcp": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/playwright": {
+ "version": "1.64.0-alpha-1790635538000",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0-alpha-1790635538000.tgz",
+ "integrity": "sha512-/5XDUMxpOd/9AojJtWDFIRV3EJZNQ0AwvK69wgt3CPzy8wTgiBG9pX45VfxtqtvIRGwfWzMhrvawNKreG69Snw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright-core": "1.64.0-alpha-1790635538000"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
+ "node_modules/playwright-core": {
+ "version": "1.64.0-alpha-1790635538000",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0-alpha-1790635538000.tgz",
+ "integrity": "sha512-pNwaXirhXMRLaRQs4NQ18EpTdtDoFwyPH9FOaaDC7YXG4hpwE2xmwCic/1srbuExVQC7zT9LJSJpDBAru+Vo9A==",
+ "license": "Apache-2.0",
+ "bin": {
+ "playwright-core": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ }
+ }
+}
diff --git a/os/packaging/playwright-mcp/package.json b/os/packaging/playwright-mcp/package.json
new file mode 100644
index 000000000..0de729181
--- /dev/null
+++ b/os/packaging/playwright-mcp/package.json
@@ -0,0 +1,9 @@
+{
+ "name": "harness-os-playwright-mcp",
+ "private": true,
+ "description": "Playwright's MCP server, bundled in Harness OS for the agents' browser tools (harness computer-use).",
+ "license": "MIT",
+ "dependencies": {
+ "@playwright/mcp": "0.0.83"
+ }
+}
diff --git a/os/root/usr/share/harness-os/AGENTS.md b/os/root/usr/share/harness-os/AGENTS.md
index 220f15ba9..9926fb64c 100644
--- a/os/root/usr/share/harness-os/AGENTS.md
+++ b/os/root/usr/share/harness-os/AGENTS.md
@@ -116,6 +116,14 @@ is required.
- Print/`Super+p` saves a full screenshot and Shift+Print/`Super+r` a region to
`~/Pictures/Screenshots`, also copied to the clipboard. When the user mentions
"the screenshot", read the newest file there. `grim` and `slurp` are installed.
+- To try what you build as a person would, use computer use when you have it:
+ `playwright` for a website (it opens it in a Chromium window of its own, clicks,
+ types, and reads the page and its console) and `desktop` for a desktop app
+ (`screenshot`, `click`, `drag`, `scroll`, `type`, `key`, each answered with the
+ screen afterwards; refused while the screen is locked). They are off until the
+ person turns them on, from their phone (the computer's Computer use) or with
+ `harness computer-use on` in a terminal outside Harness, and a running harness gets
+ them when it restarts. Never turn them on yourself: suggest it instead.
- On the PC image, the packages for supported NVIDIA Turing and newer GPUs are
`nvidia-open-lts nvidia-utils`, including RTX 4090/5090 and RTX 6000 generations.
New USB images carrying the NVIDIA bundle install them offline when the exact
diff --git a/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xml b/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xml
new file mode 100644
index 000000000..583ff6e3a
--- /dev/null
+++ b/os/root/usr/share/harness-os/virtual-keyboard-unstable-v1.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<protocol name="virtual_keyboard_unstable_v1">
+ <copyright>
+ Copyright © 2008-2011 Kristian Høgsberg
+ Copyright © 2010-2013 Intel Corporation
+ Copyright © 2012-2013 Collabora, Ltd.
+ Copyright © 2018 Purism SPC
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice (including the next
+ paragraph) shall be included in all copies or substantial portions of the
+ Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+ </copyright>
+
+ <interface name="zwp_virtual_keyboard_v1" version="1">
+ <description summary="virtual keyboard">
+ The virtual keyboard provides an application with requests which emulate
+ the behaviour of a physical keyboard.
+ </description>
+ <request name="keymap">
+ <arg name="format" type="uint" summary="keymap format, one of wl_keyboard.keymap_format"/>
+ <arg name="fd" type="fd" summary="keymap file descriptor"/>
+ <arg name="size" type="uint" summary="keymap size, in bytes"/>
+ </request>
+ <enum name="error">
+ <entry name="no_keymap" value="0" summary="No keymap was set"/>
+ </enum>
+ <request name="key">
+ <arg name="time" type="uint" summary="timestamp with millisecond granularity"/>
+ <arg name="key" type="uint" summary="key that produced the event"/>
+ <arg name="state" type="uint" summary="physical state of the key"/>
+ </request>
+ <request name="modifiers">
+ <arg name="mods_depressed" type="uint"/>
+ <arg name="mods_latched" type="uint"/>
+ <arg name="mods_locked" type="uint"/>
+ <arg name="group" type="uint"/>
+ </request>
+ <request name="destroy" type="destructor" since="1"/>
+ </interface>
+
+ <interface name="zwp_virtual_keyboard_manager_v1" version="1">
+ <enum name="error">
+ <entry name="unauthorized" value="0" summary="client not authorized to use the interface"/>
+ </enum>
+ <request name="create_virtual_keyboard">
+ <arg name="seat" type="object" interface="wl_seat"/>
+ <arg name="id" type="new_id" interface="zwp_virtual_keyboard_v1"/>
+ </request>
+ </interface>
+</protocol>
diff --git a/os/tests/test_desktop_mcp.py b/os/tests/test_desktop_mcp.py
new file mode 100644
index 000000000..b13f5b307
--- /dev/null
+++ b/os/tests/test_desktop_mcp.py
@@ -0,0 +1,149 @@
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+spec = importlib.util.spec_from_file_location('desktop_mcp', Path(__file__).resolve().parents[1] / 'desktop_mcp.py')
+desktop = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(desktop)
+
+PNG = b'\x89PNG\r\n\x1a\n' + b'\0\0\0\rIHDR' + (1280).to_bytes(4, 'big') + (720).to_bytes(4, 'big') + b'rest'
+
+
+def screen(width=1920, height=1080):
+ """A Screen on a [width]x[height] display, its devices a fake whose keymap has a few keys."""
+ with patch.object(desktop, 'environment', return_value={'WAYLAND_DISPLAY': 'wayland-0'}):
+ made = desktop.Screen()
+ made.size = (width, height)
+ places = {'Control_L': (29, False), 'Shift_L': (42, False), 'Super_L': (125, False), 'Return': (28, False),
+ 't': (20, False), 'b': (48, False), 'question': (53, True)}
+ keymap = MagicMock()
+ keymap.named.side_effect = lambda name: places[name] if name in places else (_ for _ in ()).throw(
+ desktop.Refused(f'Not a key on this keyboard: {name!r}.'))
+ keymap.character.side_effect = lambda char: {'h': (35, False), 'i': (23, False), '!': (2, True)}.get(char)
+ made.devices = MagicMock(keymap=keymap)
+ return made
+
+
+class Protocol(unittest.TestCase):
+ def ask(self, request, made=None):
+ return desktop.answer(request, lambda: made or screen())
+
+ def test_it_introduces_itself_in_the_clients_protocol_version(self):
+ reply = self.ask({'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {'protocolVersion': '2025-03-26'}})
+ self.assertEqual(reply['result']['protocolVersion'], '2025-03-26')
+ self.assertEqual(reply['result']['serverInfo']['name'], 'desktop')
+ self.assertIn('tools', reply['result']['capabilities'])
+
+ def test_it_lists_its_tools(self):
+ tools = self.ask({'jsonrpc': '2.0', 'id': 2, 'method': 'tools/list'})['result']['tools']
+ self.assertEqual([tool['name'] for tool in tools], ['screenshot', 'click', 'move', 'drag', 'scroll', 'type', 'key'])
+ json.dumps(tools)
+
+ def test_it_answers_no_notification_and_refuses_an_unknown_method(self):
+ self.assertIsNone(self.ask({'jsonrpc': '2.0', 'method': 'notifications/initialized'}))
+ self.assertEqual(self.ask({'jsonrpc': '2.0', 'id': 3, 'method': 'resources/list'})['error']['code'], -32601)
+
+ def test_a_tool_that_cannot_act_answers_an_error_in_words(self):
+ with patch.object(desktop, 'locked', return_value=True):
+ reply = self.ask({'jsonrpc': '2.0', 'id': 4, 'method': 'tools/call',
+ 'params': {'name': 'click', 'arguments': {'x': 1, 'y': 1}}})
+ self.assertTrue(reply['result']['isError'])
+ self.assertIn('locked', reply['result']['content'][0]['text'])
+
+
+class Screen(unittest.TestCase):
+ def test_a_screenshots_pixel_is_scaled_back_to_the_screen(self):
+ made = screen()
+ self.assertEqual(made.scale(), 1280 / 1920)
+ self.assertEqual(made.point(640, 360), (960, 540))
+ with self.assertRaisesRegex(desktop.Refused, 'outside the screen'):
+ made.point(1280, 10)
+
+ def test_a_narrow_screen_is_shown_as_it_is(self):
+ made = screen(1024, 768)
+ self.assertEqual(made.scale(), 1.0)
+ self.assertEqual(made.point(10, 20), (10, 20))
+
+ def test_the_screenshot_is_refused_while_locked(self):
+ with patch.object(desktop, 'locked', return_value=True), self.assertRaisesRegex(desktop.Refused, 'locked'):
+ screen().screenshot()
+
+ def test_the_screenshot_is_taken_scaled_down_to_the_width_an_agent_sees(self):
+ made = screen()
+ with patch.object(desktop, 'locked', return_value=False), \
+ patch.object(desktop.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, PNG, b'')) as run:
+ data, size = made.screenshot()
+ self.assertEqual((data, size), (PNG, (1280, 720)))
+ self.assertEqual(run.call_args.args[0], ['grim', '-t', 'png', '-s', '0.6667', '-'])
+
+ def test_an_action_answers_with_the_screen_after_it(self):
+ made = screen()
+ with patch.object(desktop, 'locked', return_value=False), patch.object(desktop.time, 'sleep'), \
+ patch.object(made, 'screenshot', return_value=(PNG, (1280, 720))):
+ content = desktop.run_tool(made, 'click', {'x': 640, 'y': 360, 'count': 2})
+ made.devices.move.assert_called_with(960, 540, (1920, 1080))
+ self.assertEqual(made.devices.button.call_count, 4)
+ self.assertEqual([part['type'] for part in content], ['text', 'image'])
+ self.assertEqual(content[1]['mimeType'], 'image/png')
+
+
+class Keys(unittest.TestCase):
+ def test_a_combination_holds_its_modifiers_around_the_key(self):
+ made = screen()
+ made.key('ctrl+shift+t')
+ made.devices.press.assert_called_with([29, 42, 20])
+ made.key('super+b')
+ made.devices.press.assert_called_with([125, 48])
+ made.key('enter')
+ made.devices.press.assert_called_with([28])
+
+ def test_a_key_shift_reaches_gets_shift(self):
+ made = screen()
+ made.key('ctrl+question')
+ made.devices.press.assert_called_with([29, 42, 53])
+
+ def test_an_unknown_modifier_or_key_is_refused(self):
+ made = screen()
+ with self.assertRaisesRegex(desktop.Refused, 'Not a modifier'):
+ made.key('hyper+t')
+ with self.assertRaisesRegex(desktop.Refused, 'Not a key'):
+ made.key('ctrl+')
+ with self.assertRaisesRegex(desktop.Refused, 'Not a key on this keyboard'):
+ made.key('nosuchkey')
+
+ def test_text_the_keymap_types_goes_through_its_own_keyboard(self):
+ made = screen()
+ with patch.object(desktop.subprocess, 'run') as run:
+ made.type('hi!')
+ run.assert_not_called()
+ self.assertEqual([call.args[0] for call in made.devices.press.call_args_list], [[35], [23], [42, 2]])
+
+ def test_accented_text_goes_through_wtype(self):
+ made = screen()
+ with patch.object(desktop.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, b'', b'')) as run:
+ made.type('chào')
+ self.assertEqual(run.call_args.args[0], ['wtype', '--', 'chào'])
+ made.devices.press.assert_not_called()
+
+
+class Environment(unittest.TestCase):
+ def test_a_server_with_no_display_finds_the_sessions(self):
+ with tempfile.TemporaryDirectory() as runtime:
+ (Path(runtime) / 'wayland-1').touch()
+ (Path(runtime) / 'wayland-1.lock').touch()
+ with patch.dict(desktop.os.environ, {'XDG_RUNTIME_DIR': runtime}, clear=True):
+ self.assertEqual(desktop.environment()['WAYLAND_DISPLAY'], 'wayland-1')
+
+ def test_no_desktop_is_said(self):
+ with tempfile.TemporaryDirectory() as runtime, \
+ patch.dict(desktop.os.environ, {'XDG_RUNTIME_DIR': runtime}, clear=True), \
+ self.assertRaisesRegex(desktop.Refused, 'No desktop'):
+ desktop.environment()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/os/tools/build-package.py b/os/tools/build-package.py
index 32f78df03..aba1c2ee2 100644
--- a/os/tools/build-package.py
+++ b/os/tools/build-package.py
@@ -16,7 +16,9 @@ import tempfile
# dnsmasq: NetworkManager's shared mode hands the setup hotspot's phone an address with it.
# github-cli (gh) and sound-theme-freedesktop (foot.ini's bell): for a computer installed before they were
# in the image (packages.x86_64) to get them with an update.
-DEPENDENCIES = ('nodejs-lts-jod', 'tmux', 'foot', 'gtklock', 'grim', 'slurp', 'dnsmasq', 'github-cli', 'sound-theme-freedesktop')
+# wtype, python-pywayland and wlr-protocols: the desktop MCP's keys and mouse (os/desktop_mcp.py).
+DEPENDENCIES = ('nodejs-lts-jod', 'tmux', 'foot', 'gtklock', 'grim', 'slurp', 'dnsmasq', 'github-cli', 'sound-theme-freedesktop',
+ 'wtype', 'python-pywayland', 'wlr-protocols')
def package_info(version, timestamp, size, compositor_dependencies=()):
@@ -83,6 +85,7 @@ def stage(source, runtime, destination, commit):
'os/hardware.py': 'usr/lib/harness-os/hardware.py',
'os/gpu_health.py': 'usr/lib/harness-os/gpu_health.py',
'os/gpu_probe.py': 'usr/lib/harness-os/gpu_probe.py',
+ 'os/desktop_mcp.py': 'usr/lib/harness-os/desktop_mcp.py',
'os/tools/hn-os': 'usr/bin/hn-os',
'os/lock.json': 'usr/share/harness-os/lock.json',
'tui/README.md': 'usr/share/harness-os/guide/tui.md',
@@ -103,6 +106,12 @@ def stage(source, runtime, destination, commit):
shutil.copyfile(runtime / local, library / target)
(library / target).chmod(0o755 if local == 'harness-tui' else 0o644)
(library / 'hn').symlink_to('harness-tui')
+ # Playwright's MCP server (build-runtime.sh), for `harness computer-use`. Without npm's .bin links: a link out of
+ # its directory is refused by the updater, and the CLI runs the server's cli.js with node.
+ playwright = runtime / 'playwright-mcp'
+ if playwright.is_dir():
+ shutil.copytree(playwright, destination / 'usr/lib/harness-os/playwright-mcp',
+ ignore=shutil.ignore_patterns('.bin', '.package-lock.json'))
for directory in ['usr/bin', 'usr/lib/harness-os']:
for path in (destination / directory).rglob('*'):
if path.is_file() and not path.is_symlink():
diff --git a/os/tools/build-runtime.sh b/os/tools/build-runtime.sh
index e10cd89ef..e35cad265 100644
--- a/os/tools/build-runtime.sh
+++ b/os/tools/build-runtime.sh
@@ -51,7 +51,16 @@ data['files'] = {path.name: {'sha256': hashlib.sha256(path.read_bytes()).hexdige
for path in sorted(folder.iterdir())}
(folder / 'source.json').write_text(json.dumps(data, indent=2) + '\n')
PY
+# Playwright's MCP server, from its lock and without install scripts, for `harness computer-use` (cli
+# agentClient/computerUse.ts): in the package only, never in a fast runtime update.
+playwright_stage=$(mktemp -d "${TMPDIR:-/tmp}/harness-playwright-mcp.XXXXXX")
+trap 'rm -rf -- "$runtime_stage" "$playwright_stage"' EXIT
+cp os/packaging/playwright-mcp/package.json os/packaging/playwright-mcp/package-lock.json "$playwright_stage/"
+npm ci --prefix "$playwright_stage" --omit=dev --ignore-scripts --no-audit --no-fund
+node "$playwright_stage/node_modules/@playwright/mcp/cli.js" --version >/dev/null
mkdir -p -- "$(dirname -- "$runtime_output")"
mkdir -- "$runtime_output"
cp "$runtime_stage/harness-tui" "$runtime_stage/cli.js" "$runtime_stage/notify.mjs" "$runtime_output/"
cp "$runtime_stage/source.json" "$runtime_output/"
+mkdir -- "$runtime_output/playwright-mcp"
+cp -R "$playwright_stage/package.json" "$playwright_stage/node_modules" "$runtime_output/playwright-mcp/"