The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
"""A pull request's Desktop and CLI, run as "Harness Test" beside the installed app.
python3 desktop/tool/test_flavor.py start [--account EMAIL]
# build this checkout's CLI and Desktop and start "Harness Test";
# the first start on a Mac names the project's test account
python3 desktop/tool/test_flavor.py status # its home, port, app and signed-in test account
python3 desktop/tool/test_flavor.py stop # quit it and remove everything it made
python3 desktop/tool/test_flavor.py forget # sign the test account out on this Mac
Design: docs/plans/2026-11-03-desktop-test-flavor-design.md. The test flavor runs with its own HOME, so everything
Desktop and the CLI keep under ~/.harness is its own. The few per-user resources HOME does not move (the daemon port,
the tmux server, Claude Code's Keychain item, Codex's home, the self-updater) are moved by flavor_env, and the copied
app gets its own bundle id and gives up the harness:// links.
"""
import base64
import binascii
import copy
import hashlib
import json
import os
import plistlib
import shutil
import signal
import socket
import subprocess
import sys
import tempfile
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
TEST_ID = 'ai.autonomous.harness.test'
TEST_NAME = 'Harness Test'
CLAUDE_KEYCHAIN_SERVICE = 'Claude Code-credentials'
# What a program needs to run, and nothing a person's session or agent pane adds: a Tester is usually a Harness agent,
# whose pane carries the person's daemon's folders, its tmux server, git's credential helper and hooks
# (GIT_CONFIG_PARAMETERS), gh's sign-in and model API keys (security review 1 of #2796).
KEPT_ENV = ('PATH', 'LANG', 'TERM', 'USER', 'LOGNAME', 'SHELL', 'TMPDIR', '__CF_USER_TEXT_ENCODING')
KEPT_ENV_PREFIXES = ('LC_',)
BACKEND = 'https://harness-api.autonomous.ai' # the CLI's default (BACKEND_WS_URL), which the test copy uses
SAME_ACCOUNT = "The test copy signed in as this Mac's own account. Sign it in with the test account instead."
class Refused(Exception):
"""A step the test flavor will not take, in words for the Tester."""
class SameAccount(Refused):
"""The test copy is, or may be, signed in as the Mac owner's account."""
def flavor_env(base, test_home, shared, port):
"""The test flavor's environment: [base] with every per-user resource moved into [test_home].
[shared] (~/.harness-test) keeps the test account's sign-in and this Mac's test computer id, reused across runs.
"""
env = {key: value for key, value in base.items() if key in KEPT_ENV or key.startswith(KEPT_ENV_PREFIXES)}
person_harness = Path(base['HOME']) / '.harness' if base.get('HOME') else None
if 'PATH' in env and person_harness:
env['PATH'] = os.pathsep.join(entry for entry in env['PATH'].split(os.pathsep)
if not _inside(Path(entry), person_harness))
claude = str(test_home / '.claude')
env.update({
'HOME': str(test_home),
'PORT': str(port),
'TMUX_TMPDIR': str(test_home / 'tmux'),
# Claude Code names its Keychain item from these; the CLI's refresh write reads only the second
# (cli/src/lib/agentCredentials.ts claudeKeychainService), so both are set.
'CLAUDE_CONFIG_DIR': claude,
'CLAUDE_SECURESTORAGE_CONFIG_DIR': claude,
'CODEX_HOME': str(test_home / '.codex'),
'HARNESS_AUTH_DIR': str(shared / 'auth'),
'ADAPTER_COMPUTER_ID_FILE': str(shared / 'computer-id'),
# The PR's CLI is what is being tested: it must never update itself into the released one.
'ADAPTER_UPDATE_DISABLE': '1',
})
return env
def _inside(path, folder):
return path == folder or folder in path.parents
def claude_keychain_service(env):
"""The Keychain service Claude Code and the CLI use under [env] (cli/src/lib/agentCredentials.ts)."""
secure = env.get('CLAUDE_SECURESTORAGE_CONFIG_DIR')
hashed = secure if secure is not None else env.get('CLAUDE_CONFIG_DIR')
if not hashed:
return CLAUDE_KEYCHAIN_SERVICE
return f'{CLAUDE_KEYCHAIN_SERVICE}-{hashlib.sha256(hashed.encode()).hexdigest()[:8]}'
def patch_info(info):
"""A copy of the app's Info.plist with the test flavor's identity and no harness:// scheme."""
patched = copy.deepcopy(info)
patched['CFBundleIdentifier'] = TEST_ID
patched['CFBundleName'] = TEST_NAME
patched['CFBundleDisplayName'] = TEST_NAME
patched['CFBundleExecutable'] = TEST_NAME
patched.pop('CFBundleURLTypes', None)
return patched
def install_cli(cli_dir, test_home, node):
"""The PR's own CLI, bundled into [test_home] where Desktop looks first (lib/core/harness_cli_runner.dart).
`npm run bundle` writes one self-contained cli.js to BUNDLE_OUT_DIR. Desktop runs it with the Node that
runtime/current-node names, which must sit inside runtime/, so [node] is linked there. Returns cli.js.
"""
harness = test_home / '.harness'
out = harness / 'cli'
out.mkdir(parents=True, exist_ok=True)
env = dict(os.environ, BUNDLE_OUT_DIR=str(out), PATH=f'{node.parent}{os.pathsep}{os.environ.get("PATH", "")}')
subprocess.run(['npm', 'run', '--silent', 'bundle'], cwd=cli_dir, env=env, check=True,
stdout=subprocess.DEVNULL)
runtime = harness / 'runtime'
runtime.mkdir(parents=True, exist_ok=True)
linked = runtime / 'node'
if linked.is_symlink() or linked.exists():
linked.unlink()
linked.symlink_to(node.resolve())
(runtime / 'current-node').write_text(f'{linked}\n')
cli = out / 'cli.js'
if not cli.is_file():
raise RuntimeError(f'the CLI bundle did not write {cli}')
return cli
def session_account(path):
"""The account id a CLI session.json (cli/src/lib/authSession.ts) was signed in to, or None.
`signInAcct` when the session has it; for an older one, the access token's subject, decoded here and never
printed or returned. Missing or unreadable is None.
"""
try:
session = json.loads(path.read_text())
except (OSError, ValueError):
return None
if not isinstance(session, dict):
return None
account = session.get('signInAcct')
if isinstance(account, str) and account:
return account
token = session.get('accessToken')
if not isinstance(token, str) or token.count('.') != 2:
return None
payload = token.split('.')[1]
try:
claims = json.loads(base64.urlsafe_b64decode(payload + '=' * (-len(payload) % 4)))
except (ValueError, binascii.Error):
return None
subject = claims.get('sub') if isinstance(claims, dict) else None
return subject if isinstance(subject, str) and subject else None
def check_not_owner(real, test):
"""Refuses a test sign-in that is the person's own account, or that cannot be told apart from it."""
if test is None or test == real:
raise SameAccount(SAME_ACCOUNT)
@dataclass(frozen=True)
class Paths:
repo: Path
shared: Path # ~/.harness-test: the test account's sign-in and this Mac's test computer id
person_home: Path # ~: read only for the person's account id
@property
def build(self):
return self.repo / 'desktop/build/test-flavor'
@property
def state(self):
return self.build / 'state.json'
@property
def app(self):
return self.build / f'{TEST_NAME}.app'
@property
def built_app(self):
return self.repo / 'desktop/build/macos/Build/Products/Debug/Harness.app'
@property
def test_session(self):
return self.shared / 'auth/session.json'
@property
def person_session(self):
return self.person_home / '.harness/auth/session.json'
@property
def account(self):
return self.shared / 'account'
class Runner:
"""The processes the test flavor starts and stops: its own build, CLI, app and tmux server, never another's."""
def run(self, argv, env=None, interactive=False, check=True):
if interactive:
return subprocess.run([str(a) for a in argv], env=env, check=check)
return subprocess.run([str(a) for a in argv], env=env, check=check, capture_output=True, text=True)
def launch(self, argv, env):
return subprocess.Popen([str(a) for a in argv], env=env, start_new_session=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).pid
def alive(self, pid):
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def account_email(self, session):
"""The email of the account a test session.json belongs to, from the backend, or None.
Reads the test account's own access token, in memory only, to ask `/api/auth/me`: never the person's.
"""
try:
stored = json.loads(session.read_text())
request = urllib.request.Request(f'{BACKEND}/api/auth/me', headers={
'Authorization': f"Bearer {stored['accessToken']}",
'x-autonomous-env': stored.get('autonomousEnv', 'prod'),
})
with urllib.request.urlopen(request, timeout=15) as reply:
email = json.loads(reply.read())['data']['user']['email']
except (OSError, ValueError, KeyError, TypeError):
return None
return email.strip().lower() if isinstance(email, str) and email.strip() else None
def executable_of(self, pid):
"""The executable a running pid runs (macOS `ps -o comm=` gives its path), or None."""
found = subprocess.run(['ps', '-p', str(pid), '-o', 'comm='], capture_output=True, text=True)
if found.returncode != 0:
return None
return found.stdout.strip() or None
def terminate(self, pid):
try:
os.kill(pid, signal.SIGTERM)
except (ProcessLookupError, PermissionError):
return
for _ in range(50):
if not self.alive(pid):
return
time.sleep(0.2)
def port_free(self, port):
with socket.socket() as s:
try:
s.bind(('127.0.0.1', port))
except OSError:
return False
return True
def choose_port(runner, start=18474, end=18999):
for port in range(start, end):
if runner.port_free(port):
return port
raise Refused(f'No free port between {start} and {end} for the test daemon.')
def _cli(test_home):
return test_home / '.harness/runtime/node', test_home / '.harness/cli/cli.js'
def _read_state(paths):
try:
return json.loads(paths.state.read_text())
except (OSError, ValueError):
return None
def _copy_app(runner, paths):
if paths.app.exists():
shutil.rmtree(paths.app)
paths.build.mkdir(parents=True, exist_ok=True)
runner.run(['ditto', paths.built_app, paths.app])
info_path = paths.app / 'Contents/Info.plist'
with info_path.open('rb') as source:
info = plistlib.load(source)
executable = paths.app / 'Contents/MacOS' / info.get('CFBundleExecutable', 'Harness')
with info_path.open('wb') as out:
plistlib.dump(patch_info(info), out)
executable.rename(paths.app / 'Contents/MacOS' / TEST_NAME)
runner.run(['codesign', '--force', '--sign', '-',
'--preserve-metadata=entitlements,requirements,flags,runtime', paths.app])
def _named_account(paths, named):
"""The project's test account for this Mac: named on its first start, kept in ~/.harness-test/account."""
kept = paths.account.read_text().strip().lower() if paths.account.exists() else None
named = named.strip().lower() if named else None
if kept and named and named != kept:
raise Refused(f"This Mac's test account is {kept}; run forget to change it.")
if not kept and not named:
raise Refused("Name the project's test account the first time: start --account EMAIL")
if not kept:
paths.shared.mkdir(parents=True, exist_ok=True)
paths.account.write_text(f'{named}\n')
return kept or named
def _person_account(paths):
"""This Mac's own account id, None when it is not signed in; refuses when its session cannot be read."""
account = session_account(paths.person_session)
if account is None and paths.person_session.exists():
raise Refused("Could not read this Mac's own sign-in to compare the test account with it.")
return account
def start(runner, paths, installer, named=None):
if paths.state.exists():
raise Refused('A test copy is already running; run stop first.')
expected = _named_account(paths, named)
person = _person_account(paths)
port = choose_port(runner)
test_home = Path(tempfile.mkdtemp(prefix='harness-test-flavor.'))
launched = False
try:
node = Path(shutil.which('node') or 'node')
cli = installer(paths.repo / 'cli', test_home, node)
env = flavor_env(dict(os.environ), test_home, paths.shared, port)
if session_account(paths.test_session) is None:
print('Sign the test copy in with the test account, in a private browser window:')
runner.run([_cli(test_home)[0], cli, 'login', '--code'], env=env, interactive=True)
try:
check_not_owner(person, session_account(paths.test_session))
# A fresh token for the question below; the test CLI refreshes its own session.
runner.run([_cli(test_home)[0], cli, 'auth', 'status', '--json'], env=env, check=False)
email = runner.account_email(paths.test_session)
if email != expected:
raise Refused(f"The test copy signed in as {email or 'an account it could not confirm'}, not the test "
f'account {expected}. Sign it in with the test account instead.')
except Refused:
runner.run([_cli(test_home)[0], cli, 'logout', '--from-sign-out-button'], env=env, check=False)
shutil.rmtree(paths.shared / 'auth', ignore_errors=True)
raise
runner.run(['bash', paths.repo / 'desktop/scripts/build-macos-debug.sh', '--no-pub',
f'--dart-define=HARNESS_DAEMON_PORT={port}'])
_copy_app(runner, paths)
if not runner.port_free(port):
raise Refused(f'Port {port} was taken; run start again.')
pid = runner.launch([paths.app / 'Contents/MacOS' / TEST_NAME], env)
launched = True
paths.state.write_text(json.dumps({'testHome': str(test_home), 'port': port, 'appPid': pid,
'app': str(paths.app)}))
finally:
if not launched:
shutil.rmtree(test_home, ignore_errors=True)
shutil.rmtree(paths.app, ignore_errors=True)
print(f'{TEST_NAME} is running: home {test_home}, port {port}, signed in as the test account {expected}.')
print('When the test is done: python3 desktop/tool/test_flavor.py stop')
return 0
def status(runner, paths):
state = _read_state(paths)
if state is None:
print('No test copy is running.')
return 0
print(f"home {state['testHome']}, port {state['port']}, app pid {state['appPid']} "
f"({'running' if runner.alive(state['appPid']) else 'not running'})")
try:
check_not_owner(session_account(paths.person_session), session_account(paths.test_session))
except SameAccount as refused:
print(refused)
return 1
expected = paths.account.read_text().strip() if paths.account.exists() else None
email = runner.account_email(paths.test_session)
if not expected or email != expected:
print(f"The test copy is signed in as {email or 'an account it could not confirm'}, not the test account "
f'{expected}. Run stop, then start.')
return 1
print(f'signed in as the test account {email}')
return 0
def stop(runner, paths):
state = _read_state(paths)
if state is None:
print('No test copy is running.')
return 0
problems = []
pid = state.get('appPid')
app_executable = str(Path(state.get('app', paths.app)) / 'Contents/MacOS' / TEST_NAME)
# Only while the pid still runs this copy: after a crash or a reboot it can be any of the person's processes.
if isinstance(pid, int) and runner.alive(pid) and runner.executable_of(pid) == app_executable:
runner.terminate(pid)
test_home = Path(state['testHome'])
node, cli = _cli(test_home)
env = flavor_env(dict(os.environ), test_home, paths.shared, state['port'])
if cli.exists():
if runner.run([node, cli, 'stop'], env=env, check=False).returncode != 0:
problems.append('the test daemon did not stop cleanly')
tmux_socket = test_home / 'tmux' / f'tmux-{os.getuid()}' / 'default'
if tmux_socket.exists():
runner.run(['tmux', '-S', tmux_socket, 'kill-server'], check=False)
for path in (test_home, Path(state.get('app', paths.app))):
shutil.rmtree(path, ignore_errors=True)
if path.exists():
problems.append(f'could not remove {path}')
paths.state.unlink(missing_ok=True)
for problem in problems:
print(f'stop: {problem}')
return 1 if problems else 0
def forget(runner, paths, installer):
if paths.state.exists():
raise Refused('A test copy is running; run stop first.')
if session_account(paths.test_session) is not None:
test_home = Path(tempfile.mkdtemp(prefix='harness-test-flavor.'))
try:
cli = installer(paths.repo / 'cli', test_home, Path(shutil.which('node') or 'node'))
env = flavor_env(dict(os.environ), test_home, paths.shared, choose_port(runner))
runner.run([_cli(test_home)[0], cli, 'logout', '--from-sign-out-button'], env=env, check=False)
finally:
shutil.rmtree(test_home, ignore_errors=True)
shutil.rmtree(paths.shared, ignore_errors=True)
print('The test account is signed out on this Mac. Its one test machine stays in the test account until removed '
'from its Machines list.')
return 0
def main(argv, runner=None, paths=None, installer=install_cli):
if runner is None and sys.platform != 'darwin':
raise Refused('The Desktop test flavor runs on macOS.')
runner = runner or Runner()
paths = paths or Paths(repo=Path(__file__).resolve().parents[2], shared=Path.home() / '.harness-test',
person_home=Path.home())
command = argv[0] if argv else ''
if command == 'start':
named = argv[argv.index('--account') + 1] if '--account' in argv[1:-1] else None
return start(runner, paths, installer, named)
if command == 'status':
return status(runner, paths)
if command == 'stop':
return stop(runner, paths)
if command == 'forget':
return forget(runner, paths, installer)
print(__doc__)
return 2
if __name__ == '__main__':
try:
sys.exit(main(sys.argv[1:]))
except Refused as refused:
print(refused, file=sys.stderr)
sys.exit(1)