The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
harnessd: the teams' prompt scopes can run in their own process (#794)
Deed71c9629368a
diff --git a/cli/e2e/teamsProcess.e2e.ts b/cli/e2e/teamsProcess.e2e.tsnew file mode 100644index 000000000..5d892c4bc--- /dev/null+++ b/cli/e2e/teamsProcess.e2e.ts@@ -0,0 +1,186 @@+/**+ * The teams' prompt scopes in their own process, on the real daemon (`HARNESSD_SERVICES=teams`, here also+ * with search and the viewers). A message written from a tab belongs to that tab's team once its engine+ * takes it; the scopes work that out in their own process, and the team features read it back through the+ * core. Whatever happens to the process costs teams alone. Stopped, hung, killed or crashing on every start:+ * - every message is still written, and agents keep working;+ * - a scope that is not known for sure reads as no team, never a wrong one;+ * - a change made meanwhile reaches the process once it is back, applied once.+ * A core that restarts starts the scopes over, as it always did.+ */+import { mkdirSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { channelTeamId } from '../src/teams/service.js'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++/** The teams process the master runs now: the last one it said it started. Read from its log, never+ * from the process table, where another daemon's services could be. */+const teamsPid = (d: IsolatedDaemon): number | null => {+ const started = [...d.log().matchAll(/\[harnessd\] service teams started \(pid (\d+)\)/g)]+ return started.length ? Number(started[started.length - 1][1]) : null+}+const restarts = (d: IsolatedDaemon) => [...d.log().matchAll(/\[harnessd\] service teams started .* restart \d+/g)].length+const connected = (d: IsolatedDaemon) => d.log().split('[services] teams connected').length - 1+const ready = (d: IsolatedDaemon) => [...d.log().matchAll(/\[cli\] ready/g)].length++const row = async (client: LocalClient, agentId: string) =>+ ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true }, 30_000)).agents)+ .find((agent) => agent.id === agentId)+async function create(d: IsolatedDaemon, client: LocalClient, folder: string, engine = 'claude'): Promise<Record<string, any>> {+ const cwd = join(d.projectsDir, folder)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()+ return until(`${folder} to bind its conversation`, async () => {+ const agent = await row(client, created.agent.id)+ return agent?.sessionId && agent.status === 'active' ? agent : null+ }, 60_000, 500)+}+/** A message written to the agent, from a tab or not, and its turn taken. */+async function say(client: LocalClient, agentId: string, content: string, tabId?: string): Promise<void> {+ const ended = client.next((frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId, 45_000, `turn_ended (${content})`)+ client.send('message', { agentId, content, ...(tabId ? { tabId } : {}) })+ await ended+}+/** The team the agent's current prompt came from, as a team member asks it. */+const scopeOf = async (client: LocalClient, agentId: string): Promise<string | null> => {+ const answer = await client.request('team_delivery', { action: 'prompt_scope', agentId }, 30_000)+ expect(answer.error, JSON.stringify(answer)).toBeUndefined()+ return answer.teamId ?? null+}+const scopeBecomes = (client: LocalClient, agentId: string, teamId: string | null, what: string) =>+ until(what, async () => (await scopeOf(client, agentId)) === teamId || null, 30_000, 200)++describe('the teams\' prompt scopes in their own process', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })+ const fresh = async (env: Record<string, string> = {}, waitForTeams = true) => {+ const d = await IsolatedDaemon.create({ env: {+ HARNESSD_SERVICES: 'teams',+ HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '200',+ HARNESSD_SERVICE_MAX_BACKOFF_MS: '1000',+ ...env,+ } })+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ if (waitForTeams) await until('teams to connect to the core', () => connected(d) >= 1 || null, 30_000, 100)+ return d+ }++ it('by default the scopes stay in the core\'s process, as before', async () => {+ const d = await fresh({ HARNESSD_SERVICES: '' }, false)+ const client = await LocalClient.connect(d)+ const agent = await create(d, client, 'scopes-default')+ await say(client, agent.id, 'hello from the alpha tab', 'tab-alpha')+ expect(await scopeOf(client, agent.id)).toBe(channelTeamId('tab-alpha'))+ await say(client, agent.id, 'hello from nowhere in particular')+ expect(await scopeOf(client, agent.id)).toBeNull()+ expect(teamsPid(d)).toBeNull()+ client.close()+ })++ it('stopped, it costs no message its write; the scope reads as no team, and the change made meanwhile lands once it resumes', async () => {+ const d = await fresh({ HARNESSD_SERVICES: 'search,viewers,teams' })+ for (const service of ['search', 'viewers']) {+ await until(`${service} to connect to the core`, () => d.log().includes(`[services] ${service} connected`) || null, 30_000, 200)+ }+ const client = await LocalClient.connect(d)+ const agent = await create(d, client, 'scopes-stopped')+ await say(client, agent.id, 'hello from the alpha tab', 'tab-alpha')+ await scopeBecomes(client, agent.id, channelTeamId('tab-alpha'), 'the alpha team, through the teams process')+ const pid = teamsPid(d)!+ process.kill(pid, 'SIGSTOP')+ try {+ await say(client, agent.id, 'hello from the beta tab', 'tab-beta')+ // Not yet known for sure: no team, never the alpha team it had.+ expect(await scopeOf(client, agent.id)).toBeNull()+ } finally {+ process.kill(pid, 'SIGCONT')+ }+ // The beta message's write and start were made while it was stopped; applied twice, they would match+ // nothing (two equal pending messages read as no team).+ await scopeBecomes(client, agent.id, channelTeamId('tab-beta'), 'the beta team, once the process resumes')+ expect(restarts(d)).toBe(0)+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('hung, it is killed at the heartbeat watch and started again; the message written meanwhile keeps its team', async () => {+ const d = await fresh({ HARNESSD_SERVICE_HEARTBEAT_TIMEOUT_MS: '2000' })+ const client = await LocalClient.connect(d)+ const agent = await create(d, client, 'scopes-hung', 'codex')+ await say(client, agent.id, 'hello from the alpha tab', 'tab-alpha')+ await scopeBecomes(client, agent.id, channelTeamId('tab-alpha'), 'the alpha team')+ const first = teamsPid(d)!+ process.kill(first, 'SIGSTOP')+ await say(client, agent.id, 'hello from the beta tab', 'tab-beta')+ await until('the master to find teams hung', () => d.log().includes('[harnessd] service teams sent no heartbeat') || null, 30_000, 200)+ await until('teams to be started again', () => restarts(d) >= 1 && connected(d) >= 2 || null, 30_000, 200)+ expect(teamsPid(d)).not.toBe(first)+ // Lost with the hung process, kept by the core, and given to the new one.+ await scopeBecomes(client, agent.id, channelTeamId('tab-beta'), 'the beta team, from the new process')+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('killed outright, messages are written while it is down; what it is told then lands once, and what it knew goes', async () => {+ // Kept down for thirty seconds: the reads below must happen while it is, however loaded the machine.+ const d = await fresh({ HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '30000' })+ const client = await LocalClient.connect(d)+ const [moving, still] = [await create(d, client, 'scopes-moving'), await create(d, client, 'scopes-still')]+ await say(client, moving.id, 'hello from the alpha tab', 'tab-alpha')+ await say(client, still.id, 'hello from the gamma tab', 'tab-gamma')+ await scopeBecomes(client, still.id, channelTeamId('tab-gamma'), 'the gamma team')+ await scopeBecomes(client, moving.id, channelTeamId('tab-alpha'), 'the alpha team')+ process.kill(teamsPid(d)!, 'SIGKILL')+ await until('the master to see teams gone', () => d.log().includes('[harnessd] service teams exited') || null, 30_000, 100)+ await say(client, moving.id, 'hello from the beta tab', 'tab-beta')+ // Down: a scope with a change on its way reads as no team; one without, as it last was.+ expect(await scopeOf(client, moving.id)).toBeNull()+ expect(await scopeOf(client, still.id)).toBe(channelTeamId('tab-gamma'))+ expect(restarts(d), 'read while teams was still down').toBe(0)+ await until('teams to be back', () => restarts(d) >= 1 && connected(d) >= 2 || null, 60_000, 200)+ await scopeBecomes(client, moving.id, channelTeamId('tab-beta'), 'the beta team, from the new process')+ // What only the old process knew went with it: no team, not the gamma team.+ await scopeBecomes(client, still.id, null, 'the old process\'s scope to be gone')+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('crashing on every start, it is parked; every message is still written, and reads as no team', async () => {+ const d = await fresh({ HARNESSD_TEST_FAULTS: 'teams.crash', HARNESSD_SERVICE_PARK_CRASHES: '3' }, false)+ const client = await LocalClient.connect(d)+ const agent = await create(d, client, 'scopes-parked')+ await until('the master to park teams', () => d.log().includes('[harnessd] service teams ended 3 times') || null, 60_000, 250)+ await say(client, agent.id, 'hello from the alpha tab', 'tab-alpha')+ await say(client, agent.id, 'and once more', 'tab-alpha')+ expect(await scopeOf(client, agent.id)).toBeNull()+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('a core that restarts leaves the teams process running, and the scopes start over with it, as they always did', async () => {+ const d = await fresh()+ let client = await LocalClient.connect(d)+ const agent = await create(d, client, 'scopes-core')+ await say(client, agent.id, 'hello from the alpha tab', 'tab-alpha')+ await scopeBecomes(client, agent.id, channelTeamId('tab-alpha'), 'the alpha team')+ const teams = teamsPid(d)+ const wired = ready(d)+ process.kill(d.corePid()!, 'SIGKILL')+ await until('a new core to finish starting', () => ready(d) > wired || null, 60_000, 200)+ await until('teams to reach the new core', () => connected(d) >= 2 || null, 30_000, 200)+ client.close()+ client = await LocalClient.connect(d)+ await until('the agent to be live again', async () => (await row(client, agent.id))?.status === 'active' || null, 60_000, 250)+ expect(await scopeOf(client, agent.id)).toBeNull()+ await say(client, agent.id, 'hello from the delta tab', 'tab-delta')+ await scopeBecomes(client, agent.id, channelTeamId('tab-delta'), 'the delta team, with the new core')+ expect(teamsPid(d)).toBe(teams)+ expect(restarts(d)).toBe(0)+ client.close()+ })+})diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 1c8e64a46..785813400 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -1,5 +1,5 @@import type { PurgeAgentService } from './lib/purgeAgentService.js'-import type { Asker } from './core/api.js'+import type { Asker, PromptScopes } from './core/api.js'import { ServiceUnavailableError } from './core/serviceHost.js'import type { ActivityFrame } from './lib/turnActivity.js'import { readSessionGitPullRequest } from './lib/sessionGitPullRequest.js'@@ -400,7 +400,7 @@ export class BackendSocket {teamStateDir = join(env.ADAPTER_DATA_DIR, 'teams')teamCommand: string | null = nullprivate teamService: TeamService | null = null- readonly swarmPromptScopes = new SwarmPromptScopes()+ swarmPromptScopes: PromptScopes = new SwarmPromptScopes()private teamMailboxService: TeamMailbox | null = nullreadChannelDesk: (() => Promise<unknown>) | null = nullwriteChannelSettings: ((enabled: boolean) => Promise<unknown>) | null = nulldiff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 2ece36e23..d4f17164f 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -172,6 +172,8 @@ import { KNOWN_SERVICES, serviceSpecs } from './harnessd/services.js'import { runSearchService } from './services/searchProcess.js'import { runViewersService } from './services/viewersProcess.js'import { runWorkspacesService } from './services/workspacesProcess.js'+import { createTeamsLink } from './core/teamsLink.js'+import { runTeamsService } from './services/teamsProcess.js'import { SEARCH_REQUESTS, startSearch } from './services/search.js'import { STORE_REQUESTS, startStore } from './services/store.js'import { startViewers } from './services/viewers.js'@@ -1909,10 +1911,6 @@ async function runForeground(session: AuthSession | null): Promise<void> {// Nothing is answered until start-up is done (see the end of this function).backend.holdRequests()daemonBoot.openRequests = () => backend.openRequests()- // The team prompt scopes are the socket's, and the core reaches them through the service host's guard- // (core/serviceHost.ts): a fault there can no longer cost a message its write, a hook its answer or- // an event its turn.- serviceHost.start('teams', (_core, started) => { started.teams = backend.swarmPromptScopes }, coreApi, TEAMS_FALLBACKS)const teams: TeamsPort = {prepare: (...args) => ports.teams?.prepare(...args) ?? (() => {}),started: (...args) => ports.teams?.started(...args),@@ -2220,12 +2218,15 @@ async function runForeground(session: AuthSession | null): Promise<void> {const viewersLink = createViewersLink(coreApi, (frame, opts) => serviceLinks.notify('viewers', frame, opts))// How the core tells workspaces in their own process what to do, and answers them (core/workspacesLink.ts).const workspacesLink = createWorkspacesLink(coreApi, (frame) => serviceLinks.notify('workspaces', frame), forgetAgentProject)+ // Every change to the prompt scopes, kept until their own process has it, and each agent's scope (core/teamsLink.ts).+ const teamsLink = createTeamsLink({ notify: (frame) => serviceLinks.notify('teams', frame) })const serviceLinks = createServiceLinks({token: serviceToken,owned: Object.fromEntries([...outOfProcess].map((name) => [name, requestsOf[name] ?? []])),// Search asks for every agent, live then stopped, with the name the apps show for it: what it indexes.answer: (service, query, payload) => service === 'viewers' ? viewersLink.answer(query, payload)- : service === 'workspaces' ? workspacesLink.answer(query, payload) : query === 'agents'+ : service === 'workspaces' ? workspacesLink.answer(query, payload)+ : service === 'teams' ? teamsLink.answer(query, payload) : query === 'agents'? { agents: coreApi.agents.all().map((session) => ({ ...session, displayName: coreApi.agents.displayName(session) })) }: { error: 'UNKNOWN_QUERY' },})@@ -2253,6 +2254,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {// Workspaces: in this process, or in its own (services/workspacesProcess.ts), told what to do and when.if (outOfProcess.has('workspaces')) ports.workspaces = workspacesLink.portelse serviceHost.start('workspaces', startWorkspaces, coreApi, WORKSPACES_FALLBACKS)+ // The prompt scopes, behind the service host's guard (a fault there costs no message its write) or in their own process.+ if (outOfProcess.has('teams')) ports.teams = backend.swarmPromptScopes = teamsLink.scopes+ else serviceHost.start('teams', (_core, started) => { started.teams = backend.swarmPromptScopes }, coreApi, TEAMS_FALLBACKS)// The harnesses installed here, and installing, updating and removing one (services/store.ts).serviceHost.serve('store', startStore, coreApi, STORE_REQUESTS)@@ -5640,7 +5644,7 @@ switch (cmd) {case '__service': { // internal: a service harnessd's master runs in its own process (services/process.ts)const socketPath = localSocketPath(env.ADAPTER_DATA_DIR, env.PORT)const runService = rest[0] === 'search' ? runSearchService : rest[0] === 'viewers' ? runViewersService- : rest[0] === 'workspaces' ? runWorkspacesService : null+ : rest[0] === 'workspaces' ? runWorkspacesService : rest[0] === 'teams' ? runTeamsService : nullif (!runService || !socketPath) {console.error(`[service] ${rest[0] ?? '(none)'}: ${socketPath ? 'no such service in this build' : 'the core has no local socket to reach'}`)process.exit(2)diff --git a/cli/src/core/AGENTS.md b/cli/src/core/AGENTS.mdindex 58dddb995..ae341a471 100644--- a/cli/src/core/AGENTS.md+++ b/cli/src/core/AGENTS.md@@ -32,3 +32,4 @@ questions. Everything else is a service ([../services/AGENTS.md](../services/AGE- `api.ts`: the contract with services. `serviceHost.ts`: services in this process.`serviceLinks.ts`: services in their own processes. `viewersLink.ts`: what the core keeps of theviewers when they run in theirs. `workspacesLink.ts`: what the core tells workspaces in theirs.+ `teamsLink.ts`: every change to the teams' prompt scopes, kept until their process has it.diff --git a/cli/src/core/api.ts b/cli/src/core/api.tsindex 1d25db068..a9e9cd9be 100644--- a/cli/src/core/api.ts+++ b/cli/src/core/api.ts@@ -167,6 +167,23 @@ export type TeamsPort = Pick<SwarmPromptScopes, 'prepare' | 'started' | 'raw' |/** What the core gets when the teams fail: the prompt is written with no team recorded for it. */export const TEAMS_FALLBACKS: PortFallbacks<TeamsPort> = { prepare: () => {}, started: undefined, raw: undefined, forget: undefined }+/** The prompt scopes whole, as the socket's team features hold them: the core's calls (`TeamsPort`),+ * and the team a prompt came from (`current`), which an agent's answer to a team's question moves+ * back (`replied`). In the core's process the socket's own; in their own, core/teamsLink.ts. */+export type PromptScopes = TeamsPort & Pick<SwarmPromptScopes, 'current' | 'replied'>++/** A change to the prompt scopes, as the core tells the teams' own process (core/teamsLink.ts,+ * services/teamsProcess.ts): numbered within one core's life, stamped with when it happened, and+ * given to the process in order, once. `raw` bytes travel as base64. */+export type TeamsEvent = { seq: number; core: string; at: number; agentId: string } & (+ | { kind: 'prepare'; text: string; tabId?: string; deliveryId?: string }+ | { kind: 'unprepare'; of: number }+ | { kind: 'started'; text: string; source: 'hook' | 'transcript'; engine?: string }+ | { kind: 'raw'; bytes: string; tabId?: string; pasted: boolean }+ | { kind: 'forget' }+ | { kind: 'replied'; teamId: string; questionId: string }+)+/** Each port is filled by the service that owns it when that service starts, and is null while the* service is off: the core never waits on one. */export interface CorePorts {diff --git a/cli/src/core/teamsLink.spec.ts b/cli/src/core/teamsLink.spec.tsnew file mode 100644index 000000000..d56a04c70--- /dev/null+++ b/cli/src/core/teamsLink.spec.ts@@ -0,0 +1,186 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import type { TeamsEvent } from './api.js'+import { createTeamsLink, KEPT_BYTES, KEPT_EVENTS } from './teamsLink.js'++const TEAM_A = 'a'.repeat(32)+const TEAM_B = 'b'.repeat(32)++function setup(over: Partial<Parameters<typeof createTeamsLink>[0]> = {}) {+ const sent: TeamsEvent[] = []+ let connected = true+ const lines: string[] = []+ let clock = 1_000+ const link = createTeamsLink({+ notify: vi.fn((frame) => { if (connected) sent.push((frame.payload as { event: TeamsEvent }).event); return connected }),+ now: () => clock,+ newId: () => 'core-1',+ log: (line) => lines.push(line),+ ...over,+ })+ return {+ link, scopes: link.scopes, sent, lines,+ down: () => { connected = false },+ up: () => { connected = true },+ tick: (ms: number) => { clock += ms },+ hello: (payload: Record<string, unknown>) => link.answer('hello', payload) as { core: string; reset: boolean; base: number; events: TeamsEvent[] },+ ack: (applied: number, scopes: Record<string, string | null> = {}, core = 'core-1') => link.answer('ack', { core, applied, scopes }),+ }+}++describe('the teams\' prompt scopes in their own process, as the core keeps them', () => {+ afterEach(() => vi.restoreAllMocks())++ it('a write gets its undo at once, never a fault, whether or not the process is there', () => {+ const { scopes, sent, down, hello } = setup()+ down()+ const undo = scopes.prepare('agent-1', 'hello team', 'tab-1', 'team:delivery')+ expect(typeof undo).toBe('function')+ undo()+ expect(sent).toEqual([])+ // Kept for the process all the same: the write and its undo, in order.+ expect(hello({ core: null }).events).toEqual([+ { kind: 'prepare', agentId: 'agent-1', text: 'hello team', tabId: 'tab-1', deliveryId: 'team:delivery', seq: 1, core: 'core-1', at: 1_000 },+ { kind: 'unprepare', agentId: 'agent-1', of: 1, seq: 2, core: 'core-1', at: 1_000 },+ ])+ })++ it('tells the process every change, numbered in order and stamped with when it happened', () => {+ const { scopes, sent, tick } = setup()+ scopes.started('agent-1', 'hello team', 'hook', 'claude')+ tick(5)+ scopes.started('agent-1', 'from the transcript')+ scopes.raw('agent-1', new Uint8Array([104, 105, 13]), 'tab-1')+ scopes.raw('agent-1', new TextEncoder().encode('pasted'), 'tab-1', true)+ scopes.replied('agent-1', TEAM_A, TEAM_B)+ scopes.forget('agent-1')+ expect(sent.map(({ seq, kind, at }) => [seq, kind, at])).toEqual([+ [1, 'started', 1_000], [2, 'started', 1_005], [3, 'raw', 1_005], [4, 'raw', 1_005], [5, 'replied', 1_005], [6, 'forget', 1_005],+ ])+ expect(sent[0]).toMatchObject({ text: 'hello team', source: 'hook', engine: 'claude' })+ expect(sent[1]).toMatchObject({ source: 'transcript' })+ expect(sent[2]).toMatchObject({ bytes: Buffer.from('hi\r').toString('base64'), tabId: 'tab-1', pasted: false })+ expect(sent[3]).toMatchObject({ bytes: Buffer.from('pasted').toString('base64'), pasted: true })+ expect(sent[4]).toMatchObject({ teamId: TEAM_A, questionId: TEAM_B })+ })++ it('answers an agent\'s team as the process reported it, and none while a change to it is on its way', () => {+ const { scopes, ack } = setup()+ expect(scopes.current('agent-1')).toBeNull()+ scopes.prepare('agent-1', 'hello team', 'tab-1')+ scopes.started('agent-1', 'hello team', 'hook')+ // Not yet applied by the process: no team rather than a stale one.+ expect(scopes.current('agent-1')).toBeNull()+ expect(ack(2, { 'agent-1': TEAM_A })).toEqual({ kept: true })+ expect(scopes.current('agent-1')).toBe(TEAM_A)+ // Writes and keys move no team: the answer stands.+ scopes.prepare('agent-1', 'next', 'tab-2')+ scopes.raw('agent-1', new Uint8Array([97]))+ expect(scopes.current('agent-1')).toBe(TEAM_A)+ scopes.started('agent-1', 'next', 'hook')+ expect(scopes.current('agent-1')).toBeNull()+ // A report from before that start is not the agent's team now.+ ack(4, { 'agent-1': TEAM_A })+ expect(scopes.current('agent-1')).toBeNull()+ ack(5, { 'agent-1': TEAM_B })+ expect(scopes.current('agent-1')).toBe(TEAM_B)+ })++ it('forgets an agent the process reports with no team', () => {+ const { scopes, ack } = setup()+ scopes.started('agent-1', 'hello', 'hook')+ ack(1, { 'agent-1': TEAM_A })+ scopes.forget('agent-1')+ ack(2, { 'agent-1': null })+ expect(scopes.current('agent-1')).toBeNull()+ })++ it('takes the report for an agent that was only written to: whatever team it had stands', () => {+ const { scopes, ack } = setup()+ scopes.prepare('agent-2', 'a message', 'tab-1')+ ack(1, { 'agent-2': TEAM_A })+ expect(scopes.current('agent-2')).toBe(TEAM_A)+ })++ it('ignores what another core\'s process acknowledges, and acknowledgements that say nothing', () => {+ const { link, scopes, ack } = setup()+ scopes.started('agent-1', 'hello', 'hook')+ expect(ack(1, { 'agent-1': TEAM_A }, 'core-0')).toEqual({ kept: false })+ expect(link.answer('ack', { core: 'core-1' })).toEqual({ kept: false })+ expect(scopes.current('agent-1')).toBeNull()+ expect(link.answer('ack', { core: 'core-1', applied: 1, scopes: 'nothing' })).toEqual({ kept: true })+ expect(scopes.current('agent-1')).toBeNull()+ })++ it('gives a process that reconnects exactly the events it lacks', () => {+ const { scopes, hello, ack, down, up } = setup()+ scopes.prepare('agent-1', 'one', 'tab-1')+ scopes.started('agent-1', 'one', 'hook')+ ack(1)+ down()+ scopes.prepare('agent-1', 'two', 'tab-1')+ up()+ // It applied the second event before its connection dropped, without saying so yet.+ const answer = hello({ core: 'core-1', applied: 2 })+ expect(answer).toMatchObject({ core: 'core-1', reset: false, base: 2 })+ expect(answer.events.map((event) => event.seq)).toEqual([3])+ })++ it('starts a new process over, with everything this side still holds', () => {+ const { scopes, hello, ack } = setup()+ scopes.started('agent-1', 'one', 'hook')+ ack(1, { 'agent-1': TEAM_A })+ scopes.prepare('agent-2', 'two', 'tab-1')+ expect(scopes.current('agent-1')).toBe(TEAM_A)+ const answer = hello({ core: null, applied: 0 })+ expect(answer).toMatchObject({ reset: true, base: 1 })+ expect(answer.events.map((event) => event.seq)).toEqual([2])+ // What the old process knew went with it: no team, not the one it had.+ expect(scopes.current('agent-1')).toBeNull()+ // With nothing held, it starts after the last event this core numbered.+ ack(2)+ expect(hello({ core: 'core-0', applied: 7 })).toEqual({ core: 'core-1', reset: true, base: 2, events: [] })+ })++ it('starts over a process that claims more than this core said, or missed events this side could not keep', () => {+ const { scopes, hello, lines, down, ack } = setup({ keptEvents: 3 })+ scopes.started('agent-1', 'one', 'hook')+ expect(hello({ core: 'core-1', applied: 5 })).toMatchObject({ reset: true, base: 0 })+ ack(1)+ down()+ for (let n = 0; n < 5; n++) scopes.raw('agent-1', new Uint8Array([97 + n]))+ expect(lines).toEqual(['[teams] kept 3 events or 16777216 bytes for the teams process; it will start over when it is back'])+ const answer = hello({ core: 'core-1', applied: 1 })+ expect(answer).toMatchObject({ reset: true, base: 3 })+ expect(answer.events.map((event) => event.seq)).toEqual([4, 5, 6])+ // Said once an outage: the next one says it again.+ for (let n = 0; n < 4; n++) scopes.raw('agent-1', new Uint8Array([97]))+ expect(lines).toHaveLength(2)+ })++ it('keeps events by size as well as by number, and lets go of what the process has', () => {+ const { scopes, hello, ack, down } = setup({ keptBytes: 2_000 })+ down()+ scopes.prepare('agent-1', 'x'.repeat(1_500), 'tab-1')+ scopes.prepare('agent-1', 'y'.repeat(1_500), 'tab-1')+ expect(hello({ core: null }).events.map((event) => event.seq)).toEqual([2])+ ack(2)+ expect(hello({ core: 'core-1', applied: 2 })).toMatchObject({ reset: false, events: [] })+ })++ it('refuses a question it does not know', () => {+ expect(setup().link.answer('agents', {})).toEqual({ error: 'UNKNOWN_QUERY' })+ })++ it('numbers its events within a life of its own, on the clock, by default', () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const events: TeamsEvent[] = []+ const link = createTeamsLink({ notify: (frame) => { events.push((frame.payload as { event: TeamsEvent }).event); return true }, keptEvents: 1 })+ link.scopes.forget('agent-1')+ link.scopes.forget('agent-2')+ expect(events[0].core).toMatch(/^[0-9a-f-]{36}$/)+ expect(Math.abs(events[0].at - Date.now())).toBeLessThan(5_000)+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('[teams] kept 1 events'))+ expect(KEPT_EVENTS).toBe(10_000)+ expect(KEPT_BYTES).toBe(16 * 1024 * 1024)+ })+})diff --git a/cli/src/core/teamsLink.ts b/cli/src/core/teamsLink.tsnew file mode 100644index 000000000..a86a33fc5--- /dev/null+++ b/cli/src/core/teamsLink.ts@@ -0,0 +1,132 @@+/**+ * The teams' prompt scopes in their own process, as the core sees them (`HARNESSD_SERVICES=teams`; the+ * process's side is services/teamsProcess.ts).+ *+ * A message's write needs one thing from the scopes at once: the undo that `prepare` returns, for a+ * write that fails. Here the core makes that itself, so a write never calls into the process or waits+ * on it, and nothing on this side can throw: teams can never cost a message its write (part 16).+ *+ * Every change (a message prepared or taken back, a prompt started, keys typed into a scoped terminal,+ * an agent forgotten, a question answered) is an event, numbered within this core's life and kept here+ * until the process says it has it. So none is lost to a process that is down, hung or restarting, and+ * the process applies each exactly once, in order:+ * - Each time it connects, it says which core it last heard from and the last event it applied+ * (`hello`). The answer is the events it lacks.+ * - Or the answer tells it to start over: when it is new, when this core is (after a restart, as the+ * scopes in the core's own process did), or when it missed events this side could no longer keep.+ * Starting over loses no event this side still holds; what the old state knew goes, and fails+ * closed.+ *+ * The socket's team features read an agent's scope back (`current`). The process reports each agent's+ * scope as it acknowledges events (`ack`), and the core answers from that report. While a change to an+ * agent's scope is on its way to the process, or after the process started over, the answer is null:+ * no team, never a wrong one. That is the fallback while the process is down, too.+ */+import { randomUUID } from 'node:crypto'+import type { PromptScopes, TeamsEvent } from './api.js'+import type { ServiceFrame } from './serviceLinks.js'++/** The most events kept for a process that has not taken them, and their rough size: past either, the+ * oldest go, and the process starts over when it is back. Typing for half an hour fits. */+export const KEPT_EVENTS = 10_000+export const KEPT_BYTES = 16 * 1024 * 1024++export interface TeamsLinkOptions {+ /** Tell the teams' process something (core/serviceLinks.ts `notify`, for `teams`). */+ notify(frame: ServiceFrame): boolean+ now?: () => number+ newId?: () => string+ log?: (line: string) => void+ keptEvents?: number+ keptBytes?: number+}++type Change = TeamsEvent extends infer E ? E extends TeamsEvent ? Omit<E, 'seq' | 'core' | 'at'> : never : never++export function createTeamsLink(options: TeamsLinkOptions) {+ const now = options.now ?? Date.now+ const log = options.log ?? ((line: string) => console.warn(line))+ const keptEvents = options.keptEvents ?? KEPT_EVENTS+ const keptBytes = options.keptBytes ?? KEPT_BYTES+ /** This core's life: a process that heard another core's events starts over. */+ const core = (options.newId ?? randomUUID)()+ let seq = 0+ /** The last event the process said it applied. */+ let acked = 0+ /** Every event the process has not acknowledged, oldest first, and their rough size. */+ const journal: Array<{ event: TeamsEvent; size: number }> = []+ let journalBytes = 0+ let dropping = false+ /** Each agent's team as the process last reported it; only those with one. */+ const reported = new Map<string, string>()+ /** The last event that could move an agent's team: until the process has it, the team is unknown. */+ const moved = new Map<string, number>()++ const record = (change: Change, movesScope: boolean): number => {+ const event = { ...change, seq: ++seq, core, at: now() } as TeamsEvent+ const size = 256 + ('text' in event ? event.text.length : 0) + ('bytes' in event ? event.bytes.length : 0)+ journal.push({ event, size })+ journalBytes += size+ while (journal.length > keptEvents || journalBytes > keptBytes) {+ journalBytes -= journal.shift()!.size+ if (!dropping) log(`[teams] kept ${keptEvents} events or ${keptBytes} bytes for the teams process; it will start over when it is back`)+ dropping = true+ }+ if (movesScope) moved.set(change.agentId, event.seq)+ options.notify({ type: 'service_event', payload: { kind: 'event', event } })+ return event.seq+ }++ const scopes: PromptScopes = {+ prepare: (agentId, text, tabId, deliveryId) => {+ const of = record({ kind: 'prepare', agentId, text, tabId, deliveryId }, false)+ return () => { record({ kind: 'unprepare', agentId, of }, false) }+ },+ started: (agentId, text, source = 'transcript', engine) => { record({ kind: 'started', agentId, text, source, engine }, true) },+ raw: (agentId, bytes, tabId, pasted = false) => {+ record({ kind: 'raw', agentId, bytes: Buffer.from(bytes).toString('base64'), tabId, pasted }, false)+ },+ forget: (agentId) => { record({ kind: 'forget', agentId }, true) },+ replied: (agentId, teamId, questionId) => { record({ kind: 'replied', agentId, teamId, questionId }, true) },+ current: (agentId) => ((moved.get(agentId) ?? 0) > acked ? null : reported.get(agentId) ?? null),+ }++ /** The process connected: the events it lacks, or everything this side holds and a fresh start. */+ const hello = (payload: Record<string, unknown>): Record<string, unknown> => {+ const theirs = payload.core === core && typeof payload.applied === 'number' ? payload.applied : null+ const first = journal[0]?.event.seq+ const reset = theirs === null || theirs > seq || (first !== undefined && first > theirs + 1)+ if (reset) reported.clear()+ dropping = false+ const base = reset ? (first ?? seq + 1) - 1 : theirs+ return { core, reset, base, events: journal.filter((entry) => entry.event.seq > base).map((entry) => entry.event) }+ }++ /** The process applied every event up to `applied`, and says where each agent it touched stands. */+ const ack = (payload: Record<string, unknown>): Record<string, unknown> => {+ if (payload.core !== core || typeof payload.applied !== 'number') return { kept: false }+ const applied = payload.applied+ acked = Math.max(acked, applied)+ while (journal.length && journal[0].event.seq <= acked) journalBytes -= journal.shift()!.size+ const teams = payload.scopes && typeof payload.scopes === 'object' ? payload.scopes as Record<string, unknown> : {}+ for (const [agentId, teamId] of Object.entries(teams)) {+ // A report older than the agent's last move is not its team: the newer one is on its way.+ if ((moved.get(agentId) ?? 0) > applied) continue+ if (typeof teamId === 'string') reported.set(agentId, teamId)+ else { reported.delete(agentId); moved.delete(agentId) }+ }+ return { kept: true }+ }++ return {+ scopes,+ /** The core's answers to the teams' questions (core/serviceLinks.ts `answer`, for `teams`). */+ answer(query: string, payload: Record<string, unknown>): Record<string, unknown> {+ if (query === 'hello') return hello(payload)+ if (query === 'ack') return ack(payload)+ return { error: 'UNKNOWN_QUERY' }+ },+ }+}++export type TeamsLink = ReturnType<typeof createTeamsLink>diff --git a/cli/src/harnessd/services.ts b/cli/src/harnessd/services.tsindex a191fd663..75132fbf9 100644--- a/cli/src/harnessd/services.ts+++ b/cli/src/harnessd/services.ts@@ -288,8 +288,8 @@ export class ServiceSupervisor {/*** The services this build can run in their own processes, with their memory budgets. Which of them do- * is `HARNESSD_SERVICES` (`search,viewers,workspaces`): off until named, while each one beds in, and the- * core runs a service that is not out here in its own process, as before.+ * is `HARNESSD_SERVICES` (`search,viewers,workspaces,teams`): off until named, while each one beds in,+ * and the core runs a service that is not out here in its own process, as before.*/export const KNOWN_SERVICES: Readonly<Record<string, Omit<ServiceSpec, 'name'>>> = {search: { heapLimitMiB: 1_024, rssLimitMiB: 2_048 },@@ -298,6 +298,8 @@ export const KNOWN_SERVICES: Readonly<Record<string, Omit<ServiceSpec, 'name'>>>viewers: { heapLimitMiB: 512, rssLimitMiB: 1_024 },// The git work runs in git's own processes; this one only holds the agents the core sent it.workspaces: { heapLimitMiB: 256, rssLimitMiB: 512 },+ // The prompt scopes hold a few drafts and fingerprints per agent: small, bounded state.+ teams: { heapLimitMiB: 256, rssLimitMiB: 512 },}/** Service timings from the environment (for tests and support); anything unset or invalid keeps its default. */diff --git a/cli/src/services/AGENTS.md b/cli/src/services/AGENTS.mdindex 0d500afd5..54f20dd0b 100644--- a/cli/src/services/AGENTS.md+++ b/cli/src/services/AGENTS.md@@ -39,8 +39,8 @@ let several people build features at once without touching the core or each othe## Running in its own processA service that can crash natively, hang or leak should run in its own process, where it costs only-itself. Search, the viewers and workspaces do: `HARNESSD_SERVICES=search,viewers,workspaces` (any of-them alone, too).+itself. Search, the viewers, workspaces and the teams' prompt scopes do:+`HARNESSD_SERVICES=search,viewers,workspaces,teams` (any of them alone, too).- `src/harnessd/services.ts` runs it (`KNOWN_SERVICES`, with its memory budget). The core routes therequests it declared (its `<NAME>_REQUESTS`) to its process, and the same handlers answer them there.@@ -61,6 +61,12 @@ them alone, too).when to sweep. A command that destroys something is never held or replayed, and what it must know(the folders in use) is asked for when it starts, never sent ahead where it could go stale.`e2e/workspacesProcess.e2e.ts` proves it.+- State built from every change must get every change exactly once. The teams show how+ (`src/services/teamsProcess.ts`, `src/core/teamsLink.ts`): the core numbers each change and keeps it+ until the process acknowledges it; on each connection the process says what it applied and gets+ what it lacks, or starts over. A value the core reads back synchronously is answered from what the+ process last reported, and is "unknown" (the fallback) while a change to it is on its way.+ `e2e/teamsProcess.e2e.ts` proves it.## Do notdiff --git a/cli/src/services/teamsProcess.spec.ts b/cli/src/services/teamsProcess.spec.tsnew file mode 100644index 000000000..e679cdec4--- /dev/null+++ b/cli/src/services/teamsProcess.spec.ts@@ -0,0 +1,237 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import type { PromptScopes, TeamsEvent } from '../core/api.js'+import { channelTeamId } from '../teams/service.js'+import { runServiceProcess, type CoreConnection, type ServiceProcessOptions } from './process.js'+import { KEPT_UNDOS, runTeamsService } from './teamsProcess.js'++// The real default reaches a real socket and this process's own channel: never in a test.+vi.mock('./process.js', () => ({ runServiceProcess: vi.fn(() => ({ stop: vi.fn() })) }))++const TEAM = channelTeamId('tab-1')+const OTHER = channelTeamId('tab-2')+const Q = 'c'.repeat(32)+const T0 = 1_000_000++/** An event of `core-1`, as the core numbers and stamps it. */+const ev = (seq: number, change: Record<string, unknown>, at = T0): TeamsEvent =>+ ({ agentId: 'agent-1', ...change, seq, core: 'core-1', at }) as TeamsEvent+const flush = () => new Promise((done) => setTimeout(done, 0))++describe('the teams\' prompt scopes in their own process', () => {+ afterEach(() => vi.clearAllMocks())++ const setup = (over: Partial<Parameters<typeof runTeamsService>[0]> = {}) => {+ let options: ServiceProcessOptions | null = null+ const service = { stop: vi.fn() }+ const lines: string[] = []+ const handle = runTeamsService({+ dataDir: '/data', socketPath: '/data/daemon-1.sock', machineId: 'm', token: 't',+ run: (given) => { options = given; return service },+ log: (line) => lines.push(line),+ ...over,+ })+ return { options: options!, service, handle, lines }+ }+ /** A core that answers `hello` with `answer`, and records every acknowledgement. */+ const connection = (answer: Record<string, unknown> | (() => Promise<Record<string, unknown>>) = { core: 'core-1', reset: true, base: 0, events: [] }) => {+ const query = vi.fn((name: string, _payload?: Record<string, unknown>) => name === 'hello'+ ? (typeof answer === 'function' ? answer() : Promise.resolve(answer))+ : Promise.resolve({ kept: true }))+ const acks = () => query.mock.calls.filter(([name]) => name === 'ack').map(([, payload]) => payload)+ const hellos = () => query.mock.calls.filter(([name]) => name === 'hello').map(([, payload]) => payload)+ return { core: { query } satisfies CoreConnection, query, acks, hellos }+ }+ /** Connected, and caught up with `events`. */+ const synced = async (options: ServiceProcessOptions, events: TeamsEvent[] = []) => {+ const core = connection({ core: 'core-1', reset: true, base: 0, events })+ options.onConnected!(core.core)+ await flush()+ return core+ }++ it('reaches the core as `teams`, through the socket and token it was given, and answers the apps nothing', () => {+ const { options } = setup()+ expect(options).toMatchObject({ name: 'teams', socketPath: '/data/daemon-1.sock', machineId: 'm', token: 't' })+ expect(options.requests).toEqual({})+ })++ it('first asks for what it lacks, applies it, then says where each agent stands', async () => {+ const { options } = setup()+ const core = await synced(options, [+ ev(1, { kind: 'prepare', text: 'hello team', tabId: 'tab-1' }),+ ev(2, { kind: 'started', text: 'hello team', source: 'hook', engine: 'claude' }),+ ])+ expect(core.hellos()).toEqual([{ core: null, applied: 0 }])+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 2, scopes: { 'agent-1': TEAM } }])+ })++ it('applies what comes next in order, once: skips what it has, and asks again for what went missing', async () => {+ const { options } = setup()+ const core = await synced(options, [ev(1, { kind: 'prepare', text: 'hello team', tabId: 'tab-1' })])+ // Already applied, from another core, not a numbered event, or not an event at all: nothing to do.+ options.onEvent!({ kind: 'event', event: ev(1, { kind: 'prepare', text: 'hello team', tabId: 'tab-1' }) })+ options.onEvent!({ kind: 'event', event: { ...ev(2, { kind: 'forget' }), core: 'core-0' } })+ options.onEvent!({ kind: 'event', event: { ...ev(2, { kind: 'forget' }), seq: 'two' } })+ options.onEvent!({ kind: 'event' })+ options.onEvent!({ kind: 'touch', sessionId: 's1' })+ options.onEvent!({ kind: 'event', event: ev(2, { kind: 'started', text: 'hello team', source: 'hook' }) })+ await flush()+ expect(core.acks().at(-1)).toEqual({ core: 'core-1', applied: 2, scopes: { 'agent-1': TEAM } })+ // The fourth before the third: ask the core again, from where it is.+ options.onEvent!({ kind: 'event', event: ev(4, { kind: 'forget' }) })+ await flush()+ expect(core.hellos().at(-1)).toEqual({ core: 'core-1', applied: 2 })+ })++ it('applies nothing before its first question is answered: the answer brings it all', async () => {+ const { options } = setup()+ let answer: (value: Record<string, unknown>) => void = () => {}+ const core = connection(() => new Promise((done) => { answer = done }))+ options.onConnected!(core.core)+ options.onEvent!({ kind: 'event', event: ev(1, { kind: 'prepare', text: 'early', tabId: 'tab-1' }) })+ answer({ core: 'core-1', reset: true, base: 0, events: [ev(1, { kind: 'prepare', text: 'early', tabId: 'tab-1' }), ev(2, { kind: 'started', text: 'early', source: 'hook' })] })+ await flush()+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 2, scopes: { 'agent-1': TEAM } }])+ })++ it('ignores an answer it cannot use: an older connection\'s, or one that says nothing', async () => {+ const { options } = setup()+ let older: (value: Record<string, unknown>) => void = () => {}+ options.onConnected!(connection(() => new Promise((done) => { older = done })).core)+ const current = connection({ core: 'core-1' })+ options.onConnected!(current.core)+ older({ core: 'core-1', reset: true, base: 0, events: [] })+ for (const answer of [{ core: 'core-1', base: 0 }, { core: 'core-1', events: [] }, { base: 0, events: [] }]) {+ options.onConnected!(connection(answer).core)+ }+ options.onConnected!(connection(async () => { throw new Error('the core went away') }).core)+ await flush()+ // Never caught up: what comes is not applied.+ options.onEvent!({ kind: 'event', event: ev(1, { kind: 'forget' }) })+ expect(current.acks()).toEqual([])+ })++ it('applies a replay only from where it stands: out of place and foreign events are not its', async () => {+ const { options } = setup()+ const core = await synced(options, [+ null as unknown as TeamsEvent,+ ev(1, { kind: 'prepare', text: 'one', tabId: 'tab-1' }),+ { ...ev(2, { kind: 'started', text: 'one', source: 'hook' }), core: 'core-0' },+ ev(3, { kind: 'started', text: 'one', source: 'hook' }),+ ])+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 1, scopes: { 'agent-1': null } }])+ })++ it('takes back a write that failed, so the prompt it would have scoped finds no team', async () => {+ const { options } = setup()+ const core = await synced(options, [+ ev(1, { kind: 'prepare', text: 'never written', tabId: 'tab-1' }),+ ev(2, { kind: 'unprepare', of: 1 }),+ // An undo for a write it never saw, or saw taken back already, does nothing.+ ev(3, { kind: 'unprepare', of: 1 }),+ ev(4, { kind: 'unprepare', of: 99 }),+ ev(5, { kind: 'started', text: 'never written', source: 'hook' }),+ ])+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 5, scopes: { 'agent-1': null } }])+ })++ it('keeps a bounded number of undos', async () => {+ const { options } = setup()+ const writes = Array.from({ length: KEPT_UNDOS + 1 }, (_, n) => ev(n + 1, { kind: 'prepare', text: `m${n}`, tabId: 'tab-1', agentId: `agent-${n}` }))+ const core = await synced(options, [...writes, ev(KEPT_UNDOS + 2, { kind: 'unprepare', of: 1, agentId: 'agent-0' })])+ expect(core.acks()[0]).toMatchObject({ applied: KEPT_UNDOS + 2 })+ })++ it('applies each event at the time it happened, so a replay ages as the original would have', async () => {+ const minute = 60_000+ const { options } = setup()+ const core = await synced(options, [+ ev(1, { kind: 'prepare', text: 'quick', tabId: 'tab-1', agentId: 'quick' }, T0),+ ev(2, { kind: 'started', text: 'quick', source: 'hook', agentId: 'quick' }, T0 + minute),+ ev(3, { kind: 'prepare', text: 'slow', tabId: 'tab-2', agentId: 'slow' }, T0),+ ev(4, { kind: 'started', text: 'slow', source: 'hook', agentId: 'slow' }, T0 + 6 * minute),+ ])+ // Past five minutes, a message is no longer matched to the prompt that starts.+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 4, scopes: { quick: TEAM, slow: null } }])+ })++ it('reads keys typed into a scoped terminal, and answers to a team\'s question', async () => {+ const { options } = setup()+ const question = `team:${OTHER}:${Q}:question`+ const core = await synced(options, [+ ev(1, { kind: 'raw', bytes: Buffer.from('typed\r').toString('base64'), tabId: 'tab-1', pasted: false }),+ ev(2, { kind: 'started', text: 'typed', source: 'hook' }),+ ev(3, { kind: 'prepare', text: 'a question', deliveryId: question, agentId: 'asked' }),+ ev(4, { kind: 'started', text: 'a question', source: 'transcript', agentId: 'asked' }),+ ])+ expect(core.acks().at(-1)).toMatchObject({ scopes: { 'agent-1': TEAM, asked: OTHER } })+ options.onEvent!({ kind: 'event', event: ev(5, { kind: 'replied', teamId: OTHER, questionId: Q, agentId: 'asked' }) })+ options.onEvent!({ kind: 'event', event: ev(6, { kind: 'forget' }) })+ await flush()+ expect(core.acks().at(-1)).toEqual({ core: 'core-1', applied: 6, scopes: { asked: null, 'agent-1': null } })+ })++ it('starts over when the core says so, and goes on from where the core says', async () => {+ const { options } = setup()+ const first = await synced(options, [ev(1, { kind: 'prepare', text: 'hello', tabId: 'tab-1' })])+ expect(first.acks()).toHaveLength(1)+ // The same core, reconnected: nothing lost, nothing over.+ const again = connection({ core: 'core-1', reset: false, base: 1, events: [ev(2, { kind: 'started', text: 'hello', source: 'hook' })] })+ options.onConnected!(again.core)+ await flush()+ expect(again.hellos()).toEqual([{ core: 'core-1', applied: 1 }])+ expect(again.acks()).toEqual([{ core: 'core-1', applied: 2, scopes: { 'agent-1': TEAM } }])+ // A new core: the old scopes go.+ const fresh = connection({ core: 'core-2', reset: true, base: 0, events: [{ ...ev(1, { kind: 'started', text: 'hello', source: 'hook' }), core: 'core-2' }] })+ options.onConnected!(fresh.core)+ await flush()+ expect(fresh.hellos()).toEqual([{ core: 'core-1', applied: 2 }])+ expect(fresh.acks()).toEqual([{ core: 'core-2', applied: 1, scopes: { 'agent-1': null } }])+ })++ it('goes on past an event it cannot apply, and one of a kind it does not know', async () => {+ const { options, lines } = setup()+ const core = await synced(options, [+ ev(1, { kind: 'prepare', tabId: 'tab-1' }),+ ev(2, { kind: 'newer-kind' }),+ ev(3, { kind: 'forget' }),+ ])+ expect(lines).toEqual([expect.stringMatching(/^\[teams\] event 1 could not be applied · /)])+ expect(core.acks()).toEqual([{ core: 'core-1', applied: 3, scopes: { 'agent-1': null } }])+ const thrown = setup({ scopes: () => ({ ...fakeScopes(), forget: () => { throw 'gone' } }) })+ await synced(thrown.options, [ev(1, { kind: 'forget' })])+ expect(thrown.lines).toEqual(['[teams] event 1 could not be applied · gone'])+ })++ it('says how far it got once per burst, and shrugs off a core that went away', async () => {+ const { options } = setup()+ const core = await synced(options)+ for (let seq = 1; seq <= 3; seq++) options.onEvent!({ kind: 'event', event: ev(seq, { kind: 'raw', bytes: 'YQ==', tabId: 'tab-1', pasted: false }) })+ await flush()+ expect(core.acks()).toHaveLength(2)+ expect(core.acks()[1]).toEqual({ core: 'core-1', applied: 3, scopes: { 'agent-1': null } })+ core.query.mockImplementation(async () => { throw new Error('the core went away') })+ options.onEvent!({ kind: 'event', event: ev(4, { kind: 'forget' }) })+ await flush()+ })++ it('stops its connection when stopped', () => {+ const { handle, service } = setup()+ handle.stop()+ expect(service.stop).toHaveBeenCalledOnce()+ })++ it('runs as a real service process by default, logging on the console', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ runTeamsService({ dataDir: '/data', socketPath: '/data/daemon-1.sock', machineId: 'm', token: 't' })+ const options = vi.mocked(runServiceProcess).mock.calls.at(-1)![0]+ expect(options).toMatchObject({ name: 'teams', socketPath: '/data/daemon-1.sock' })+ await synced(options, [ev(1, { kind: 'started', source: 'hook' })])+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('[teams] event 1 could not be applied'))+ warn.mockRestore()+ })+})++function fakeScopes(): PromptScopes {+ return { prepare: () => () => {}, started: () => {}, raw: () => {}, forget: () => {}, replied: () => {}, current: () => null }+}diff --git a/cli/src/services/teamsProcess.ts b/cli/src/services/teamsProcess.tsnew file mode 100644index 000000000..151a74138--- /dev/null+++ b/cli/src/services/teamsProcess.ts@@ -0,0 +1,132 @@+/**+ * The teams' prompt scopes in their own process (`harness __service teams`, with `HARNESSD_SERVICES=teams`).+ *+ * The same scopes as in the core's process (teams/promptScope.ts), fed the core's changes as numbered+ * events (core/teamsLink.ts) and applied here in order, exactly once:+ * - An event this process already has is skipped.+ * - One that arrives too early (the one before it went missing) makes it ask the core again.+ * - Each connection starts by telling the core the last event it applied. The core answers with the+ * events it lacks, or tells it to start over.+ *+ * Each event is applied at the time it happened, so one replayed later ages as it would have: a message+ * stays matchable for the same five minutes. After applying events, the process tells the core where+ * each agent it touched stands. That is what the socket's team features read back.+ */+import type { PromptScopes, TeamsEvent } from '../core/api.js'+import { SwarmPromptScopes } from '../teams/promptScope.js'+import { runServiceProcess, type CoreConnection, type ServiceProcess } from './process.js'++export interface TeamsServiceOptions {+ dataDir: string+ socketPath: string+ machineId: string+ token: string+ /** Swapped in tests for one that does not touch the real socket or process. */+ run?: typeof runServiceProcess+ /** Swapped in tests for scopes they can watch; given the clock to read. */+ scopes?: (now: () => number) => PromptScopes+ log?: (line: string) => void+}++/** The most undos kept for writes that may yet fail. A write fails within seconds; these are minutes. */+export const KEPT_UNDOS = 1_024++export function runTeamsService(options: TeamsServiceOptions): ServiceProcess {+ const log = options.log ?? ((line: string) => console.warn(line))+ const make = options.scopes ?? ((now: () => number) => new SwarmPromptScopes(now))+ /** The time of the event being applied: every call into the scopes is made from one. */+ let clock = 0+ const now = (): number => clock+ let scopes = make(now)+ /** The core whose events the scopes hold, and the last of them applied. */+ let core: string | null = null+ let applied = 0+ let connection: CoreConnection | null = null+ /** The connection whose first question is answered: events may be applied as they come on it. */+ let synced: CoreConnection | null = null+ /** Each prepared write's undo, by its event: an unprepare names the write it takes back. */+ const undos = new Map<number, () => void>()+ const touched = new Set<string>()+ let acking = false++ /** One event, in its place: counted as applied even when it cannot be, so the stream goes on. */+ const apply = (event: TeamsEvent): void => {+ applied = event.seq+ clock = event.at+ try {+ switch (event.kind) {+ case 'prepare':+ undos.set(event.seq, scopes.prepare(event.agentId, event.text, event.tabId, event.deliveryId))+ if (undos.size > KEPT_UNDOS) undos.delete(undos.keys().next().value!)+ break+ case 'unprepare': {+ const undo = undos.get(event.of)+ undos.delete(event.of)+ undo?.()+ break+ }+ case 'started': scopes.started(event.agentId, event.text, event.source, event.engine); break+ case 'raw': scopes.raw(event.agentId, Buffer.from(event.bytes, 'base64'), event.tabId, event.pasted); break+ case 'forget': scopes.forget(event.agentId); break+ case 'replied': scopes.replied(event.agentId, event.teamId, event.questionId); break+ // A kind a newer core says: nothing this process knows how to do.+ default: break+ }+ touched.add(event.agentId)+ } catch (error) {+ log(`[teams] event ${event.seq} could not be applied · ${error instanceof Error ? error.message : String(error)}`)+ }+ }++ /** Tell the core how far this process got, and where each agent it touched stands; once per burst. */+ const acknowledge = (via: CoreConnection): void => {+ if (acking) return+ acking = true+ queueMicrotask(() => {+ acking = false+ const report = Object.fromEntries([...touched].map((agentId) => [agentId, scopes.current(agentId)]))+ touched.clear()+ void via.query('ack', { core, applied, scopes: report }).catch(() => {})+ })+ }++ /** Ask the core for what this process lacks, and apply it before anything new. */+ const sync = (via: CoreConnection): void => {+ synced = null+ void via.query('hello', { core, applied }).then((answer) => {+ if (connection !== via || typeof answer.core !== 'string' || typeof answer.base !== 'number' || !Array.isArray(answer.events)) return+ if (answer.reset === true) {+ scopes = make(now)+ undos.clear()+ }+ core = answer.core+ applied = answer.base+ for (const event of answer.events as TeamsEvent[]) if (event?.core === core && event.seq === applied + 1) apply(event)+ synced = via+ acknowledge(via)+ }, () => {})+ }++ const run = options.run ?? runServiceProcess+ const service = run({+ name: 'teams',+ socketPath: options.socketPath,+ machineId: options.machineId,+ token: options.token,+ // The apps ask teams nothing here: the team features answer them in the core's socket.+ requests: {},+ onEvent: (payload) => {+ const event = payload.event as TeamsEvent | undefined+ if (payload.kind !== 'event' || !synced || event?.core !== core || typeof event.seq !== 'number' || event.seq <= applied) return+ // The one before it never came: ask the core again rather than skip it.+ if (event.seq !== applied + 1) { sync(synced); return }+ apply(event)+ acknowledge(synced)+ },+ onConnected: (via) => {+ connection = via+ sync(via)+ },+ })+ return { stop: () => service.stop() }+}diff --git a/docs/design/2026-10-03-harnessd.md b/docs/design/2026-10-03-harnessd.mdindex ae207e4b4..07a3f2b6b 100644--- a/docs/design/2026-10-03-harnessd.md+++ b/docs/design/2026-10-03-harnessd.md@@ -493,6 +493,18 @@ Done so far:loop and a core restart each cost workspaces alone. It also proves that only the process renames abranch and the new name reaches the agent's frame, and that the sweep removes an idle unusedworktree, keeps one an agent works in, and skips the one decided while the process was down.+- **The teams' prompt scopes in their own process** (`HARNESSD_SERVICES=teams`, alone or beside the+ others). A message's write needs one thing from them at once: the undo for a write that fails. The+ core makes it itself, so a write never calls into the process and teams can never cost a message its+ write. Every change (a message prepared or taken back, a prompt started, keys typed into a scoped+ terminal, an agent forgotten, a question answered) is numbered and kept in the core until the+ process acknowledges it (`cli/src/core/teamsLink.ts`). On each connection the process says what it+ applied and gets the rest, applied in order, once, at the time each change happened; a new core, a+ fresh process or a gap starts it over. The team features read an agent's scope from what the+ process last reported, and read no team, never a wrong one, while a change to it is on its way.+ `cli/e2e/teamsProcess.e2e.ts` proves on the real daemon that a stop, a hang, a kill, a crash loop+ and a core restart each cost teams alone: every message is written, and a change made meanwhile+ lands once the process is back.- **Services answer their own requests** (core/serviceHost.ts `route`). A service declares the frametypes it answers (`<NAME>_REQUESTS`) and returns their handlers from its start. The socket handssuch a request to it before its own switch, with who asked (`Asker`: local, owner), and never