Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

The viewers serve themselves to remote clients: the forwarder and the rendered surfaces leave the core (#922)

Deef59526ede0f0
diff --git a/cli/e2e/viewersProcess.e2e.ts b/cli/e2e/viewersProcess.e2e.ts
index f773dccc1..399535c97 100644
--- a/cli/e2e/viewersProcess.e2e.ts
+++ b/cli/e2e/viewersProcess.e2e.ts
@@ -7,11 +7,17 @@
* working, and the master starts them again, which stops the viewer server the dead process left behind
* and starts a new one whose URL replaces the old. A core that restarts leaves the viewers to their
* process: the viewer keeps its URL, and the new core answers with it.
+ *
+ * A client that cannot reach this machine's loopback (the phone, another machine's window) is served the
+ * viewer over its own connection (`viewer_request` and the stream after it) by the viewers' process too,
+ * through the core (core/viewerStreams.ts): refused at once while the viewers are down, and served again
+ * once the master has brought them back.
*/
import { execFileSync } from 'node:child_process'
import { mkdirSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { randomUUID } from 'node:crypto'
import { LocalClient, type Frame } from './harness/client.js'
import { IsolatedDaemon, until } from './harness/daemon.js'
@@ -95,6 +101,25 @@ const serves = async (url: string): Promise<string | null> => {
return response?.ok ? response.text() : null
}
+/** A viewer's page as a client that cannot reach this machine's loopback fetches it: over its connection,
+ * answered by whoever serves the viewers. Its status and body, or why the stream was closed. */
+async function overConnection(client: LocalClient, agentId: string, url: string): Promise<{ status?: number; body: string; closed?: string }> {
+ const streamId = randomUUID()
+ const mine = (frame: Frame) => frame.payload?.streamId === streamId
+ const since = client.frames.length
+ const ended = client.next((frame) => mine(frame) && (frame.type === 'viewer_end' || frame.type === 'viewer_close'), 15_000, `the end of viewer stream ${streamId}`)
+ client.send('viewer_request', { streamId, agentId, origin: new URL(url).origin, path: '/', method: 'GET', headers: {}, upgrade: false })
+ // A GET has no body: the request's own end, as the window's proxy sends it.
+ client.send('viewer_end', { streamId })
+ const last = await ended
+ const frames = client.frames.slice(since).filter(mine)
+ const body = Buffer.concat(frames.filter((frame) => frame.type === 'viewer_data').map((frame) => Buffer.from(String(frame.payload.data), 'base64'))).toString()
+ const status = frames.find((frame) => frame.type === 'viewer_response')?.payload.status
+ return { status, body, ...(last.type === 'viewer_close' ? { closed: String(last.payload.error) } : {}) }
+}
+const connects = (d: IsolatedDaemon) => [...d.log().matchAll(/\[services\] viewers connected/g)].length
+const disconnects = (d: IsolatedDaemon) => [...d.log().matchAll(/\[services\] viewers disconnected/g)].length
+
describe('the DSH viewers in their own process', () => {
let daemon: IsolatedDaemon | undefined
/** Every daemon a test started, closed by the test or not: none may leave a viewer server behind. */
@@ -108,11 +133,12 @@ describe('the DSH viewers in their own process', () => {
try { process.kill(pid, 'SIGKILL') } catch { /* gone */ }
}
})
- const fresh = async (services: string) => {
+ const fresh = async (services: string, env: Record<string, string> = {}) => {
const d = await IsolatedDaemon.create({ env: {
HARNESSD_SERVICES: services,
HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '200',
HARNESSD_SERVICE_MAX_BACKOFF_MS: '1000',
+ ...env,
} })
daemon = d
started.push(d)
@@ -126,10 +152,46 @@ describe('the DSH viewers in their own process', () => {
const d = await fresh('none')
const client = await LocalClient.connect(d)
const sketch = await create(d, client, 'sketch-default', { dsh: HARNESS })
- const answer = await serves(await viewerUrl(client, sketch.id))
+ const url = await viewerUrl(client, sketch.id)
+ const answer = await serves(url)
expect(answer).toMatch(/^sketch viewer \d+$/)
expect(ancestors(Number(answer!.split(' ').pop()))).toContain(d.corePid())
expect(viewersPid(d)).toBeNull()
+ // Served over a client's connection from the core's process as well.
+ expect(await overConnection(client, sketch.id, url)).toEqual({ status: 200, body: answer })
+ client.close()
+ })
+
+ it('serves a viewer over a client\'s connection from the viewers\' process, refuses it at once while they are down, and serves it again once they are back', async () => {
+ // Long enough a pause before the master starts them again to ask while they are down.
+ const d = await fresh('viewers', { HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '4000', HARNESSD_SERVICE_MAX_BACKOFF_MS: '4000' })
+ const client = await LocalClient.connect(d)
+ const sketch = await create(d, client, 'sketch-streamed', { dsh: HARNESS })
+ const before = await viewerUrl(client, sketch.id)
+ const page = await overConnection(client, sketch.id, before)
+ expect(page).toEqual({ status: 200, body: expect.stringMatching(/^sketch viewer \d+$/) })
+ expect(ancestors(Number(page.body.split(' ').pop()))).toContain(viewersPid(d))
+ // A rendered surface's question reaches the viewers too (closing one needs no browser).
+ expect(await client.request('viewer_surface', { surfaceId: 'e2e', agentId: sketch.id, op: 'close' })).toMatchObject({ closed: true })
+
+ // Killed: until the master starts them again, a stream is refused at once, never left waiting.
+ const gone = disconnects(d)
+ process.kill(viewersPid(d)!, 'SIGKILL')
+ await until('the core to see the viewers go', () => disconnects(d) > gone || null, 10_000, 50)
+ const startedAt = Date.now()
+ expect(await overConnection(client, sketch.id, before)).toEqual({ body: '', closed: 'Viewer is no longer available' })
+ expect(await client.request('viewer_surface', { surfaceId: 'e2e', agentId: sketch.id, op: 'close' }))
+ .toMatchObject({ error: 'SERVICE_UNAVAILABLE', service: 'viewers' })
+ expect(Date.now() - startedAt).toBeLessThan(2_000)
+ expect(restarts(d)).toBe(0)
+ await turn(client, sketch.id, 'while its viewers were down')
+
+ // Back: a new viewer, served over the connection again.
+ await until('the master to restart the viewers', () => restarts(d) >= 1 || null, 30_000, 200)
+ await until('the viewers to reconnect', () => connects(d) >= 2 || null, 30_000, 200)
+ const after = await viewerUrl(client, sketch.id, before)
+ expect(await overConnection(client, sketch.id, after)).toEqual({ status: 200, body: expect.stringMatching(/^sketch viewer \d+$/) })
+ expect(d.coresStarted()).toBe(1)
client.close()
})
diff --git a/cli/src/architecture.spec.ts b/cli/src/architecture.spec.ts
index f52de5757..907263a0b 100644
--- a/cli/src/architecture.spec.ts
+++ b/cli/src/architecture.spec.ts
@@ -159,8 +159,10 @@ const RUN_FOREGROUND_BUDGET = 2_230
* gateway's sessions (lib/autonomous-device/overGateway.ts): 1,460. Models' grid commands, grid name and
* lists left it for the models service (step 7): 1,391. Then 1,200 when the Devices tab's requests
* became the devices' own (step 9, D1), 1,207 → 1,194. Then 1,195 when the Wi-Fi device's relay left
- * it for the devices' process (step 9, D3): the socket hands its sessions' events on (core/wifi.ts). */
-const BACKEND_SOCKET_BUDGET = 1_195
+ * it for the devices' process (step 9, D3): the socket hands its sessions' events on (core/wifi.ts). Then
+ * 1,182 when the viewer forwarder and the interactive viewers it held left for the viewers
+ * (core/viewerStreams.ts). */
+const BACKEND_SOCKET_BUDGET = 1_185
/** Exceptions, each with its reason. Keep this short. */
const SERVICE_MAY_IMPORT: Record<string, string> = {
@@ -279,7 +281,13 @@ const SERVICE_MAY_IMPORT: Record<string, string> = {
// (core/serviceHost.ts, core/serviceLinks.ts, the socket's close paths), held requests for an experiment
// still starting included: what lets a service keep work per connection and stop it when its asker goes,
// which held the command bar's two doors in the core (step 4).
-const CORE_CLOSURE_BUDGET = 75_590
+//
+// Then at 74,974 in 342, from 75,493 in 344: what serves this machine's viewers to a client over its
+// connection (lib/viewerForwarder.ts, lib/interactiveViewer.ts and the stream they run on, lib/viewerWire.ts)
+// runs in the viewers' process, beside the viewer servers it forwards to (services/viewers.ts), and with it
+// the headless browser capture the surfaces render with (sharing/viewer.ts), Share's other user. The core
+// keeps the frame types it gates (lib/viewerFrames.ts) and hands each frame on (core/viewerStreams.ts).
+const CORE_CLOSURE_BUDGET = 75_070
/** What is not the core's, by path: each goes to a service or its own process, in the plan's order. */
const EDGE: RegExp[] = [
@@ -289,6 +297,8 @@ const EDGE: RegExp[] = [
// The relay's own parts, the gateway's alone: the windows' sessions to other machines, P2P and STUN, the
// remote viewers' proxy, and the shaping of what goes up the link.
/^lib\/(remoteRelay|terminalP2p|stunSelect|remoteViewerProxy|deviceRecentTrim|commanderReplay)\.ts$/,
+ // The viewers' own: a viewer served to a client over its connection, and the stream it runs on.
+ /^lib\/(viewerForwarder|interactiveViewer|viewerWire)\.ts$/,
// The Store's and the viewers' parts of dsh; the launch path (installed, manifest, launch, runtime, …) is the core's.
/^dsh\/(catalog|install|update|updates|registry|wire|service|lock|builtins|viewer|viewerLedger|verdict|artifacts)\.ts$/,
// Search's index; the readers of other engines' sessions (external.ts, externals/) are the core's, for adoption.
@@ -310,7 +320,6 @@ const CORE_MAY_REACH: Record<string, string> = {
'lib/sessionSearch/transcript.ts': 'the readers of other engines\' sessions keep this helper: it moves beside them, out of search\'s folder',
'lib/sessionSearch/turns.ts': 'the handoff (lib/agentHandoff.ts), in the edge host once CoreApi gives it the stopped agents, recaps and discovery it reads',
'services/shell.ts': 'shell setup and launch receipts, in the edge host; only the argv launch stays in the core (#893)',
- 'sharing/viewer.ts': 'the windows\' interactive viewers capture with it (lib/interactiveViewer.ts): headless Chrome belongs with the viewers\' process (plan, "Sharing")',
}
describe('the daemon\'s shape', () => {
diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.ts
index 352433f26..9ed916f61 100644
--- a/cli/src/backendSocket.spec.ts
+++ b/cli/src/backendSocket.spec.ts
@@ -420,7 +420,8 @@ describe('viewer forwarding authentication', () => {
it('allows interactive viewers only on a sealed owner web connection or trusted loopback', async () => {
const socket = relaySocket('token')
const internals = socket as any
- const request = vi.spyOn(socket.interactiveViewers, 'request').mockResolvedValue({ data: 'jpeg' })
+ const request = vi.fn(async () => ({ data: 'jpeg' }))
+ socket.viewerStreams = { frame: vi.fn(), surface: request, closed: vi.fn(), closedAll: vi.fn() }
vi.spyOn(gatewayOf(socket).e2ee, 'hasSession').mockReturnValue(true)
const role = vi.spyOn(gatewayOf(socket).e2ee, 'sessionRole').mockReturnValue('web')
const clear = { type: 'viewer_surface', payload: { requestId: 'one', surfaceId: 'surface', agentId: 'a', op: 'frame' } }
@@ -439,14 +440,46 @@ describe('viewer forwarding authentication', () => {
socket.registerLocalClient('local:viewer', { sendFrame: () => true, sendBinary: () => true })
await dispatchDown(socket, clear, 'local:viewer')
expect(request).toHaveBeenCalledWith('local:viewer', clear.payload)
+ // With nothing to render it, the surface is answered unavailable rather than left unanswered.
+ const local = vi.fn(() => true)
+ socket.registerLocalClient('local:none', { sendFrame: local, sendBinary: () => true })
+ socket.viewerStreams = null
+ await dispatchDown(socket, { ...clear, payload: { ...clear.payload, requestId: 'two' } }, 'local:none')
+ await vi.waitFor(() => expect(local).toHaveBeenCalledWith({ type: 'viewer_surface_result', payload: { requestId: 'two', error: 'VIEWER_UNAVAILABLE' } }))
await socket.unregisterLocalClient('local:viewer')
await socket.stop()
})
+ it('hands a viewer stream\'s answers to the one connection that opened it, and ends its streams as connections go', async () => {
+ const socket = relaySocket('token')
+ const streams = { frame: vi.fn(), surface: vi.fn(), closed: vi.fn(), closedAll: vi.fn() }
+ socket.viewerStreams = streams
+ const local = vi.fn(() => true)
+ socket.registerLocalClient('local:viewer', { sendFrame: local, sendBinary: () => true })
+ expect(socket.sendViewerFrame('local:viewer', 'viewer_response', { streamId: 's1', status: 200 })).toBe(true)
+ expect(local).toHaveBeenCalledWith({ type: 'viewer_response', payload: { streamId: 's1', status: 200 } })
+ // A remote connection's go through the gateway, which says whether it could take them.
+ const target = vi.spyOn(gatewayOf(socket), 'target').mockReturnValue(false)
+ expect(socket.sendViewerFrame('remote', 'viewer_data', { streamId: 's1', data: 'AA==' })).toBe(false)
+ expect(target).toHaveBeenCalledWith('remote', 'viewer_data', { streamId: 's1', data: 'AA==' })
+ await socket.unregisterLocalClient('local:viewer')
+ expect(streams.closed).toHaveBeenCalledWith('local:viewer')
+ socket.fromGateway.client('r1', { role: 'web', label: null, identity: 'PUB', direct: false })
+ socket.fromGateway.client('r1', null)
+ expect(streams.closed).toHaveBeenCalledWith('r1')
+ await socket.fromGateway.disconnected('r2')
+ expect(streams.closed).toHaveBeenCalledWith('r2')
+ socket.fromGateway.linkDown()
+ expect(streams.closedAll).toHaveBeenCalledTimes(1)
+ await socket.stop()
+ expect(streams.closedAll).toHaveBeenCalledTimes(2)
+ })
+
it('requires encryption and a web-role session remotely, while permitting trusted local clients', async () => {
const socket = relaySocket('token')
const internals = socket as any
- const handle = vi.spyOn(socket.viewerForwarder, 'handle').mockImplementation(() => {})
+ const handle = vi.fn()
+ socket.viewerStreams = { frame: handle, surface: vi.fn(), closed: vi.fn(), closedAll: vi.fn() }
const role = vi.spyOn(gatewayOf(socket).e2ee, 'sessionRole').mockReturnValue('web')
const frame = { type: 'viewer_request', payload: { streamId: 'v', agentId: 'a' } }
const unwrap = vi.spyOn(gatewayOf(socket).e2ee, 'unwrapDown').mockReturnValue(frame)
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index b831a8d11..dc0fd506a 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -1,6 +1,7 @@
import type { PurgeAgentService } from './lib/purgeAgentService.js'
import { SHARE_REQUESTS, TEAMS_REQUESTS, type Asker, type BackendNotice, type GatewayEvents, type GatewayPort, type ModelsPort, type RemoteClient, type RemoteRole, type RemoteTransport } from './core/api.js'
import { ServiceUnavailableError } from './core/serviceHost.js'
+import type { ViewerStreams } from './core/viewerStreams.js'
import type { ActivityFrame } from './lib/turnActivity.js'
import { MonitorCompletions } from './lib/harnessMonitor.js'
import type { WifiCore } from './core/wifi.js'
@@ -32,10 +33,8 @@ import { engineInstallRecipe } from './lib/engineInstall.js'
import { agentFrame, type AgentDshContext, type AgentFrame } from './lib/agentFrame.js'
import { agentTokenUsage } from './lib/agentTokenUsage.js'
import { terminalHandoffRequest } from './lib/terminalHandoff.js'
-import { ViewerForwarder } from './lib/viewerForwarder.js'
-import { InteractiveViewers } from './lib/interactiveViewer.js'
import { OwnerCommands, OWNER_COMMAND_TYPES } from './lib/ownerCommands.js'
-import { VIEWER_DOWN_TYPES } from './lib/viewerWire.js'
+import { VIEWER_DOWN_TYPES } from './lib/viewerFrames.js'
import type { TerminalStreamManager } from './lib/terminalStreamManager.js'
import { encodeTerminalLocal, type TerminalBinaryClear } from './lib/terminalBinary.js'
import { BACKEND_ONLY_DOWN_TYPES, GATEWAY_REQUEST_TYPES, isLocalClientId, logSafeType, PAIR_REQUESTS, PLATE_REQUEST, rpcResultType, type DownTransport } from './lib/relayFrames.js'
@@ -170,15 +169,13 @@ export class BackendSocket {
/** What the daemon knows about an agent's DSH companions (viewer URL, verdict); null when nothing. */
activityFrameProvider: ((session: RegisteredSession) => ActivityFrame | null) | null = null
dshFrameProvider: ((session: RegisteredSession) => AgentDshContext | null) | null = null
- viewerTargetProvider: ((agentId: string) => string | null) | null = null
- readonly interactiveViewers = new InteractiveViewers(agentId => this.viewerTargetProvider?.(agentId) ?? null)
- readonly viewerForwarder = new ViewerForwarder({
- target: (agentId) => this.viewerTargetProvider?.(agentId) ?? null,
- send: (connId, type, payload) => {
- if (this.localClients.has(connId)) { this.sendTo(connId, { type, payload }); return true }
- return this.throughGateway('viewer', (gateway) => gateway.target(connId, type, payload), false)
- },
- })
+ /** This machine's viewers, served to a client over its connection: the viewers' (core/viewerStreams.ts). */
+ viewerStreams: ViewerStreams | null = null
+ /** A viewer stream's frame to the one connection that opened it: false when it cannot reach it. */
+ sendViewerFrame(connId: string, type: string, payload: Record<string, unknown>): boolean {
+ if (this.localClients.has(connId)) { this.sendTo(connId, { type, payload }); return true }
+ return this.throughGateway('viewer', (gateway) => gateway.target(connId, type, payload), false)
+ }
/** Injectable for queue-isolation tests; production uses the machine-local probe. */
engineProbeProvider: typeof probeEngines = probeEngines
readonly ownerCommands = new OwnerCommands()
@@ -341,13 +338,12 @@ export class BackendSocket {
this.wifi?.session(connId, client)
if (client) { this.remoteClients.set(connId, client); return }
if (!this.remoteClients.delete(connId)) return
- this.viewerForwarder.closeConnection(connId); this.interactiveViewers.closeConnection(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
+ this.viewerStreams?.closed(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
},
disconnected: async (connId) => {
this.wifi?.dropped(connId)
this.remoteClients.delete(connId)
- this.viewerForwarder.closeConnection(connId)
- this.interactiveViewers.closeConnection(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
+ this.viewerStreams?.closed(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
await this.terminalStreams?.closeConnection(
connId,
'client connection closed',
@@ -365,8 +361,7 @@ export class BackendSocket {
},
linkDown: () => {
this.observers?.closeAll()
- this.viewerForwarder.closeAll()
- this.interactiveViewers.closeAll(); this.ownerCommands.closeAll(); for (const connId of this.remoteClients.keys()) this.onConnectionClosed?.(connId)
+ this.viewerStreams?.closedAll(); this.ownerCommands.closeAll(); for (const connId of this.remoteClients.keys()) this.onConnectionClosed?.(connId)
void this.terminalStreams?.closeConnectionsWhere(
(connId) => !isLocalClientId(connId),
'backend disconnected',
@@ -459,8 +454,7 @@ export class BackendSocket {
async stop(): Promise<void> {
this.closed = true
this.closeAgentService?.dispose()
- this.viewerForwarder.closeAll()
- this.interactiveViewers.closeAll(); this.ownerCommands.closeAll()
+ this.viewerStreams?.closedAll(); this.ownerCommands.closeAll()
await this.terminalStreams?.stop()
await this.gatewayPort?.stop()
}
@@ -633,8 +627,7 @@ export class BackendSocket {
if (!this.localClients.delete(connId)) return
if (!this.toolClients.delete(connId)) this.onLocalClient?.(connId, false)
this.rowStateWindows.delete(connId)
- this.viewerForwarder.closeConnection(connId)
- this.interactiveViewers.closeConnection(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
+ this.viewerStreams?.closed(connId); this.ownerCommands.closeConnection(connId); this.onConnectionClosed?.(connId)
// The last one leaving before any link could hear it attach: nothing happened, as far as the backend
// is concerned, and the gateway does not tell a later link otherwise.
this.gatewayPort?.localClients(this.localClients.size)
@@ -835,7 +828,7 @@ export class BackendSocket {
if (type === 'viewer_surface') {
if (!owner) return
// Rendering and input never hold up terminal traffic on the ordered machine queue.
- void this.interactiveViewers.request(connId, payload)
+ void (this.viewerStreams?.surface(connId, payload) ?? Promise.reject(new Error('no viewers')))
.then(result => reply(type, requestId, result))
.catch(() => reply(type, requestId, { error: 'VIEWER_UNAVAILABLE' }))
return
@@ -849,9 +842,7 @@ export class BackendSocket {
}
if (type.startsWith('viewer_')) {
- if (VIEWER_DOWN_TYPES.has(type) && owner) {
- this.viewerForwarder.handle(connId, type, payload)
- }
+ if (VIEWER_DOWN_TYPES.has(type) && owner) this.viewerStreams?.frame(connId, type, payload)
return
}
diff --git a/cli/src/core/AGENTS.md b/cli/src/core/AGENTS.md
index c163aeeb0..cf4dd257d 100644
--- a/cli/src/core/AGENTS.md
+++ b/cli/src/core/AGENTS.md
@@ -40,7 +40,8 @@ questions. Everything else is a service ([../services/AGENTS.md](../services/AGE
`../services/inline.ts`, and only when one runs in this process instead.
- `api.ts`: the contract with services, their ports, fallbacks and requests. `serviceHost.ts`: services
in this process. `serviceLinks.ts`: services in their own processes. `viewersLink.ts`: what the core
- keeps of the viewers when they run in theirs. `workspacesLink.ts`: what the core tells workspaces in
+ keeps of the viewers when they run in theirs. `viewerStreams.ts`: a client's viewer stream and rendered
+ frames, handed to the viewers, and refused at once while they are down. `workspacesLink.ts`: what the core tells workspaces in
theirs. `teamsLink.ts`: every change to the teams' prompt scopes, kept until their process has it once Tab
collaboration is on, which of its deliveries may be written, and the scopes' own two questions.
`monitorLink.ts`: the monitor's port, asked of its process. `storeLink.ts`: what the Store in its
diff --git a/cli/src/core/api.spec.ts b/cli/src/core/api.spec.ts
index f324d62b4..a1e306f89 100644
--- a/cli/src/core/api.spec.ts
+++ b/cli/src/core/api.spec.ts
@@ -46,6 +46,7 @@ describe('the core API services stand on', () => {
syncSession: vi.fn(),
runtimeModels: vi.fn(async () => []),
viewerChanged: vi.fn(),
+ viewerFrame: vi.fn(() => true),
gridNamed: vi.fn(),
gridModelsChanged: vi.fn(),
dshInstallStatus: vi.fn(),
@@ -102,6 +103,11 @@ describe('the core API services stand on', () => {
expect(core.agents.sync).toBe(deps.syncSession)
expect(core.agents.runtimeModels).toBe(deps.runtimeModels)
expect(core.clients.viewerChanged).toBe(deps.viewerChanged)
+ // A viewer stream's frames reach a client; nothing else a viewers process names does.
+ expect(core.clients.viewerFrame('c1', 'viewer_data', { streamId: 's' })).toBe(true)
+ expect(deps.viewerFrame).toHaveBeenCalledWith('c1', 'viewer_data', { streamId: 's' })
+ expect(core.clients.viewerFrame('c1', 'agent_synced', {})).toBe(false)
+ expect(deps.viewerFrame).toHaveBeenCalledTimes(1)
expect(core.clients.dshInstallStatus).toBe(deps.dshInstallStatus)
expect(core.agents.advertised().map((s) => s.agentId)).toEqual(['live'])
expect(core.clients.gridNamed).toBe(deps.gridNamed)
diff --git a/cli/src/core/api.ts b/cli/src/core/api.ts
index 1720e526f..f094612ec 100644
--- a/cli/src/core/api.ts
+++ b/cli/src/core/api.ts
@@ -33,6 +33,7 @@ import type { ExternalSessions, OpenSessions } from '../lib/sessionSearch/extern
import type { SessionSearchIndex } from '../lib/sessionSearch/indexer.js'
import type { StoppedAgentStore } from '../lib/stoppedAgents.js'
import type { TerminalBinaryClear } from '../lib/terminalBinary.js'
+import { VIEWER_UP_TYPES } from '../lib/viewerFrames.js'
import type { RouteAnswer } from '../localWsServer.js'
import type { SwarmPromptScopes } from '../teams/promptScope.js'
import type { DeviceInputStatus } from './deviceInput.js'
@@ -197,6 +198,10 @@ export interface CoreApi {
clients: {
/** An agent's viewer moved: the windows' viewer panes forward to the new one. */
viewerChanged(agentId: string): void
+ /** A frame of a viewer stream for the one connection that opened it (`viewer_response`, `viewer_data`,
+ * `viewer_ack`, `viewer_end`, `viewer_close`): false when it cannot reach that connection. Only those
+ * types; the core drops any other. */
+ viewerFrame(connId: string, type: string, payload: Record<string, unknown>): boolean
/** The account's grid has a name: the models picker answers with it at once. */
gridNamed(name: string): void
/** What the models picker lists may have changed (a local model started or stopped, grid set up):
@@ -527,12 +532,26 @@ export interface ViewersPort {
forwardingUrl(agentId: string): string | null
/** Stop every viewer and watch, for a restart or a shutdown. */
stop(): Promise<void>
+ /** A client's frame of a viewer stream (`viewer_request`, `viewer_data`, …), from the connection `connId`:
+ * how a client that cannot reach this machine's loopback (a phone, another machine's window) is served
+ * a viewer. False when the viewers did not take it. */
+ stream(connId: string, type: string, payload: Record<string, unknown>): boolean
+ /** A client's `viewer_surface`: a frame of a viewer rendered for it by a headless browser, or why not. */
+ surface(connId: string, payload: Record<string, unknown>): Promise<Record<string, unknown>>
+ /** A client's connection ended (`connId`), or every one did (none: the link went down, or the socket is
+ * stopping): its viewer streams and surfaces close. */
+ closed(connId?: string): void
}
+/** What a client is told for a surface the viewers cannot render while they are off or failing. */
+export const VIEWERS_UNAVAILABLE = { error: 'SERVICE_UNAVAILABLE', service: 'viewers', retryable: true }
+
/** What the core gets when the viewers fail: agents' frames carry no DSH context and the windows
- * no viewer, and a restart or shutdown goes on. */
+ * no viewer, a client's viewer stream is refused at once (core/viewerStreams.ts) and its surface
+ * answered unavailable, and a restart or shutdown goes on. */
export const VIEWERS_FALLBACKS: PortFallbacks<ViewersPort> = {
attach: undefined, detach: undefined, frameContext: null, forwardingUrl: null, stop: later(undefined),
+ stream: false, surface: later(VIEWERS_UNAVAILABLE), closed: undefined,
}
/**
@@ -1131,6 +1150,7 @@ export interface CoreApiDeps {
syncSession: CoreApi['agents']['sync']
runtimeModels: CoreApi['agents']['runtimeModels']
viewerChanged: CoreApi['clients']['viewerChanged']
+ viewerFrame: CoreApi['clients']['viewerFrame']
gridNamed: CoreApi['clients']['gridNamed']
gridModelsChanged: CoreApi['clients']['gridModelsChanged']
dshInstallStatus: CoreApi['clients']['dshInstallStatus']
@@ -1172,7 +1192,7 @@ export interface CoreApiDeps {
export function createCoreApi({
dataDir, registry, stoppedAgents, databaseHistory, externalSessions, openSessions, syncSession, runtimeModels, viewerChanged,
gridNamed, gridModelsChanged, dshInstallStatus, mintGridName, accessToken, lane, observerKey, observer, privateGridName, machineName, backend, onNotice,
- runtimeProfile, setRuntime, fork, create, dsh, windows, daemon, turns, questions, terminals = TERMINALS_OFF, machine, activityText,
+ runtimeProfile, setRuntime, fork, create, dsh, windows, daemon, turns, questions, terminals = TERMINALS_OFF, viewerFrame, machine, activityText,
signedIn, environment, machines, sendLocal, sendToWindow, hasWindow, devicesChanged, dialWatching, wifi, log = (line) => console.warn(line),
}: CoreApiDeps): CoreApi {
// The devices reach the windows through these alone, and only with their own frames: a devices process
@@ -1208,6 +1228,9 @@ export function createCoreApi({
account: { mintGridName, accessToken, lane, observerKey, privateGridName, machineName, backend, onNotice, signedIn, environment, machines },
clients: {
viewerChanged, gridNamed, gridModelsChanged, dshInstallStatus, windows, observer, hasWindow, devicesChanged, dialWatching,
+ // A viewers process speaks to the core as a service: what it may send a client is a viewer stream's
+ // frame, decided here, never whatever it names.
+ viewerFrame: (connId, type, payload) => VIEWER_UP_TYPES.has(type) && viewerFrame(connId, type, payload),
sendLocal: (frame) => {
if (windowFrames.has(frame?.type)) sendLocal(frame)
else refused(frame?.type)
diff --git a/cli/src/core/main.ts b/cli/src/core/main.ts
index 04c3af53f..74c6df3ee 100644
--- a/cli/src/core/main.ts
+++ b/cli/src/core/main.ts
@@ -123,6 +123,7 @@ import { createCoreApi, DEVICES_FALLBACKS, DEVICES_REQUESTS, emptyPorts, EXPERIM
import { createServiceHost, testFaults } from './serviceHost.js'
import { startStalls } from './stall.js'
import { createServiceLinks, type ServiceLinks } from './serviceLinks.js'
+import { createViewerStreams } from './viewerStreams.js'
import { createViewersLink } from './viewersLink.js'
import { createWorkspacesLink } from './workspacesLink.js'
import { createMonitorLink } from './monitorLink.js'
@@ -662,10 +663,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
openSessions,
syncSession,
runtimeModels,
- viewerChanged: (agentId) => {
- backendRef?.viewerForwarder.refresh(agentId)
- backendRef?.interactiveViewers.refresh(agentId)
- },
+ viewerChanged: () => {}, // nothing in the core forwards to a viewer: the viewers follow their own (services/viewers.ts)
+ viewerFrame: (connId, type, payload) => backendRef?.sendViewerFrame(connId, type, payload) ?? false,
gridNamed: (name) => backendRef?.setHarnessGridName(name),
gridModelsChanged: () => { void backendRef?.pushGridModels() },
privateGridName: async () => backendRef?.gridName() ?? null,
@@ -851,7 +850,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
forget: (agentId) => ports.teams?.forget(agentId),
canWrite: (deliveryId) => ports.teams?.canWrite(deliveryId) ?? false,
}
- backend.viewerTargetProvider = (agentId) => ports.viewers?.forwardingUrl(agentId) ?? null
+ // This machine's viewers, served to a client over its connection by the viewers (core/viewerStreams.ts).
+ backend.viewerStreams = createViewerStreams(() => ports.viewers, (connId, type, payload) => backend.sendViewerFrame(connId, type, payload))
// The harnesses the release bundles (the Model Manager, Devices, the Harness Monitor), before restore
// relaunches an agent on one. Here and not in the Store's process: its lean bundle would carry a second
@@ -1149,7 +1149,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// Directors' (core/orchestratorLink.ts).
const orchestratorLink = createOrchestratorLink((frame) => serviceLinks.notify('orchestrator', frame))
// What the core keeps of the viewers in their own process, for the frames it builds (core/viewersLink.ts).
- const viewersLink = createViewersLink(coreApi, (frame, opts) => serviceLinks.notify('viewers', frame, opts))
+ const viewersLink = createViewersLink(coreApi, (frame, opts) => serviceLinks.notify('viewers', frame, opts),
+ { call: (type, payload, waitMs) => serviceLinks.call('viewers', type, payload, waitMs), buffered: () => serviceLinks.buffered('viewers') })
// How the core tells workspaces in their own process what to do, and answers them (core/workspacesLink.ts).
const workspacesLink = createWorkspacesLink(coreApi, (frame) => serviceLinks.notify('workspaces', frame), forgetAgentProject)
// Tab collaboration and teams, an experiment: the prompt scopes and the teams beside them, both in its process
@@ -1203,8 +1204,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
: service === 'wifi' ? wifiLink.answer(query, payload)
: service === 'gateway' && gatewayLink ? gatewayLink.answer(query, payload) : answerAgentQuery(coreApi, query)),
// The gateway's own traffic: its remote clients and what they sent, and its comings and goings; and what
- // the devices tell the core (a turn, a frame for the windows, a dial on the wire).
- notice: (service, payload) => { if (service === 'gateway') gatewayLink?.notice(payload); else if (service === 'devices') devicesLink.notice(payload); else if (service === 'wifi') wifiLink.notice(payload) },
+ // the devices tell the core (a turn, a frame for the windows, a dial on the wire); a viewer stream's answers.
+ notice: (service, payload) => { if (service === 'gateway') gatewayLink?.notice(payload); else if (service === 'devices') devicesLink.notice(payload); else if (service === 'wifi') wifiLink.notice(payload); else if (service === 'viewers') viewersLink.notice(payload) },
binary: (service, bytes) => { if (service === 'gateway') gatewayLink?.binary(bytes) },
connected: (service) => {
if (service === 'gateway') gatewayLink?.connected()
diff --git a/cli/src/core/serviceHost.spec.ts b/cli/src/core/serviceHost.spec.ts
index 56039c668..4e3b6a785 100644
--- a/cli/src/core/serviceHost.spec.ts
+++ b/cli/src/core/serviceHost.spec.ts
@@ -22,7 +22,10 @@ const SEARCH = {
const testPort = (h: { ports: CorePorts }): TestPort => h.ports.search as unknown as TestPort
const VIEWERS: PortFallbacks<ViewersPort> = {
attach: undefined, detach: undefined, frameContext: null, forwardingUrl: null, stop: later(undefined),
+ stream: false, surface: later({}), closed: undefined,
}
+/** The viewer streams' members, which these tests never call. */
+const STREAMS = { stream: vi.fn(() => true), surface: vi.fn(async () => ({})), closed: vi.fn() }
function host(options: Parameters<typeof createServiceHost>[1] = {}) {
const ports = emptyPorts()
@@ -118,7 +121,7 @@ describe('hosting services so one that fails cannot take the core down', () => {
const h = host()
let viewersStop: () => Promise<void> = async () => { throw new Error('viewer hung') }
h.services.start('viewers', (_core, ports) => {
- ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: () => viewersStop() }
+ ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: () => viewersStop(), ...STREAMS }
}, fakeCore(), VIEWERS)
await expect(h.ports.viewers!.stop()).resolves.toBeUndefined()
viewersStop = () => { throw new Error('sync throw in async member') }
@@ -129,7 +132,7 @@ describe('hosting services so one that fails cannot take the core down', () => {
it('passes a promise that resolves straight through', async () => {
const h = host()
h.services.start('viewers', (_core, ports) => {
- ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => 'http://v'), stop: async () => {} }
+ ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => 'http://v'), stop: async () => {}, ...STREAMS }
}, fakeCore(), VIEWERS)
expect(h.ports.viewers!.forwardingUrl('a1')).toBe('http://v')
await expect(h.ports.viewers!.stop()).resolves.toBeUndefined()
@@ -192,7 +195,7 @@ describe('hosting services so one that fails cannot take the core down', () => {
const async = host({ maxFailures: 1 })
async.services.start('viewers', (_core, ports) => {
- ports.viewers = { attach: () => { throw new Error('x') }, detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: async () => { throw new Error('viewer stuck') } }
+ ports.viewers = { attach: () => { throw new Error('x') }, detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: async () => { throw new Error('viewer stuck') }, ...STREAMS }
}, fakeCore(), VIEWERS)
async.ports.viewers!.attach({} as never)
await new Promise((resolve) => setImmediate(resolve))
diff --git a/cli/src/core/viewerStreams.spec.ts b/cli/src/core/viewerStreams.spec.ts
new file mode 100644
index 000000000..71c2b95f5
--- /dev/null
+++ b/cli/src/core/viewerStreams.spec.ts
@@ -0,0 +1,64 @@
+import { describe, expect, it, vi } from 'vitest'
+import { VIEWERS_UNAVAILABLE } from './api.js'
+import { createViewerStreams, NOT_SERVED } from './viewerStreams.js'
+
+function setup(taken = true) {
+ const viewers = {
+ stream: vi.fn(() => taken),
+ surface: vi.fn(async () => ({ data: 'jpeg' })),
+ closed: vi.fn(),
+ }
+ let on: typeof viewers | null = viewers
+ const send = vi.fn(() => true)
+ const streams = createViewerStreams(() => on, send)
+ return { viewers, send, streams, off: () => { on = null } }
+}
+
+describe('this machine\'s viewers, served to a client over its connection', () => {
+ it('hands each frame of a stream to the viewers, and says nothing itself when they take it', () => {
+ const { viewers, send, streams } = setup()
+ streams.frame('c1', 'viewer_request', { streamId: 's1', agentId: 'a1' })
+ expect(viewers.stream).toHaveBeenCalledWith('c1', 'viewer_request', { streamId: 's1', agentId: 'a1' })
+ expect(send).not.toHaveBeenCalled()
+ })
+
+ it('refuses a stream at once when the viewers do not take it, so the client never waits out its timeout', () => {
+ const { send, streams } = setup(false)
+ streams.frame('c1', 'viewer_request', { streamId: 's1' })
+ streams.frame('c1', 'viewer_data', { streamId: 's1', data: 'AA==' })
+ expect(send.mock.calls).toEqual([
+ ['c1', 'viewer_close', { streamId: 's1', error: NOT_SERVED }],
+ ['c1', 'viewer_close', { streamId: 's1', error: NOT_SERVED }],
+ ])
+ // A close needs no answer, and a frame of no stream has nobody to tell.
+ streams.frame('c1', 'viewer_close', { streamId: 's1' })
+ streams.frame('c1', 'viewer_request', { streamId: '../bad id' })
+ streams.frame('c1', 'viewer_request', {})
+ expect(send).toHaveBeenCalledTimes(2)
+ })
+
+ it('refuses every stream while the viewers are off, and answers a surface unavailable', async () => {
+ const { viewers, send, streams, off } = setup()
+ off()
+ streams.frame('c1', 'viewer_request', { streamId: 's1' })
+ expect(send).toHaveBeenCalledWith('c1', 'viewer_close', { streamId: 's1', error: NOT_SERVED })
+ await expect(streams.surface('c1', { surfaceId: 'v' })).resolves.toEqual(VIEWERS_UNAVAILABLE)
+ streams.closed('c1')
+ streams.closedAll()
+ expect(viewers.stream).not.toHaveBeenCalled()
+ expect(viewers.closed).not.toHaveBeenCalled()
+ })
+
+ it('asks the viewers for a client\'s rendered frame', async () => {
+ const { viewers, streams } = setup()
+ await expect(streams.surface('c1', { surfaceId: 'v' })).resolves.toEqual({ data: 'jpeg' })
+ expect(viewers.surface).toHaveBeenCalledWith('c1', { surfaceId: 'v' })
+ })
+
+ it('tells the viewers a connection went, or that every one did', () => {
+ const { viewers, streams } = setup()
+ streams.closed('c1')
+ streams.closedAll()
+ expect(viewers.closed.mock.calls).toEqual([['c1'], []])
+ })
+})
diff --git a/cli/src/core/viewerStreams.ts b/cli/src/core/viewerStreams.ts
new file mode 100644
index 000000000..56b8aa6d7
--- /dev/null
+++ b/cli/src/core/viewerStreams.ts
@@ -0,0 +1,47 @@
+/**
+ * This machine's viewers, served to a client that cannot reach its loopback: the phone, the web, a window
+ * on another of the owner's machines. Such a client asks for a viewer's pages over its own connection
+ * (`viewer_request` and the stream's frames after it) or asks for a rendered frame of it
+ * (`viewer_surface`), and the viewers answer it (services/viewers.ts): in their own process by default,
+ * where the viewer servers already run, or in this one.
+ *
+ * The socket hands every such frame here, after its gates, and nothing here waits: a stream's frame is
+ * passed on as it comes, and its answers go back to that one connection when the viewers send them
+ * (`CoreApi.clients.viewerFrame`). While the viewers are off, hung behind a full socket or failing, a
+ * stream is refused at once with `viewer_close`, so a client never waits out its own timeout for a viewer
+ * nobody is serving, and a surface is answered unavailable.
+ */
+import { viewerStreamId } from '../lib/viewerFrames.js'
+import { VIEWERS_UNAVAILABLE, type ViewersPort } from './api.js'
+
+export interface ViewerStreams {
+ /** A client's frame of a viewer stream. */
+ frame(connId: string, type: string, payload: Record<string, unknown>): void
+ /** A client's `viewer_surface`, answered as its reply. */
+ surface(connId: string, payload: Record<string, unknown>): Promise<Record<string, unknown>>
+ /** A client's connection ended. */
+ closed(connId: string): void
+ /** Every client's did: the link went down, or the socket is stopping. */
+ closedAll(): void
+}
+
+/** What a stream nobody serves is told: the same words the viewers use for a viewer that is gone. */
+export const NOT_SERVED = 'Viewer is no longer available'
+
+export function createViewerStreams(
+ /** The viewers' port as it is now: null while they are off. */
+ viewers: () => Pick<ViewersPort, 'stream' | 'surface' | 'closed'> | null,
+ /** A frame to one connection, as the viewers' own answers go. */
+ send: (connId: string, type: string, payload: Record<string, unknown>) => boolean,
+): ViewerStreams {
+ return {
+ frame: (connId, type, payload) => {
+ if (viewers()?.stream(connId, type, payload)) return
+ // A close needs no answer; any other frame of a stream nobody holds now ends it on the client's side.
+ if (type !== 'viewer_close' && viewerStreamId(payload.streamId)) send(connId, 'viewer_close', { streamId: payload.streamId, error: NOT_SERVED })
+ },
+ surface: async (connId, payload) => (await viewers()?.surface(connId, payload)) ?? VIEWERS_UNAVAILABLE,
+ closed: (connId) => { viewers()?.closed(connId) },
+ closedAll: () => { viewers()?.closed() },
+ }
+}
diff --git a/cli/src/core/viewersLink.spec.ts b/cli/src/core/viewersLink.spec.ts
index 7662d1201..2915ee0fa 100644
--- a/cli/src/core/viewersLink.spec.ts
+++ b/cli/src/core/viewersLink.spec.ts
@@ -1,7 +1,8 @@
import { describe, expect, it, vi } from 'vitest'
import type { RegisteredSession } from '../lib/registry.js'
import { fakeCore } from '../testing/fakeCore.js'
-import { createViewersLink } from './viewersLink.js'
+import { VIEWERS_UNAVAILABLE } from './api.js'
+import { createViewersLink, SURFACE_WAIT_MS, VIEWERS_BUFFER_LIMIT } from './viewersLink.js'
const agent = (over: Partial<RegisteredSession> = {}) =>
({ agentId: 'a1', sessionId: 's1', engine: 'claude', dsh: 'acme/blender', cwd: '/work/scene', ...over }) as RegisteredSession
@@ -45,35 +46,24 @@ describe('the viewers in their own process, as the core keeps them', () => {
expect(link.answer('context', { agentId: 'a1', context: said, forwardingUrl: 'http://127.0.0.1:7001/', requestId: 'q1', query: 'context' })).toEqual({ kept: true })
expect(port.frameContext(agent())).toEqual(said)
expect(port.forwardingUrl('a1')).toBe('http://127.0.0.1:7001/')
- expect(core.clients.viewerChanged).toHaveBeenCalledWith('a1')
expect(core.agents.sync).toHaveBeenCalledWith(agent())
// An agent without a harness has nothing to say on its frame, whatever was kept under its id.
expect(port.frameContext(agent({ dsh: undefined }))).toBeNull()
})
- it('a change that moves no viewer sends the frame without disturbing the viewer panes; no change does nothing', () => {
+ it('a change sends the frame again; no change does nothing', () => {
const { core, link, port } = setup()
port.attach(agent())
link.answer('context', { agentId: 'a1', context: context(), forwardingUrl: null })
- // Its first word moved nothing a viewer pane follows: no viewer yet.
- expect(core.clients.viewerChanged).not.toHaveBeenCalled()
expect(core.agents.sync).toHaveBeenCalledTimes(1)
const verdict = { ready: true, summary: 'ok', errors: 0, warnings: 0, artifact: null, phases: [], updatedAt: null }
link.answer('context', { agentId: 'a1', context: context({ verdict }), forwardingUrl: null })
expect(port.frameContext(agent())).toMatchObject({ verdict })
- expect(core.clients.viewerChanged).not.toHaveBeenCalled()
expect(core.agents.sync).toHaveBeenCalledTimes(2)
expect(link.answer('context', { agentId: 'a1', context: context({ verdict }), forwardingUrl: null })).toEqual({ kept: true })
expect(core.agents.sync).toHaveBeenCalledTimes(2)
})
- it('follows a viewer URL that moves even when nothing forwards to it', () => {
- const { core, link, port } = setup()
- port.attach(agent())
- link.answer('context', { agentId: 'a1', context: context({ viewerUrl: 'http://elsewhere.test/' }), forwardingUrl: null })
- expect(core.clients.viewerChanged).toHaveBeenCalledWith('a1')
- })
-
it('waits for the terminal before sending a frame, which would otherwise read as "agent gone"', () => {
const detached = setup(agent(), false)
detached.port.attach(agent())
@@ -107,22 +97,14 @@ describe('the viewers in their own process, as the core keeps them', () => {
expect(port.forwardingUrl('a1')).toBeNull()
})
- it('a detach forgets the agent, lets the viewer panes go of a viewer it had, and is held until the process hears it', () => {
- const { core, notify, link, port } = setup()
+ it('a detach forgets the agent, and is held until the process hears it', () => {
+ const { notify, link, port } = setup()
port.attach(agent())
link.answer('context', { agentId: 'a1', context: context({ viewerUrl: 'http://127.0.0.1:7001/' }), forwardingUrl: 'http://127.0.0.1:7001/' })
- vi.mocked(core.clients.viewerChanged).mockClear()
port.detach('a1')
- expect(core.clients.viewerChanged).toHaveBeenCalledWith('a1')
expect(notify).toHaveBeenLastCalledWith({ type: 'service_event', payload: { kind: 'detach', agentId: 'a1' } }, { untilDelivered: true })
expect(port.frameContext(agent())).toBeNull()
- // One with no viewer, or never heard of, has no viewer pane to let go of.
- vi.mocked(core.clients.viewerChanged).mockClear()
- port.attach(agent({ agentId: 'a2' }))
- link.answer('context', { agentId: 'a2', context: null, forwardingUrl: null })
- port.detach('a2')
- port.detach('never')
- expect(core.clients.viewerChanged).not.toHaveBeenCalled()
+ expect(port.forwardingUrl('a1')).toBeNull()
})
it('answers from what it last knew while the process cannot hear anything', () => {
@@ -147,3 +129,76 @@ describe('the viewers in their own process, as the core keeps them', () => {
expect(notify).not.toHaveBeenCalled()
})
})
+
+describe('a viewer served to a client over its connection, through the viewers\' process', () => {
+ const streaming = (buffered = 0) => {
+ const core = fakeCore()
+ const notify = vi.fn(() => true)
+ const call = vi.fn(async (_type: string, payload: Record<string, unknown>) => ({ data: 'jpeg', asked: payload }))
+ const link = createViewersLink(core, notify, { call, buffered: () => buffered })
+ return { core, notify, call, link, port: link.port }
+ }
+
+ it('tells the process each frame of a client\'s stream as it comes, never held for a process that is down', () => {
+ const { notify, port } = streaming()
+ expect(port.stream('c1', 'viewer_request', { streamId: 's1' })).toBe(true)
+ expect(notify).toHaveBeenCalledWith({ type: 'service_event', payload: { kind: 'stream', connId: 'c1', type: 'viewer_request', frame: { streamId: 's1' } } })
+ notify.mockReturnValue(false)
+ expect(port.stream('c1', 'viewer_data', { streamId: 's1' })).toBe(false)
+ expect(notify).toHaveBeenCalledTimes(2)
+ })
+
+ it('refuses a stream\'s frame while the process is not reading, rather than keep it', () => {
+ const { notify, port } = streaming(VIEWERS_BUFFER_LIMIT + 1)
+ expect(port.stream('c1', 'viewer_request', { streamId: 's1' })).toBe(false)
+ expect(notify).not.toHaveBeenCalled()
+ })
+
+ it('asks the process for a client\'s rendered frame, with the connection it came over, waiting no longer than a client does', async () => {
+ const { call, port } = streaming()
+ await expect(port.surface('c1', { surfaceId: 'v' })).resolves.toEqual({ data: 'jpeg', asked: { surfaceId: 'v', connId: 'c1' } })
+ expect(call).toHaveBeenCalledWith('surface', { surfaceId: 'v', connId: 'c1' }, SURFACE_WAIT_MS)
+ })
+
+ it('without a link to the process, a surface is answered unavailable and a stream is not taken', async () => {
+ const link = createViewersLink(fakeCore(), vi.fn(() => false))
+ await expect(link.port.surface('c1', {})).resolves.toEqual(VIEWERS_UNAVAILABLE)
+ expect(link.port.stream('c1', 'viewer_request', { streamId: 's1' })).toBe(false)
+ })
+
+ it('tells the process a connection went, or every one did', () => {
+ const { notify, port } = streaming()
+ port.closed('c1')
+ port.closed()
+ expect(notify.mock.calls).toEqual([
+ [{ type: 'service_event', payload: { kind: 'closed', connId: 'c1' } }],
+ [{ type: 'service_event', payload: { kind: 'closed' } }],
+ ])
+ })
+
+ it('hands what a stream answers to its connection, and ends the stream in the process when it cannot', () => {
+ const { core, notify, link } = streaming()
+ vi.mocked(core.clients.viewerFrame).mockReturnValue(true)
+ link.notice({ kind: 'viewer', connId: 'c1', type: 'viewer_response', payload: { streamId: 's1', status: 200 } })
+ expect(core.clients.viewerFrame).toHaveBeenCalledWith('c1', 'viewer_response', { streamId: 's1', status: 200 })
+ expect(notify).not.toHaveBeenCalled()
+ vi.mocked(core.clients.viewerFrame).mockReturnValue(false)
+ link.notice({ kind: 'viewer', connId: 'c1', type: 'viewer_data', payload: { streamId: 's1', data: 'AA==' } })
+ expect(notify).toHaveBeenCalledWith({ type: 'service_event', payload: {
+ kind: 'stream', connId: 'c1', type: 'viewer_close', frame: { streamId: 's1', error: 'Viewer connection closed' },
+ } })
+ // A close that could not be delivered, or a frame of no stream, needs nothing more.
+ link.notice({ kind: 'viewer', connId: 'c1', type: 'viewer_close', payload: { streamId: 's1' } })
+ link.notice({ kind: 'viewer', connId: 'c1', type: 'viewer_data' })
+ expect(notify).toHaveBeenCalledTimes(1)
+ expect(core.clients.viewerFrame).toHaveBeenLastCalledWith('c1', 'viewer_data', {})
+ })
+
+ it('ignores a notice that is not a stream\'s frame for a connection', () => {
+ const { core, link } = streaming()
+ link.notice({ kind: 'other', connId: 'c1', type: 'viewer_data' })
+ link.notice({ kind: 'viewer', connId: 7, type: 'viewer_data' })
+ link.notice({ kind: 'viewer', connId: 'c1' })
+ expect(core.clients.viewerFrame).not.toHaveBeenCalled()
+ })
+})
diff --git a/cli/src/core/viewersLink.ts b/cli/src/core/viewersLink.ts
index 6d9b9650b..df2898b45 100644
--- a/cli/src/core/viewersLink.ts
+++ b/cli/src/core/viewersLink.ts
@@ -7,9 +7,9 @@
* cannot wait on another process, so the viewers tell the core each agent's answers whenever they change
* (a `service_query` of kind `context`), the core keeps the last of them here, and the port answers from
* that: while the process is down, with what it last knew, or the fallbacks for an agent it never heard
- * of. Each change then does in the core what the viewers did in their process: the windows' viewer panes
- * follow a viewer that moved (`clients.viewerChanged`), and the frame of an agent whose terminal is
- * attached goes out again.
+ * of. Each change then does in the core what the viewers did in their process: the frame of an agent
+ * whose terminal is attached goes out again. (A viewer that moved used to move the core's streams to it
+ * too; they are the viewers' own now, and follow it in their process.)
*
* Attach and detach go to the process as notifications (`service_event`). A detach it missed would leave
* a viewer running, so it is held until the process hears it. An attach it missed is caught up instead:
@@ -21,24 +21,47 @@
* process says of an agent before it hears the agent's detach must not bring the forgotten agent's
* viewer back. The process says everything again on each attach, which is how the core's first word on
* an agent arrives after a core restart, when the process connects before the core has attached anyone.
+ *
+ * A client served a viewer over its connection (core/viewerStreams.ts): each frame of its stream is told
+ * to the process as it comes, never held (a stream dies with the process that held it), and refused
+ * instead while the process is not reading, so a hung one cannot pile a client's uploads up in the core.
+ * What the stream answers comes back as a notice for that connection; one the core cannot deliver ends
+ * that stream in the process, as a frame the forwarder could not send ended it in the core's. A rendered
+ * frame (`viewer_surface`) is asked of the process (`surface`), and answered unavailable, as a routed
+ * request is, if it is down or slower than a client waits.
*/
import type { AgentDshContext } from '../lib/agentFrame.js'
-import type { CoreApi, ViewersPort } from './api.js'
+import { viewerStreamId } from '../lib/viewerFrames.js'
+import { VIEWERS_UNAVAILABLE, type CoreApi, type ViewersPort } from './api.js'
import type { ServiceFrame } from './serviceLinks.js'
/** Tell the viewers' process something (core/serviceLinks.ts `notify`, for `viewers`). */
export type NotifyViewers = (frame: ServiceFrame, opts?: { untilDelivered?: boolean }) => boolean
+/** The rest of the link to the viewers' process (core/serviceLinks.ts `call` and `buffered`, for `viewers`). */
+export interface ViewersChannel {
+ call(type: string, payload: Record<string, unknown>, waitMs?: number): Promise<Record<string, unknown>>
+ buffered(): number
+}
+
+/** Past this many bytes waiting on the socket to the viewers' process, it is not reading: a stream's frame
+ * is refused rather than kept. A client sends at most a viewer window (128 KiB) per stream unacknowledged
+ * (lib/viewerWire.ts), so a process that reads keeps far less than this waiting. */
+export const VIEWERS_BUFFER_LIMIT = 4 * 1024 * 1024
+
+/** How long a client's rendered frame may wait on the viewers: under the 25 s the desktop gives the
+ * request, so it hears why rather than timing out. Starting a headless browser takes a few seconds. */
+export const SURFACE_WAIT_MS = 20_000
+
+const CLOSED: ViewersChannel = { call: async () => VIEWERS_UNAVAILABLE, buffered: () => 0 }
+
/** What the viewers last said of one agent. */
interface Known {
context: AgentDshContext | null
forwardingUrl: string | null
}
-/** The two URLs a change in which the windows' viewer panes must follow. */
-const urls = (known: Known | undefined): string => JSON.stringify([known?.context?.viewerUrl ?? null, known?.forwardingUrl ?? null])
-
-export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, notify: NotifyViewers) {
+export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, notify: NotifyViewers, channel: ViewersChannel = CLOSED) {
const attached = new Set<string>()
const known = new Map<string, Known>()
@@ -50,10 +73,7 @@ export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, not
},
detach: (agentId) => {
attached.delete(agentId)
- const had = known.get(agentId)
known.delete(agentId)
- // The windows' viewer panes let go of its viewer at once, as when it stopped in this process.
- if (urls(had) !== urls(undefined)) core.clients.viewerChanged(agentId)
notify({ type: 'service_event', payload: { kind: 'detach', agentId } }, { untilDelivered: true })
},
frameContext: (session) => (session.dsh ? known.get(session.agentId)?.context ?? null : null),
@@ -61,6 +81,21 @@ export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, not
// The viewers are the process's: a core restarting or stopping leaves them running for the next core,
// and the master stops the process itself when the daemon stops.
stop: async () => {},
+ stream: (connId, type, frame) =>
+ channel.buffered() <= VIEWERS_BUFFER_LIMIT && notify({ type: 'service_event', payload: { kind: 'stream', connId, type, frame } }),
+ surface: (connId, payload) => channel.call('surface', { ...payload, connId }, SURFACE_WAIT_MS),
+ closed: (connId) => { notify({ type: 'service_event', payload: { kind: 'closed', ...(connId === undefined ? {} : { connId }) } }) },
+ }
+
+ /** What the process tells the core without asking: a stream's frame for one client's connection. */
+ const notice = (payload: Record<string, unknown>): void => {
+ if (payload.kind !== 'viewer' || typeof payload.connId !== 'string' || typeof payload.type !== 'string') return
+ const frame = payload.payload && typeof payload.payload === 'object' ? payload.payload as Record<string, unknown> : {}
+ if (core.clients.viewerFrame(payload.connId, payload.type, frame)) return
+ // Not delivered: the stream ends in the process too, without another word to a client it cannot reach.
+ if (payload.type !== 'viewer_close' && viewerStreamId(frame.streamId)) {
+ port.stream(payload.connId, 'viewer_close', { streamId: frame.streamId, error: 'Viewer connection closed' })
+ }
}
/** The process says what one agent's frame says of its harness now. */
@@ -74,7 +109,6 @@ export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, not
const before = known.get(agentId)
if (JSON.stringify(before) === JSON.stringify(next)) return { kept: true }
known.set(agentId, next)
- if (urls(before) !== urls(next)) core.clients.viewerChanged(agentId)
// Only once the agent's terminal is attached: a frame with none reads to the desktop as "agent gone"
// (services/viewers.ts, `syncCompanion`). The attach's own sync carries what arrived first.
const session = core.agents.byAgent(agentId)
@@ -84,6 +118,7 @@ export function createViewersLink(core: Pick<CoreApi, 'agents' | 'clients'>, not
return {
port,
+ notice,
/** The core's answers to the viewers' questions (core/serviceLinks.ts `answer`, for `viewers`). */
answer(query: string, payload: Record<string, unknown>): Record<string, unknown> {
// Every agent the core has with a harness, dormant ones too: the ones it attaches at start.
diff --git a/cli/src/lib/remoteViewerRelay.spec.ts b/cli/src/lib/remoteViewerRelay.spec.ts
index 1e3d49d09..b7ab6e019 100644
--- a/cli/src/lib/remoteViewerRelay.spec.ts
+++ b/cli/src/lib/remoteViewerRelay.spec.ts
@@ -8,6 +8,8 @@ import { gatewayOf, relaySocket } from '../testing/relaySocket.js'
import { attachLocalWsServer } from '../localWsServer.js'
import { env } from '../config/env.js'
import { DshViewerManager } from '../dsh/viewer.js'
+import { createViewerStreams } from '../core/viewerStreams.js'
+import { serveViewers } from '../services/viewers.js'
import { E2eeStore } from './e2ee/store.js'
import { b64e, fingerprint, newIdentity } from './e2ee/core.js'
import { MachinePeerStore } from './e2ee/machinePeers.js'
@@ -93,13 +95,16 @@ Promise.all([
const wss = new WebSocketServer({server});
wss.on('connection', ws => ws.on('message', (data, binary) => ws.send(data, {binary})));
server.listen(Number(process.env.HARNESS_VIEWER_PORT), '127.0.0.1');`)
+ // The daemon's side as the core wires it: the viewers' own serving (services/viewers.ts), reached through
+ // the core's streams (core/viewerStreams.ts), as with HARNESSD_SERVICES=none.
+ const served = serveViewers((agentId) => manager.forwardingUrl(agentId), (connId, type, payload) => daemon.sendViewerFrame(connId, type, payload))
+ daemon.viewerStreams = createViewerStreams(() => served, (connId, type, payload) => daemon.sendViewerFrame(connId, type, payload))
const manager = new DshViewerManager({
onUrl: (agentId, viewerUrl) => {
- daemon.viewerForwarder.refresh(agentId)
+ served.refresh(agentId)
daemon.send({ type: 'agent_synced', payload: { agent: { id: agentId, viewerUrl, name: 'remote viewer fixture' } } })
},
})
- daemon.viewerTargetProvider = (agentId) => manager.forwardingUrl(agentId)
cleanup.push(() => manager.stopAll())
const dsh: InstalledDsh = {
id: 'test/viewer', realDir: dir, dir, source: '', ref: null, commit: null, linked: false, installedAt: 0,
diff --git a/cli/src/lib/viewerFrames.ts b/cli/src/lib/viewerFrames.ts
new file mode 100644
index 000000000..c483eb14f
--- /dev/null
+++ b/cli/src/lib/viewerFrames.ts
@@ -0,0 +1,13 @@
+/**
+ * A viewer stream's frame types and stream ids: what the core gates and hands on, without the stream
+ * itself (lib/viewerWire.ts), which runs where the viewers do (services/viewers.ts). Kept apart so the
+ * core's process, which only routes these frames, does not load the stream and its HTTP forwarding.
+ */
+
+/** Pairwise encrypted JSON frames; no payload may be logged or broadcast. */
+export const VIEWER_DOWN_TYPES = new Set(['viewer_request', 'viewer_data', 'viewer_ack', 'viewer_end', 'viewer_close'])
+export const VIEWER_UP_TYPES = new Set(['viewer_response', 'viewer_data', 'viewer_ack', 'viewer_end', 'viewer_close'])
+
+export function viewerStreamId(value: unknown): value is string {
+ return typeof value === 'string' && /^[a-zA-Z0-9-]{1,64}$/.test(value)
+}
diff --git a/cli/src/lib/viewerWire.ts b/cli/src/lib/viewerWire.ts
index e75334c1d..8ad24af91 100644
--- a/cli/src/lib/viewerWire.ts
+++ b/cli/src/lib/viewerWire.ts
@@ -1,8 +1,6 @@
import { Duplex } from 'node:stream'
-/** Pairwise encrypted JSON frames; no payload may be logged or broadcast. */
-export const VIEWER_DOWN_TYPES = new Set(['viewer_request', 'viewer_data', 'viewer_ack', 'viewer_end', 'viewer_close'])
-export const VIEWER_UP_TYPES = new Set(['viewer_response', 'viewer_data', 'viewer_ack', 'viewer_end', 'viewer_close'])
+export { VIEWER_DOWN_TYPES, VIEWER_UP_TYPES, viewerStreamId } from './viewerFrames.js'
export const VIEWER_CHUNK_BYTES = 32 * 1024
export const VIEWER_WINDOW_BYTES = 128 * 1024
export const VIEWER_MAX_STREAMS = 64
@@ -19,10 +17,6 @@ export function viewerTarget(raw: unknown): URL | null {
} catch { return null }
}
-export function viewerStreamId(value: unknown): value is string {
- return typeof value === 'string' && /^[a-zA-Z0-9-]{1,64}$/.test(value)
-}
-
/**
* A half-closeable, bounded byte stream over ordered frames. Credit is returned only while the
* consumer is draining. This keeps a video or a stalled WebSocket from buffering an entire file
diff --git a/cli/src/services/modelsProcess.spec.ts b/cli/src/services/modelsProcess.spec.ts
index 668d937f7..82f622f20 100644
--- a/cli/src/services/modelsProcess.spec.ts
+++ b/cli/src/services/modelsProcess.spec.ts
@@ -245,6 +245,7 @@ describe('the core API models runs on in its own process', () => {
expect(api.external.open.known()).toEqual(new Map())
expect(await api.external.open.fresh()).toEqual(new Map())
api.clients.viewerChanged('a1')
+ expect(api.clients.viewerFrame('c1', 'viewer_data', {})).toBe(false)
api.clients.dshInstallStatus({})
api.clients.windows({ type: 'orchestrator_changed', payload: {} })
expect(api.clients.observer('observer:x', 'observer_frame', {})).toBe(false)
diff --git a/cli/src/services/modelsProcess.ts b/cli/src/services/modelsProcess.ts
index f0eba8d88..b3ef63754 100644
--- a/cli/src/services/modelsProcess.ts
+++ b/cli/src/services/modelsProcess.ts
@@ -111,6 +111,7 @@ export function modelsCoreApi(dataDir: string, ask: (query: string, payload?: Pa
},
clients: {
viewerChanged: () => {},
+ viewerFrame: () => false,
gridNamed: (name) => told('gridNamed', { name }),
gridModelsChanged: () => told('gridModelsChanged'),
dshInstallStatus: () => {},
diff --git a/cli/src/services/processCoreApi.spec.ts b/cli/src/services/processCoreApi.spec.ts
index a4188e898..e03239f00 100644
--- a/cli/src/services/processCoreApi.spec.ts
+++ b/cli/src/services/processCoreApi.spec.ts
@@ -49,6 +49,7 @@ describe('the core API a light service runs on in its own process', () => {
await expect(api.account.privateGridName()).resolves.toBeNull()
expect(api.account.machineName()).toBeNull()
api.clients.viewerChanged('a1')
+ expect(api.clients.viewerFrame('c1', 'viewer_data', {})).toBe(false)
api.clients.gridNamed('grid')
api.clients.gridModelsChanged()
api.clients.dshInstallStatus({ phase: 'clone' })
diff --git a/cli/src/services/processCoreApi.ts b/cli/src/services/processCoreApi.ts
index fa8432ce9..164f23c1e 100644
--- a/cli/src/services/processCoreApi.ts
+++ b/cli/src/services/processCoreApi.ts
@@ -148,7 +148,7 @@ export function processCoreApi(dataDir: string, service: string, view: AgentsVie
...UNASKED.account,
},
clients: {
- viewerChanged: () => {}, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {},
+ viewerChanged: () => {}, viewerFrame: () => false, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {},
windows: (frame) => { void ask?.('windows', { frame }).catch(() => {}) },
// Handed to the core in order; whether the relay took it is the core's to know, and a lost observer's
// close follows.
diff --git a/cli/src/services/searchProcess.spec.ts b/cli/src/services/searchProcess.spec.ts
index 1af49673a..a4b62f16c 100644
--- a/cli/src/services/searchProcess.spec.ts
+++ b/cli/src/services/searchProcess.spec.ts
@@ -139,6 +139,7 @@ describe('search in its own process', () => {
await expect(api.account.privateGridName()).resolves.toBeNull()
expect(api.account.machineName()).toBeNull()
api.clients.viewerChanged('live')
+ expect(api.clients.viewerFrame('c1', 'viewer_data', {})).toBe(false)
api.clients.gridNamed('grid')
api.clients.gridModelsChanged()
api.clients.dshInstallStatus({ phase: 'clone' })
diff --git a/cli/src/services/searchProcess.ts b/cli/src/services/searchProcess.ts
index e8ec0b5f4..ba5d1be0e 100644
--- a/cli/src/services/searchProcess.ts
+++ b/cli/src/services/searchProcess.ts
@@ -77,7 +77,7 @@ export function searchCoreApi(
...ACCOUNT_BACKEND_OFF,
...UNASKED.account,
},
- clients: { viewerChanged: () => {}, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients },
+ clients: { viewerChanged: () => {}, viewerFrame: () => false, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients },
daemon: DAEMON_UNKNOWN,
wifi: UNASKED.wifi,
}
diff --git a/cli/src/services/viewers.spec.ts b/cli/src/services/viewers.spec.ts
index 216d79a95..e3218cbed 100644
--- a/cli/src/services/viewers.spec.ts
+++ b/cli/src/services/viewers.spec.ts
@@ -8,7 +8,27 @@ import type { RegisteredSession } from '../lib/registry.js'
import { fakeCore } from '../testing/fakeCore.js'
import { startViewers } from './viewers.js'
-const built = vi.hoisted(() => ({ ledger: [] as any[], viewers: [] as any[], verdicts: [] as any[], order: [] as string[] }))
+const built = vi.hoisted(() => ({ ledger: [] as any[], viewers: [] as any[], verdicts: [] as any[], order: [] as string[], forwarders: [] as any[], surfaces: [] as any[] }))
+// The streams and the rendered surfaces are tested where they live (lib/viewerForwarder.spec.ts,
+// lib/interactiveViewer.spec.ts); here, only that the service hands them what it holds.
+vi.mock('../lib/viewerForwarder.js', () => ({
+ ViewerForwarder: class {
+ handle = vi.fn()
+ refresh = vi.fn()
+ closeConnection = vi.fn()
+ closeAll = vi.fn()
+ constructor(readonly deps: any) { built.forwarders.push(this) }
+ },
+}))
+vi.mock('../lib/interactiveViewer.js', () => ({
+ InteractiveViewers: class {
+ request = vi.fn(async () => ({ data: 'jpeg' }))
+ refresh = vi.fn()
+ closeConnection = vi.fn()
+ closeAll = vi.fn()
+ constructor(readonly target: (agentId: string) => string | null) { built.surfaces.push(this) }
+ },
+}))
vi.mock('../dsh/installed.js', () => ({ installedDsh: vi.fn() }))
vi.mock('../dsh/catalog.js', () => ({ catalogEntry: vi.fn() }))
vi.mock('../dsh/manifest.js', () => ({ dshVerdictPath: vi.fn(() => '.harness/verdict.json'), dshViewerName: vi.fn(() => 'Blender view') }))
@@ -42,13 +62,13 @@ const agent = (over: Partial<RegisteredSession> = {}) =>
({ agentId: 'a1', sessionId: 's1', engine: 'claude', dsh: 'blender', cwd: '/work/scene', ...over }) as RegisteredSession
function setup(live: RegisteredSession | null = agent(), terminalAvailable = true) {
- built.ledger.length = 0; built.viewers.length = 0; built.verdicts.length = 0; built.order.length = 0
+ built.ledger.length = 0; built.viewers.length = 0; built.verdicts.length = 0; built.order.length = 0; built.forwarders.length = 0; built.surfaces.length = 0
const core = fakeCore({
agents: { byAgent: vi.fn(() => live ?? undefined), terminalAvailable: vi.fn(() => terminalAvailable), sync: vi.fn() },
})
const ports = emptyPorts()
startViewers(core, ports)
- return { core, port: ports.viewers as ViewersPort, ledger: built.ledger[0], viewers: built.viewers[0], verdicts: built.verdicts[0] }
+ return { core, port: ports.viewers as ViewersPort, ledger: built.ledger[0], viewers: built.viewers[0], verdicts: built.verdicts[0], forwarder: built.forwarders[0], surfaces: built.surfaces[0] }
}
describe('the DSH viewers service', () => {
@@ -178,10 +198,51 @@ describe('the DSH viewers service', () => {
})
it('forwards the windows to the viewer, and stops every viewer and then every watch', async () => {
- const { port, viewers } = setup()
+ const { port, viewers, forwarder, surfaces } = setup()
expect(port.forwardingUrl('a1')).toBe('http://127.0.0.1:9/a1')
expect(viewers.forwardingUrl).toHaveBeenCalledWith('a1')
await port.stop()
expect(built.order.slice(-2)).toEqual(['viewers stopped', 'verdicts stopped'])
+ // No client is left streaming from a viewer that is going.
+ expect(forwarder.closeAll).toHaveBeenCalled()
+ expect(surfaces.closeAll).toHaveBeenCalled()
+ })
+
+ describe('a viewer served to a client over its connection', () => {
+ it('streams from the agent\'s viewer, and answers the one connection that asked through the core', () => {
+ const { core, port, forwarder } = setup()
+ expect(port.stream('c1', 'viewer_request', { streamId: 's1' })).toBe(true)
+ expect(forwarder.handle).toHaveBeenCalledWith('c1', 'viewer_request', { streamId: 's1' })
+ // Only the viewer this service allocated, as the windows' pane forwards to it.
+ expect(forwarder.deps.target('a1')).toBe('http://127.0.0.1:9/a1')
+ vi.mocked(core.clients.viewerFrame).mockReturnValueOnce(true)
+ expect(forwarder.deps.send('c1', 'viewer_response', { streamId: 's1' })).toBe(true)
+ expect(core.clients.viewerFrame).toHaveBeenCalledWith('c1', 'viewer_response', { streamId: 's1' })
+ })
+
+ it('renders a frame of it for a client, from the same viewer', async () => {
+ const { port, surfaces } = setup()
+ await expect(port.surface('c1', { surfaceId: 'v' })).resolves.toEqual({ data: 'jpeg' })
+ expect(surfaces.request).toHaveBeenCalledWith('c1', { surfaceId: 'v' })
+ expect(surfaces.target('a1')).toBe('http://127.0.0.1:9/a1')
+ })
+
+ it('closes a connection\'s streams and surfaces when it goes, and every one when the link does', () => {
+ const { port, forwarder, surfaces } = setup()
+ port.closed('c1')
+ expect(forwarder.closeConnection).toHaveBeenCalledWith('c1')
+ expect(surfaces.closeConnection).toHaveBeenCalledWith('c1')
+ expect(forwarder.closeAll).not.toHaveBeenCalled()
+ port.closed()
+ expect(forwarder.closeAll).toHaveBeenCalled()
+ expect(surfaces.closeAll).toHaveBeenCalled()
+ })
+
+ it('stops forwarding to a viewer that moved or stopped, before another process can take its port', () => {
+ const { viewers, forwarder, surfaces } = setup()
+ viewers.deps.onUrl('a1', null)
+ expect(forwarder.refresh).toHaveBeenCalledWith('a1')
+ expect(surfaces.refresh).toHaveBeenCalledWith('a1')
+ })
})
})
diff --git a/cli/src/services/viewers.ts b/cli/src/services/viewers.ts
index 108a35070..2ec269bd3 100644
--- a/cli/src/services/viewers.ts
+++ b/cli/src/services/viewers.ts
@@ -4,6 +4,12 @@
* verdict file its scripts write. Both are keyed on the agent, attached wherever an agent with a
* `dsh` comes into being (create, restore, discovery) and detached where it is forgotten.
*
+ * It also serves those viewers to a client that cannot reach this machine's loopback (the phone, the web,
+ * a window on another of the owner's machines): a viewer's pages streamed over the client's own connection
+ * (lib/viewerForwarder.ts), and frames of it rendered by a headless browser (lib/interactiveViewer.ts).
+ * They ran in the core's process until 6 October; here they run where the viewer servers they forward to
+ * do, and whatever they cost (a stalled stream, a browser per surface) is the viewers' alone.
+ *
* A service on the core boundary (docs/design/2026-10-03-harnessd.md, step 13): it reads the core
* only through `CoreApi`, and the core reaches it only through `ports.viewers`.
*/
@@ -16,8 +22,39 @@ import { DshVerdictWatcher, type DshVerdict } from '../dsh/verdict.js'
import { DshViewerManager } from '../dsh/viewer.js'
import { ViewerLedger } from '../dsh/viewerLedger.js'
import type { AgentDshContext } from '../lib/agentFrame.js'
+import { InteractiveViewers } from '../lib/interactiveViewer.js'
import { sid } from '../lib/log.js'
import type { RegisteredSession } from '../lib/registry.js'
+import { ViewerForwarder } from '../lib/viewerForwarder.js'
+
+/** This machine's viewers, served to clients over their connections: what `ViewersPort` names `stream`,
+ * `surface` and `closed`, and what follows a viewer that moved or stopped (`refresh`). */
+export function serveViewers(
+ /** Where an agent's viewer is served now, or null. */
+ target: (agentId: string) => string | null,
+ /** A stream's frame to the one connection that opened it (`CoreApi.clients.viewerFrame`). */
+ send: (connId: string, type: string, payload: Record<string, unknown>) => boolean,
+) {
+ const forwarder = new ViewerForwarder({ target, send })
+ const surfaces = new InteractiveViewers(target)
+ return {
+ stream: (connId: string, type: string, payload: Record<string, unknown>): boolean => {
+ forwarder.handle(connId, type, payload)
+ return true
+ },
+ surface: (connId: string, payload: Record<string, unknown>) => surfaces.request(connId, payload),
+ closed: (connId?: string): void => {
+ if (connId === undefined) { forwarder.closeAll(); surfaces.closeAll(); return }
+ forwarder.closeConnection(connId)
+ surfaces.closeConnection(connId)
+ },
+ /** Before another process can take the old viewer's port, nothing may still be forwarded to it. */
+ refresh: (agentId: string): void => {
+ forwarder.refresh(agentId)
+ surfaces.refresh(agentId)
+ },
+ }
+}
export function startViewers(core: CoreApi, ports: CorePorts): void {
const dshFrames = new Map<string, { viewerUrl: string | null; verdict: DshVerdict | null }>()
@@ -54,9 +91,11 @@ export function startViewers(core: CoreApi, ports: CorePorts): void {
// restart. Only pids whose live start time matches what that daemon recorded are touched.
const viewerLedger = new ViewerLedger({ log: (line) => console.log(line) })
viewerLedger.reapOrphans()
+ const served = serveViewers((agentId) => dshViewers.forwardingUrl(agentId), (connId, type, payload) => core.clients.viewerFrame(connId, type, payload))
const dshViewers = new DshViewerManager({
onUrl: (agentId, url) => {
dshFrameFor(agentId).viewerUrl = url
+ served.refresh(agentId)
core.clients.viewerChanged(agentId)
syncCompanion(agentId)
},
@@ -103,8 +142,12 @@ export function startViewers(core: CoreApi, ports: CorePorts): void {
forwardingUrl: (agentId) => dshViewers.forwardingUrl(agentId),
// The next daemon starts its own viewers for the agents it restores; these must not hold the ports.
stop: async () => {
+ served.closed()
await dshViewers.stopAll()
await dshVerdicts.stop()
},
+ stream: served.stream,
+ surface: served.surface,
+ closed: served.closed,
}
}
diff --git a/cli/src/services/viewersProcess.spec.ts b/cli/src/services/viewersProcess.spec.ts
index 41c67cfdf..915e4c8ec 100644
--- a/cli/src/services/viewersProcess.spec.ts
+++ b/cli/src/services/viewersProcess.spec.ts
@@ -21,6 +21,9 @@ function fakeViewers() {
frameContext: vi.fn((session: RegisteredSession) => contexts.get(session.agentId) ?? null),
forwardingUrl: vi.fn((agentId: string) => urls.get(agentId) ?? null),
stop: vi.fn(async () => {}),
+ stream: vi.fn((_connId: string, _type: string, _payload: Record<string, unknown>) => true),
+ surface: vi.fn(async (_connId: string, payload: Record<string, unknown>) => ({ data: 'jpeg', asked: payload })),
+ closed: vi.fn((_connId?: string) => {}),
} satisfies ViewersPort
return { port, contexts, urls }
}
@@ -54,7 +57,50 @@ describe('the viewers in their own process', () => {
it('reaches the core as `viewers`, through the socket and token it was given, and answers the apps nothing', () => {
const { options } = setup()
expect(options).toMatchObject({ name: 'viewers', socketPath: '/data/daemon-1.sock', machineId: 'm', token: 't' })
- expect(options.requests).toEqual({})
+ // Only the core's own question, which no client's request is routed as (core/viewersLink.ts).
+ expect(Object.keys(options.requests)).toEqual(['surface'])
+ })
+
+ it('serves a client\'s viewer stream through the core: its frames in, its answers out to that connection', async () => {
+ const { viewers, api, options } = setup()
+ // Not connected: an answer has nowhere to go, and the stream ends rather than waiting.
+ expect(api.clients.viewerFrame('c1', 'viewer_response', { streamId: 's1' })).toBe(false)
+ const notice = vi.fn()
+ options.onConnected!({ ...connection().core, notice })
+ options.onEvent!({ kind: 'stream', connId: 'c1', type: 'viewer_request', frame: { streamId: 's1', path: '/' } })
+ expect(viewers.port.stream).toHaveBeenCalledWith('c1', 'viewer_request', { streamId: 's1', path: '/' })
+ options.onEvent!({ kind: 'stream', connId: 'c1', type: 'viewer_end' })
+ expect(viewers.port.stream).toHaveBeenLastCalledWith('c1', 'viewer_end', {})
+ expect(api.clients.viewerFrame('c1', 'viewer_response', { streamId: 's1', status: 200 })).toBe(true)
+ expect(notice).toHaveBeenCalledWith('viewer', { connId: 'c1', type: 'viewer_response', payload: { streamId: 's1', status: 200 } })
+ // A rendered frame, for the connection the core says asked; the connection is not the surface's to read.
+ await expect(options.requests.surface({ connId: 'c1', surfaceId: 'v', op: 'frame' }, { local: true, owner: true }))
+ .resolves.toEqual({ data: 'jpeg', asked: { surfaceId: 'v', op: 'frame' } })
+ expect(viewers.port.surface).toHaveBeenCalledWith('c1', { surfaceId: 'v', op: 'frame' })
+ // A connection gone, then every one: the link went down.
+ options.onEvent!({ kind: 'closed', connId: 'c1' })
+ options.onEvent!({ kind: 'closed' })
+ expect(viewers.port.closed.mock.calls).toEqual([['c1'], [undefined]])
+ // What is not a stream's frame is not passed on.
+ options.onEvent!({ kind: 'stream', connId: 7, type: 'viewer_data' })
+ options.onEvent!({ kind: 'stream', connId: 'c1' })
+ expect(viewers.port.stream).toHaveBeenCalledTimes(2)
+ })
+
+ it('ends every stream and surface when the core goes: their connections were that core\'s', () => {
+ const { viewers, api, options } = setup()
+ const notice = vi.fn()
+ options.onConnected!({ ...connection().core, notice })
+ options.onDisconnected!()
+ expect(viewers.port.closed).toHaveBeenCalledWith()
+ expect(api.clients.viewerFrame('c1', 'viewer_data', { streamId: 's1' })).toBe(false)
+ expect(notice).not.toHaveBeenCalled()
+ })
+
+ it('cannot answer a stream over a connection with no way to say it', () => {
+ const { api, options } = setup()
+ options.onConnected!(connection().core)
+ expect(api.clients.viewerFrame('c1', 'viewer_data', { streamId: 's1' })).toBe(false)
})
it('attaches what the core attaches, tells the core what the agent\'s frame says, and detaches what it detaches', async () => {
@@ -239,6 +285,8 @@ describe('the viewers in their own process', () => {
api.clients.dshInstallStatus({ phase: 'clone' })
api.clients.windows({ type: 'orchestrator_changed', payload: {} })
expect(api.clients.observer('observer:x', 'observer_frame', {})).toBe(false)
+ // Built on its own, it reaches no client.
+ expect(api.clients.viewerFrame('c1', 'viewer_data', {})).toBe(false)
})
it('runs as a real service by default: its viewers, on its own link to the core', async () => {
diff --git a/cli/src/services/viewersProcess.ts b/cli/src/services/viewersProcess.ts
index a4a113f7a..9c42fbbea 100644
--- a/cli/src/services/viewersProcess.ts
+++ b/cli/src/services/viewersProcess.ts
@@ -16,6 +16,12 @@
* alone: a crash or a leak costs the viewers, and the master starts them again. A new process stops the
* viewer servers a crashed one left running before it starts its own (dsh/viewerLedger.ts). A core restart
* no longer restarts every viewer: they stay up here, and the new core hears their URLs at once.
+ *
+ * A client served a viewer over its connection (services/viewers.ts `serveViewers`) reaches this process
+ * through the core: each frame of its stream is told here (`service_event` `stream`), and what the stream
+ * answers goes back to that connection through the core (`service_notice` `viewer`). A rendered frame
+ * (`viewer_surface`) is the core's question (`surface`), which only the core asks. The connections are the
+ * core's: when it goes, every stream it carried goes with it.
*/
import type { CoreApi } from '../core/api.js'
import { ACCOUNT_BACKEND_OFF, AGENT_ACTIONS_OFF, DAEMON_UNKNOWN, DELIVERIES_OFF, emptyPorts, LANE_OFF, resolveAgent, TERMINALS_OFF } from '../core/api.js'
@@ -44,7 +50,12 @@ const isSession = (value: unknown): value is RegisteredSession =>
* what the service would do to the core's windows. Whether an agent's terminal is attached is the core's
* to judge when it hears; what the viewers never ask (search, sign-in) answers as nothing.
*/
-export function viewersCoreApi(dataDir: string, sessions: ReadonlyMap<string, RegisteredSession>, tell: (agentId: string) => void): CoreApi {
+export function viewersCoreApi(
+ dataDir: string,
+ sessions: ReadonlyMap<string, RegisteredSession>,
+ tell: (agentId: string) => void,
+ viewerFrame: CoreApi['clients']['viewerFrame'] = () => false,
+): CoreApi {
const attached = (): RegisteredSession[] => [...sessions.values()]
return {
dataDir,
@@ -83,7 +94,7 @@ export function viewersCoreApi(dataDir: string, sessions: ReadonlyMap<string, Re
...ACCOUNT_BACKEND_OFF,
...UNASKED.account,
},
- clients: { viewerChanged: tell, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients },
+ clients: { viewerChanged: tell, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients, viewerFrame },
daemon: DAEMON_UNKNOWN,
wifi: UNASKED.wifi,
}
@@ -111,7 +122,13 @@ export function runViewersService(options: ViewersServiceOptions): ServiceProces
void core.query('context', state).catch(() => {})
}
- ;(options.start ?? startViewers)(viewersCoreApi(options.dataDir, sessions, tell), ports)
+ /** A stream's frame for one client: through the core, which alone holds the client's connection. */
+ const viewerFrame = (connId: string, type: string, payload: Record<string, unknown>): boolean => {
+ if (!core?.notice) return false
+ core.notice('viewer', { connId, type, payload })
+ return true
+ }
+ ;(options.start ?? startViewers)(viewersCoreApi(options.dataDir, sessions, tell, viewerFrame), ports)
if (!ports.viewers) throw new Error('the viewers did not start')
const viewers = ports.viewers
@@ -155,10 +172,20 @@ export function runViewersService(options: ViewersServiceOptions): ServiceProces
socketPath: options.socketPath,
machineId: options.machineId,
token: options.token,
- // The apps ask the viewers nothing: only the core does, through the context it keeps.
- requests: {},
+ // The apps ask the viewers nothing directly. The core asks for a client's rendered frame (`surface`),
+ // under a type no client's request is routed as, with the connection the client asked over.
+ requests: {
+ surface: (payload) => {
+ const { connId, ...asked } = payload
+ return viewers.surface(String(connId), asked)
+ },
+ },
onEvent: (payload) => {
- if (payload.kind === 'attach' && isSession(payload.session)) {
+ if (payload.kind === 'stream' && typeof payload.connId === 'string' && typeof payload.type === 'string') {
+ viewers.stream(payload.connId, payload.type, payload.frame && typeof payload.frame === 'object' ? payload.frame as Record<string, unknown> : {})
+ } else if (payload.kind === 'closed') {
+ viewers.closed(typeof payload.connId === 'string' ? payload.connId : undefined)
+ } else if (payload.kind === 'attach' && isSession(payload.session)) {
heard.add(payload.session.agentId)
attach(payload.session)
} else if (payload.kind === 'detach' && typeof payload.agentId === 'string') {
@@ -174,6 +201,11 @@ export function runViewersService(options: ViewersServiceOptions): ServiceProces
// context: a new core hears everything, and one that only dropped the connection what changed.
void connection.query('agents').then((answer) => { if (core === connection) catchUp(answer.agents) }, () => {})
},
+ // Every client's connection was that core's: their streams and surfaces end with it.
+ onDisconnected: () => {
+ core = null
+ viewers.closed()
+ },
})
return { stop }
}
diff --git a/cli/src/services/workspacesProcess.spec.ts b/cli/src/services/workspacesProcess.spec.ts
index 8c34a926e..6754fb531 100644
--- a/cli/src/services/workspacesProcess.spec.ts
+++ b/cli/src/services/workspacesProcess.spec.ts
@@ -179,6 +179,7 @@ describe('workspaces in their own process', () => {
await expect(api.questions.answerReviewed({} as never)).resolves.toBe(false)
api.clients.gridModelsChanged()
api.clients.viewerChanged('a1')
+ expect(api.clients.viewerFrame('c1', 'viewer_data', {})).toBe(false)
api.clients.gridNamed('grid')
api.clients.dshInstallStatus({ phase: 'clone' })
api.clients.windows({ type: 'orchestrator_changed', payload: {} })
diff --git a/cli/src/services/workspacesProcess.ts b/cli/src/services/workspacesProcess.ts
index 1ad864f09..7be9df8e9 100644
--- a/cli/src/services/workspacesProcess.ts
+++ b/cli/src/services/workspacesProcess.ts
@@ -85,7 +85,7 @@ export function workspacesCoreApi(
...ACCOUNT_BACKEND_OFF,
...UNASKED.account,
},
- clients: { viewerChanged: () => {}, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients },
+ clients: { viewerChanged: () => {}, viewerFrame: () => false, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {}, windows: () => {}, observer: () => false, ...UNASKED.clients },
daemon: DAEMON_UNKNOWN,
wifi: UNASKED.wifi,
}
diff --git a/cli/src/testing/fakeCore.ts b/cli/src/testing/fakeCore.ts
index 57b825cd3..9a1442f8c 100644
--- a/cli/src/testing/fakeCore.ts
+++ b/cli/src/testing/fakeCore.ts
@@ -69,7 +69,7 @@ export function fakeCore(over: Overrides = {}): CoreApi {
...over.account,
},
clients: {
- viewerChanged: vi.fn(), gridNamed: vi.fn(), gridModelsChanged: vi.fn(), dshInstallStatus: vi.fn(), windows: vi.fn(), observer: vi.fn(() => true),
+ viewerChanged: vi.fn(), viewerFrame: vi.fn(() => false), gridNamed: vi.fn(), gridModelsChanged: vi.fn(), dshInstallStatus: vi.fn(), windows: vi.fn(), observer: vi.fn(() => true),
sendLocal: vi.fn(), sendToWindow: vi.fn(() => true), hasWindow: vi.fn(() => true), devicesChanged: vi.fn(), dialWatching: vi.fn(),
...over.clients,
},
diff --git a/cli/src/testing/teamFixture.ts b/cli/src/testing/teamFixture.ts
index a70e837c3..c91894832 100644
--- a/cli/src/testing/teamFixture.ts
+++ b/cli/src/testing/teamFixture.ts
@@ -71,7 +71,7 @@ export function attachTeams<T extends BackendSocket>(socket: T): T & TeamFixture
},
},
clients: {
- viewerChanged: () => {}, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {},
+ viewerChanged: () => {}, viewerFrame: () => false, gridNamed: () => {}, gridModelsChanged: () => {}, dshInstallStatus: () => {},
windows: (frame) => socket.sendLocal(frame),
observer: () => false,
...UNASKED.clients,
diff --git a/cli/vitest.viewer.config.ts b/cli/vitest.viewer.config.ts
index 9e3b22f31..154227b60 100644
--- a/cli/vitest.viewer.config.ts
+++ b/cli/vitest.viewer.config.ts
@@ -15,7 +15,7 @@ export default defineConfig({
coverage: {
enabled: true,
provider: 'v8',
- include: ['src/lib/viewerWire.ts', 'src/lib/viewerForwarder.ts', 'src/lib/remoteViewerProxy.ts'],
+ include: ['src/lib/viewerFrames.ts', 'src/lib/viewerWire.ts', 'src/lib/viewerForwarder.ts', 'src/lib/remoteViewerProxy.ts'],
reporter: ['text', 'json-summary', 'json', 'html'],
reportsDirectory: 'coverage/remote-viewers',
thresholds: { perFile: true, statements: 100, branches: 100, functions: 100, lines: 100 },