The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
harness-shares: one machine read per Swarm, and a burst of share changes tells each recipient once (#3925) (#3415)
harness-for-autonomous[bot]20111ba134cf
diff --git a/backend/integration/harnessSharesList.test.ts b/backend/integration/harnessSharesList.test.tsindex 8119c3be3..72268bac1 100644--- a/backend/integration/harnessSharesList.test.ts+++ b/backend/integration/harnessSharesList.test.ts@@ -54,7 +54,7 @@ const list = async (as: AuthUser = recipient) => {const listedIds = (machines: Awaited<ReturnType<typeof list>>) => machines.flatMap(m => m.shares.map(s => s.id)).sort()const percentile = (values: number[], p: number) => [...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * p))]!-interface Profiled { collection: string; plans: string[]; keysExamined: number; docsExamined: number; returned: number }+interface Profiled { collection: string; plans: string[]; keysExamined: number; docsExamined: number; returned: number; reads: number }/** What Mongo did for each collection while `read` ran: plans, documents examined and rows returned. */async function profiled(read: () => Promise<unknown>): Promise<Profiled[]> {const db = database.db@@ -68,7 +68,8 @@ async function profiled(read: () => Promise<unknown>): Promise<Profiled[]> {for (const row of found.cursor.firstBatch) {const collection = row.ns.slice(row.ns.indexOf('.') + 1)if (collection.startsWith('system.') || collection.startsWith('$cmd')) continue- const entry = byCollection.get(collection) ?? { collection, plans: [], keysExamined: 0, docsExamined: 0, returned: 0 }+ const entry = byCollection.get(collection) ?? { collection, plans: [], keysExamined: 0, docsExamined: 0, returned: 0, reads: 0 }+ entry.reads++if (row.planSummary && !entry.plans.includes(row.planSummary)) entry.plans.push(row.planSummary)entry.keysExamined += row.keysExamined ?? 0entry.docsExamined += row.docsExamined ?? 0@@ -166,6 +167,17 @@ describe('the harness shares list on a Swarm-sized database', () => {expect(numbers.waveOf20Ms).toBeLessThan(1000)})+ // Task #3925 (Ken's scan, 11-06 16:20 VN): after v1.4.24 the list cost 19.6 s of database time per pod-minute, mostly one+ // Machine.findFirst per machine its shared agents run on (~850 a pod-minute). One list reads its machines at most twice:+ // the route's own read and one for every grant it checks, however many machines the Swarm's agents run on.+ it('reads the machines at most twice a call, however many machines its shared agents run on', async () => {+ await list()+ const reads = await profiled(list)+ const machines = reads.find(r => r.collection === 'machines')+ expect(CONFIG.machines).toBeGreaterThan(2)+ expect(machines?.reads ?? 0).toBeLessThanOrEqual(2)+ })+it('reads every collection through an index, examining about as many documents as it returns', async () => {const reads = await profiled(list)process.stdout.write(`harness shares list reads: ${JSON.stringify(reads)}\n`)diff --git a/backend/src/routes/harnessShares.test.ts b/backend/src/routes/harnessShares.test.tsindex edabd27b6..ab334a1f6 100644--- a/backend/src/routes/harnessShares.test.ts+++ b/backend/src/routes/harnessShares.test.ts@@ -19,7 +19,7 @@ vi.mock('../lib/prisma.js', () => ({ prisma: mocks.prisma, machineAlive: { OR: [vi.mock('../lib/bus.js', () => ({ getAgentPresenceMany: mocks.presence, publishShareChanged: mocks.changed,publishDeviceMachineListChanged: mocks.listChanged }))vi.mock('../lib/ssoAuth.js', async original => ({ ...await original<typeof import('../lib/ssoAuth.js')>(), authenticateAccessToken: mocks.auth }))-import { harnessShareRoutes, recipientShare } from './harnessShares.js'+import { harnessShareRoutes, recipientShare, SHARES_TOLD_MS } from './harnessShares.js'import { registerAuthMiddleware } from '../middlewares/authMiddleware.js'import { errorHandler } from '../middlewares/errorHandler.js'import { logger } from '../utils/logger.js'@@ -191,6 +191,25 @@ describe('account-bound harness invitations', () => {// Named in the log, so a burst of `machines_changed` on someone's app can be traced to the invitation behind it.expect(info).toHaveBeenCalledWith('harness share: told the recipient it changed', { shareId: id, recipientId: 'ken' })})+ // Task #3925 (Ken's scan, 11-06): a Swarm's agent restarts made a share per new agent for each member, and each one told every+ // app of theirs to read its shares again: a burst of starts was a burst of share-list reads. A recipient is told at once, then+ // at most once more a window, so the last change is never lost.+ it('tells a recipient once for a burst of changes, then once more when the burst is over', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ try {+ mocks.prisma.user.findUnique.mockResolvedValue({ id: 'ken' })+ for (let i = 0; i < 5; i++) {+ const id = randomUUID()+ mocks.prisma.harnessShare.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce({ id, recipientEmail: '[email protected]' })+ expect((await put({ ...input(), agentId: `agent-${i}` }, id)).statusCode).toBe(200)+ }+ expect(mocks.listChanged).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(SHARES_TOLD_MS)+ expect(mocks.listChanged).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(SHARES_TOLD_MS * 2)+ expect(mocks.listChanged).toHaveBeenCalledTimes(2)+ } finally { vi.useRealTimers() }+ })it('tells the invited account when the owner takes the invitation back', async () => {const id = randomUUID()mocks.prisma.harnessShare.findUnique.mockResolvedValue({ id, recipientEmail: '[email protected]' })diff --git a/backend/src/routes/harnessShares.ts b/backend/src/routes/harnessShares.tsindex 6502a9e43..799a4bab6 100644--- a/backend/src/routes/harnessShares.ts+++ b/backend/src/routes/harnessShares.ts@@ -35,13 +35,37 @@ const body = z.object({* it changed, and every few minutes otherwise.* Best effort — the invitation is already persisted, and an address with no account has nobody to tell.*/-async function tellRecipient(shareId: string): Promise<void> {+/** How often at most a recipient's apps are told their shares changed (task #3925): told at once, then once more after a burst.+ * A Swarm's agent restarts made a share per new agent for each member, and each told every app of theirs to read its shares+ * again: harness-shares became the backend's top database cost (Ken's scan, 11-06 16:20 VN). */+export const SHARES_TOLD_MS = 10_000++/** Tells a recipient's apps their shares changed: at once, then at most once a SHARES_TOLD_MS, with one more at the end of a+ * burst so the last change is never lost. Per server process, which a recipient's apps each hear through the bus. */+function shareListTeller(): (recipientId: string) => Promise<void> {+ const held = new Map<string, { again: boolean }>()+ const tell = async (recipientId: string): Promise<void> => {+ const window = held.get(recipientId)+ if (window) { window.again = true; return }+ const next = { again: false }+ held.set(recipientId, next)+ setTimeout(() => {+ held.delete(recipientId)+ // The bus never rejects a publish (bus.ts safePublish): nothing to catch.+ if (next.again) void tell(recipientId)+ }, SHARES_TOLD_MS).unref?.()+ await publishDeviceMachineListChanged(recipientId, { reason: 'harness_shares' })+ }+ return tell+}++async function tellRecipient(shareId: string, tell: (recipientId: string) => Promise<void>): Promise<void> {try {const share = await prisma.harnessShare.findUnique({ where: { id: shareId } })if (!share) returnconst recipient = await prisma.user.findUnique({ where: { email: share.recipientEmail }, select: { id: true } })if (recipient) {- await publishDeviceMachineListChanged(recipient.id, { reason: 'harness_shares' })+ await tell(recipient.id)logger.info('harness share: told the recipient it changed', { shareId, recipientId: recipient.id })}// A private link admits the recipient only through this invitation (harnessLinks.ts recipientLink): its viewers check again.@@ -66,6 +90,7 @@ export async function recipientShare(id: string, user: AuthUser, db: PrismaClien}export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {+ const tellShareList = shareListTeller()app.get('/api/harness-shares', async (req, reply) => {const shares = await prisma.harnessShare.findMany({ where: {recipientEmail: req.user!.email.trim().toLowerCase(), autonomousEnv: req.user!.autonomousEnv,@@ -138,7 +163,7 @@ export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {if (!existing || existing.revokedAt != null || existing.name !== data.name || existing.engine !== data.engine|| (existing.superTabId ?? null) !== data.superTabId || existing.ownerPublicKey !== data.ownerPublicKey) {await publishShareChanged(req.params.id)- await tellRecipient(req.params.id)+ await tellRecipient(req.params.id, tellShareList)}sendSuccess(reply, { id: req.params.id })},@@ -157,7 +182,7 @@ export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {return sendError(reply, 'Invitation not found.', 'NOT_FOUND', 404)}await publishShareChanged(req.params.id)- await tellRecipient(req.params.id)+ await tellRecipient(req.params.id, tellShareList)sendSuccess(reply, { removed: true })})}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 860a7face..7ad39d82b 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -584,6 +584,19 @@ export class GrantReads {this.answers.set(key, answer)return answer}++ /** Every machine of [wanted] not read yet, in one read, with the grant's own filter (machineAlive): each grant check then finds+ * its machine here, or none, instead of a read of its own (task #3925: the harness shares list read one per machine its+ * shared agents run on, ~850 a pod-minute after v1.4.24). */+ async machines(db: Pick<PrismaClient, 'machine'>, environment: string, wanted: ReadonlyArray<{ machineId: string; ownerId: string }>): Promise<void> {+ const missing = wanted.filter(w => !this.answers.has(machineKey(environment, w.machineId, w.ownerId)))+ if (!missing.length) return+ const rows = db.machine.findMany({ where: { machineId: { in: [...new Set(missing.map(w => w.machineId))] }, autonomousEnv: environment, ...machineAlive } })+ for (const w of missing) {+ this.answers.set(machineKey(environment, w.machineId, w.ownerId), rows.then(found => found.find(m => m.machineId === w.machineId && m.userId === w.ownerId) ?? null))+ }+ await rows+ }}/** Another Swarm too busy to say whether a shared session is confidential there: the list hides those shares and lists the rest,@@ -4425,7 +4438,9 @@ export class SuperTabStore {const byTab = new Map<string, T[]>()for (const share of shares) if (share.superTabId) byTab.set(share.superTabId, [...byTab.get(share.superTabId) ?? [], share])const observable = new Set<T>()- for (const [tabId, tabShares] of byTab) for (const share of await this.observable(user, tabId, tabShares)) observable.add(share)+ // One request's grant reads, shared by every Swarm it lists (task #3925).+ const reads = new GrantReads()+ for (const [tabId, tabShares] of byTab) for (const share of await this.observable(user, tabId, tabShares, reads)) observable.add(share)return shares.filter(s => s.superTabId ? observable.has(s) : user.sub === s.ownerId || !anywhere.has(key(s)))}@@ -4434,7 +4449,7 @@ export class SuperTabStore {reassemble(await readParts(this.db, await this.view([tabId]), tabId, { agentIds }), { share: true })/** mayObserve for each of `shares`, all in Swarm `tabId`, from one load of it. */- private async observable<T extends { machineId: string; agentId: string; ownerId: string }>(user: AuthUser, tabId: string, shares: T[]): Promise<T[]> {+ private async observable<T extends { machineId: string; agentId: string; ownerId: string }>(user: AuthUser, tabId: string, shares: T[], reads = new GrantReads()): Promise<T[]> {if (isTeamSession(user)) return []let state: SuperTabStatetry {@@ -4446,11 +4461,13 @@ export class SuperTabStore {const grants = new Map<string, Promise<boolean>>()const granted = (target: Agent) => {const key = JSON.stringify([target.contributorId, target.machineId, target.machineOwnerId, target.hostGrant])- if (!grants.has(key)) grants.set(key, this.agentGrant(this.db, state, target).then(() => true,+ if (!grants.has(key)) grants.set(key, this.agentGrant(this.db, state, target, reads).then(() => true,error => { if (error instanceof SuperTabError) return false; throw error }))return grants.get(key)!}const targets = shares.map(share => state.agents.find(a => a.machineId === share.machineId && a.harnessId === share.agentId && a.machineOwnerId === share.ownerId))+ // The machines every grant below reads, in one read for the Swarm (task #3925).+ await reads.machines(this.db, state.environment, targets.flatMap(t => t ? [{ machineId: t.machineId, ownerId: t.machineOwnerId }] : []))// Confidential in another Swarm closes it here too; in this one, its current Lead stays open to readers (projects.ts watchClosed).const elsewhere = await confidentialSessions(this.db, targets.flatMap(t => t ? [t.id] : []), tabId, this.cachedState, busyListing)const allowed: T[] = []