Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

harness-shares: one machine read per Swarm, and a burst of share changes tells each recipient once (#3925) (#3415)

harness-for-autonomous[bot]20111ba134cf
diff --git a/backend/integration/harnessSharesList.test.ts b/backend/integration/harnessSharesList.test.ts
index 8119c3be3..72268bac1 100644
--- a/backend/integration/harnessSharesList.test.ts
+++ b/backend/integration/harnessSharesList.test.ts
@@ -54,7 +54,7 @@ const list = async (as: AuthUser = recipient) => {
const listedIds = (machines: Awaited<ReturnType<typeof list>>) => machines.flatMap(m => m.shares.map(s => s.id)).sort()
const percentile = (values: number[], p: number) => [...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * p))]!
-interface Profiled { collection: string; plans: string[]; keysExamined: number; docsExamined: number; returned: number }
+interface Profiled { collection: string; plans: string[]; keysExamined: number; docsExamined: number; returned: number; reads: number }
/** What Mongo did for each collection while `read` ran: plans, documents examined and rows returned. */
async function profiled(read: () => Promise<unknown>): Promise<Profiled[]> {
const db = database.db
@@ -68,7 +68,8 @@ async function profiled(read: () => Promise<unknown>): Promise<Profiled[]> {
for (const row of found.cursor.firstBatch) {
const collection = row.ns.slice(row.ns.indexOf('.') + 1)
if (collection.startsWith('system.') || collection.startsWith('$cmd')) continue
- const entry = byCollection.get(collection) ?? { collection, plans: [], keysExamined: 0, docsExamined: 0, returned: 0 }
+ const entry = byCollection.get(collection) ?? { collection, plans: [], keysExamined: 0, docsExamined: 0, returned: 0, reads: 0 }
+ entry.reads++
if (row.planSummary && !entry.plans.includes(row.planSummary)) entry.plans.push(row.planSummary)
entry.keysExamined += row.keysExamined ?? 0
entry.docsExamined += row.docsExamined ?? 0
@@ -166,6 +167,17 @@ describe('the harness shares list on a Swarm-sized database', () => {
expect(numbers.waveOf20Ms).toBeLessThan(1000)
})
+ // Task #3925 (Ken's scan, 11-06 16:20 VN): after v1.4.24 the list cost 19.6 s of database time per pod-minute, mostly one
+ // Machine.findFirst per machine its shared agents run on (~850 a pod-minute). One list reads its machines at most twice:
+ // the route's own read and one for every grant it checks, however many machines the Swarm's agents run on.
+ it('reads the machines at most twice a call, however many machines its shared agents run on', async () => {
+ await list()
+ const reads = await profiled(list)
+ const machines = reads.find(r => r.collection === 'machines')
+ expect(CONFIG.machines).toBeGreaterThan(2)
+ expect(machines?.reads ?? 0).toBeLessThanOrEqual(2)
+ })
+
it('reads every collection through an index, examining about as many documents as it returns', async () => {
const reads = await profiled(list)
process.stdout.write(`harness shares list reads: ${JSON.stringify(reads)}\n`)
diff --git a/backend/src/routes/harnessShares.test.ts b/backend/src/routes/harnessShares.test.ts
index edabd27b6..ab334a1f6 100644
--- a/backend/src/routes/harnessShares.test.ts
+++ b/backend/src/routes/harnessShares.test.ts
@@ -19,7 +19,7 @@ vi.mock('../lib/prisma.js', () => ({ prisma: mocks.prisma, machineAlive: { OR: [
vi.mock('../lib/bus.js', () => ({ getAgentPresenceMany: mocks.presence, publishShareChanged: mocks.changed,
publishDeviceMachineListChanged: mocks.listChanged }))
vi.mock('../lib/ssoAuth.js', async original => ({ ...await original<typeof import('../lib/ssoAuth.js')>(), authenticateAccessToken: mocks.auth }))
-import { harnessShareRoutes, recipientShare } from './harnessShares.js'
+import { harnessShareRoutes, recipientShare, SHARES_TOLD_MS } from './harnessShares.js'
import { registerAuthMiddleware } from '../middlewares/authMiddleware.js'
import { errorHandler } from '../middlewares/errorHandler.js'
import { logger } from '../utils/logger.js'
@@ -191,6 +191,25 @@ describe('account-bound harness invitations', () => {
// Named in the log, so a burst of `machines_changed` on someone's app can be traced to the invitation behind it.
expect(info).toHaveBeenCalledWith('harness share: told the recipient it changed', { shareId: id, recipientId: 'ken' })
})
+ // Task #3925 (Ken's scan, 11-06): a Swarm's agent restarts made a share per new agent for each member, and each one told every
+ // app of theirs to read its shares again: a burst of starts was a burst of share-list reads. A recipient is told at once, then
+ // at most once more a window, so the last change is never lost.
+ it('tells a recipient once for a burst of changes, then once more when the burst is over', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ try {
+ mocks.prisma.user.findUnique.mockResolvedValue({ id: 'ken' })
+ for (let i = 0; i < 5; i++) {
+ const id = randomUUID()
+ mocks.prisma.harnessShare.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce({ id, recipientEmail: '[email protected]' })
+ expect((await put({ ...input(), agentId: `agent-${i}` }, id)).statusCode).toBe(200)
+ }
+ expect(mocks.listChanged).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(SHARES_TOLD_MS)
+ expect(mocks.listChanged).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(SHARES_TOLD_MS * 2)
+ expect(mocks.listChanged).toHaveBeenCalledTimes(2)
+ } finally { vi.useRealTimers() }
+ })
it('tells the invited account when the owner takes the invitation back', async () => {
const id = randomUUID()
mocks.prisma.harnessShare.findUnique.mockResolvedValue({ id, recipientEmail: '[email protected]' })
diff --git a/backend/src/routes/harnessShares.ts b/backend/src/routes/harnessShares.ts
index 6502a9e43..799a4bab6 100644
--- a/backend/src/routes/harnessShares.ts
+++ b/backend/src/routes/harnessShares.ts
@@ -35,13 +35,37 @@ const body = z.object({
* it changed, and every few minutes otherwise.
* Best effort — the invitation is already persisted, and an address with no account has nobody to tell.
*/
-async function tellRecipient(shareId: string): Promise<void> {
+/** How often at most a recipient's apps are told their shares changed (task #3925): told at once, then once more after a burst.
+ * A Swarm's agent restarts made a share per new agent for each member, and each told every app of theirs to read its shares
+ * again: harness-shares became the backend's top database cost (Ken's scan, 11-06 16:20 VN). */
+export const SHARES_TOLD_MS = 10_000
+
+/** Tells a recipient's apps their shares changed: at once, then at most once a SHARES_TOLD_MS, with one more at the end of a
+ * burst so the last change is never lost. Per server process, which a recipient's apps each hear through the bus. */
+function shareListTeller(): (recipientId: string) => Promise<void> {
+ const held = new Map<string, { again: boolean }>()
+ const tell = async (recipientId: string): Promise<void> => {
+ const window = held.get(recipientId)
+ if (window) { window.again = true; return }
+ const next = { again: false }
+ held.set(recipientId, next)
+ setTimeout(() => {
+ held.delete(recipientId)
+ // The bus never rejects a publish (bus.ts safePublish): nothing to catch.
+ if (next.again) void tell(recipientId)
+ }, SHARES_TOLD_MS).unref?.()
+ await publishDeviceMachineListChanged(recipientId, { reason: 'harness_shares' })
+ }
+ return tell
+}
+
+async function tellRecipient(shareId: string, tell: (recipientId: string) => Promise<void>): Promise<void> {
try {
const share = await prisma.harnessShare.findUnique({ where: { id: shareId } })
if (!share) return
const recipient = await prisma.user.findUnique({ where: { email: share.recipientEmail }, select: { id: true } })
if (recipient) {
- await publishDeviceMachineListChanged(recipient.id, { reason: 'harness_shares' })
+ await tell(recipient.id)
logger.info('harness share: told the recipient it changed', { shareId, recipientId: recipient.id })
}
// A private link admits the recipient only through this invitation (harnessLinks.ts recipientLink): its viewers check again.
@@ -66,6 +90,7 @@ export async function recipientShare(id: string, user: AuthUser, db: PrismaClien
}
export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {
+ const tellShareList = shareListTeller()
app.get('/api/harness-shares', async (req, reply) => {
const shares = await prisma.harnessShare.findMany({ where: {
recipientEmail: req.user!.email.trim().toLowerCase(), autonomousEnv: req.user!.autonomousEnv,
@@ -138,7 +163,7 @@ export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {
if (!existing || existing.revokedAt != null || existing.name !== data.name || existing.engine !== data.engine
|| (existing.superTabId ?? null) !== data.superTabId || existing.ownerPublicKey !== data.ownerPublicKey) {
await publishShareChanged(req.params.id)
- await tellRecipient(req.params.id)
+ await tellRecipient(req.params.id, tellShareList)
}
sendSuccess(reply, { id: req.params.id })
},
@@ -157,7 +182,7 @@ export async function harnessShareRoutes(app: FastifyInstance): Promise<void> {
return sendError(reply, 'Invitation not found.', 'NOT_FOUND', 404)
}
await publishShareChanged(req.params.id)
- await tellRecipient(req.params.id)
+ await tellRecipient(req.params.id, tellShareList)
sendSuccess(reply, { removed: true })
})
}
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 860a7face..7ad39d82b 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -584,6 +584,19 @@ export class GrantReads {
this.answers.set(key, answer)
return answer
}
+
+ /** Every machine of [wanted] not read yet, in one read, with the grant's own filter (machineAlive): each grant check then finds
+ * its machine here, or none, instead of a read of its own (task #3925: the harness shares list read one per machine its
+ * shared agents run on, ~850 a pod-minute after v1.4.24). */
+ async machines(db: Pick<PrismaClient, 'machine'>, environment: string, wanted: ReadonlyArray<{ machineId: string; ownerId: string }>): Promise<void> {
+ const missing = wanted.filter(w => !this.answers.has(machineKey(environment, w.machineId, w.ownerId)))
+ if (!missing.length) return
+ const rows = db.machine.findMany({ where: { machineId: { in: [...new Set(missing.map(w => w.machineId))] }, autonomousEnv: environment, ...machineAlive } })
+ for (const w of missing) {
+ this.answers.set(machineKey(environment, w.machineId, w.ownerId), rows.then(found => found.find(m => m.machineId === w.machineId && m.userId === w.ownerId) ?? null))
+ }
+ await rows
+ }
}
/** Another Swarm too busy to say whether a shared session is confidential there: the list hides those shares and lists the rest,
@@ -4425,7 +4438,9 @@ export class SuperTabStore {
const byTab = new Map<string, T[]>()
for (const share of shares) if (share.superTabId) byTab.set(share.superTabId, [...byTab.get(share.superTabId) ?? [], share])
const observable = new Set<T>()
- for (const [tabId, tabShares] of byTab) for (const share of await this.observable(user, tabId, tabShares)) observable.add(share)
+ // One request's grant reads, shared by every Swarm it lists (task #3925).
+ const reads = new GrantReads()
+ for (const [tabId, tabShares] of byTab) for (const share of await this.observable(user, tabId, tabShares, reads)) observable.add(share)
return shares.filter(s => s.superTabId ? observable.has(s) : user.sub === s.ownerId || !anywhere.has(key(s)))
}
@@ -4434,7 +4449,7 @@ export class SuperTabStore {
reassemble(await readParts(this.db, await this.view([tabId]), tabId, { agentIds }), { share: true })
/** mayObserve for each of `shares`, all in Swarm `tabId`, from one load of it. */
- private async observable<T extends { machineId: string; agentId: string; ownerId: string }>(user: AuthUser, tabId: string, shares: T[]): Promise<T[]> {
+ private async observable<T extends { machineId: string; agentId: string; ownerId: string }>(user: AuthUser, tabId: string, shares: T[], reads = new GrantReads()): Promise<T[]> {
if (isTeamSession(user)) return []
let state: SuperTabState
try {
@@ -4446,11 +4461,13 @@ export class SuperTabStore {
const grants = new Map<string, Promise<boolean>>()
const granted = (target: Agent) => {
const key = JSON.stringify([target.contributorId, target.machineId, target.machineOwnerId, target.hostGrant])
- if (!grants.has(key)) grants.set(key, this.agentGrant(this.db, state, target).then(() => true,
+ if (!grants.has(key)) grants.set(key, this.agentGrant(this.db, state, target, reads).then(() => true,
error => { if (error instanceof SuperTabError) return false; throw error }))
return grants.get(key)!
}
const targets = shares.map(share => state.agents.find(a => a.machineId === share.machineId && a.harnessId === share.agentId && a.machineOwnerId === share.ownerId))
+ // The machines every grant below reads, in one read for the Swarm (task #3925).
+ await reads.machines(this.db, state.environment, targets.flatMap(t => t ? [{ machineId: t.machineId, ownerId: t.machineOwnerId }] : []))
// Confidential in another Swarm closes it here too; in this one, its current Lead stays open to readers (projects.ts watchClosed).
const elsewhere = await confidentialSessions(this.db, targets.flatMap(t => t ? [t.id] : []), tabId, this.cachedState, busyListing)
const allowed: T[] = []