Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm: an agent's messages still reach the Lead after its task moved (prod alarm lost 4h50m) (#2243)

0xken150941150bc58b
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.ts
index fc4e129a4..a4fd8c717 100644
--- a/backend/integration/superTabLiveness.test.ts
+++ b/backend/integration/superTabLiveness.test.ts
@@ -539,6 +539,85 @@ describe('work that loses its owner', () => {
})
})
+// 10-29: after its task moved, a prod-watch agent's "the backend worker is down" alarm and its answers to the Lead's questions
+// were stored with no delivery at all, and the worker stayed down 4 h 50 min. Its status stays refused; its messages arrive.
+describe('messages from an agent whose task moved', () => {
+ /** Every agent the message has a delivery for, from the stored rows. */
+ const deliveredTo = async (messageId: string) => (await db.superTabDelivery.findMany({ where: { messageId } })).map(d => d.agentId).sort()
+ const post = async (f: Awaited<ReturnType<typeof swarm>>, agentId: string, text: string, extra: { replyTo?: string; recipientIds?: string[] } = {}) =>
+ String((await f.execute({ type: 'message.send', conversationId: f.group, text, replyTo: extra.replyTo ?? null, recipientIds: extra.recipientIds ?? [] },
+ host, agentId)).result.messageId)
+
+ it('reach the Lead once the task is lost, its alarm and its reply to the Lead\'s question alike, while its status stays refused', async () => {
+ const f = await swarm()
+ await f.observe('test-host', [], { stoppedHarnessIds: [f.harness.worker] })
+ expect((await f.assignment()).status).toBe('lost')
+ const alarm = await post(f, f.worker, 'Severe, prod: the backend worker has been down for 44 minutes.')
+ const question = await post(f, f.lead, 'Is it still down?', { recipientIds: [f.worker] })
+ const answer = await post(f, f.worker, 'Still down at 22:50.', { replyTo: question })
+ expect(await deliveredTo(alarm)).toEqual([f.lead])
+ expect(await deliveredTo(answer)).toEqual([f.lead])
+ expect((await f.leadInbox()).map(m => m.id)).toEqual(expect.arrayContaining([alarm, answer]))
+ await expect(f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: 'Still watching' }, host, f.worker))
+ .rejects.toMatchObject({ code: 'ASSIGNMENT_LOST' })
+ })
+
+ it('reach the Lead and the agents they name after the task moved to another agent, and nobody outside the group', async () => {
+ const f = await swarm()
+ await f.observe('test-host', [], { stoppedHarnessIds: [f.harness.worker] })
+ const other = await f.add(f.group, 'test-host', f.harness.other, 'Other')
+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: other })).result.assignmentId)
+ const elsewhere = String((await f.execute({ type: 'conversation.create', title: 'Ship the lexer', brief: 'Implement it',
+ checks: [{ description: 'Verified', independent: false }] })).result.conversationId)
+ const outsider = await f.add(elsewhere, 'test-host', randomUUID(), 'Outsider')
+ expect(await deliveredTo(await post(f, f.worker, 'The worker is down.'))).toEqual([f.lead])
+ expect(await deliveredTo(await post(f, f.worker, 'Over to you.', { recipientIds: [other] }))).toEqual([f.lead, other].sort())
+ const stored = await db.superTabMessage.count({ where: { tabId: f.tab } })
+ await expect(post(f, f.worker, 'Psst.', { recipientIds: [outsider] })).rejects.toMatchObject({ code: 'INVALID_RECIPIENT' })
+ expect(await db.superTabMessage.count({ where: { tabId: f.tab } })).toBe(stored)
+ expect(await db.superTabDelivery.count({ where: { tabId: f.tab, agentId: outsider } })).toBe(0)
+ // The new worker and the Lead talk as before; the Lead's plain post reaches the new worker, not the old one.
+ expect(await deliveredTo(await post(f, other, 'Taking over.'))).toEqual([f.lead])
+ expect(await deliveredTo(await post(f, f.lead, 'Go ahead.'))).toEqual([other])
+ await expect(f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: null }, host, f.worker))
+ .rejects.toMatchObject({ code: 'ASSIGNMENT_LOST' })
+ await f.execute({ type: 'work.status', conversationId: f.group, assignmentId: next, status: 'working', result: 'Taking over' }, host, other)
+ })
+
+ it('reach the Lead after the Lead cancelled the lost task', async () => {
+ const f = await swarm()
+ await f.observe('test-host', [], { stoppedHarnessIds: [f.harness.worker] })
+ await f.execute({ type: 'work.cancel', conversationId: f.group, assignmentId: f.assignmentId, reason: 'Not needed any more' }, host, f.lead)
+ expect(await deliveredTo(await post(f, f.worker, 'Stopping; the last check was clean.'))).toEqual([f.lead])
+ })
+
+ // Review of #2243: only the current assignment and the latest lost attempt were recognized.
+ it('reach the Lead after the task was cancelled and assigned to another agent', async () => {
+ const f = await swarm()
+ await f.execute({ type: 'work.cancel', conversationId: f.group, assignmentId: f.assignmentId, reason: 'Moving it' }, host, f.lead)
+ const other = await f.add(f.group, 'test-host', f.harness.other, 'Other')
+ await f.execute({ type: 'work.assign', conversationId: f.group, agentId: other })
+ expect(await deliveredTo(await post(f, f.worker, 'Severe: the worker is down.'))).toEqual([f.lead])
+ await expect(f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: null }, host, f.worker))
+ .rejects.toMatchObject({ code: 'STALE_ASSIGNMENT' })
+ })
+
+ it('reach the Lead from the first agent after the next agent lost the task too', async () => {
+ const f = await swarm()
+ await f.observe('test-host', [], { stoppedHarnessIds: [f.harness.worker] })
+ const other = await f.add(f.group, 'test-host', f.harness.other, 'Other')
+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: other })).result.assignmentId)
+ await f.observe('test-host', [], { stoppedHarnessIds: [f.harness.worker, f.harness.other] })
+ expect(await f.assignment()).toMatchObject({ id: next, status: 'lost' })
+ expect(await deliveredTo(await post(f, f.worker, 'Severe: the worker is down.'))).toEqual([f.lead])
+ expect(await deliveredTo(await post(f, other, 'Lost my session.'))).toEqual([f.lead])
+ await expect(f.execute({ type: 'work.status', conversationId: f.group, assignmentId: f.assignmentId, status: 'working', result: null }, host, f.worker))
+ .rejects.toMatchObject({ code: 'STALE_ASSIGNMENT' })
+ await expect(f.execute({ type: 'work.status', conversationId: f.group, assignmentId: next, status: 'working', result: null }, host, other))
+ .rejects.toMatchObject({ code: 'ASSIGNMENT_LOST' })
+ })
+})
+
/** A client that records the key of every SwarmGroup row a read returns. */
function recordingGroups() {
const keys: string[] = []
diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.ts
index bc4b7dd00..7c268716b 100644
--- a/backend/src/superTabs/commands.test.ts
+++ b/backend/src/superTabs/commands.test.ts
@@ -245,7 +245,8 @@ describe('Super Tab collaboration policy', () => {
const worker = f.add('worker')
const posted = f.run({ type: 'message.send', conversationId: 'lead', text: 'Which behavior?', replyTo: null, recipientIds: [] },
{ kind: 'agent', agentId: worker, machineOwnerId: 'host-owner' }).messages[0]!
- expect(posted.recipients).toEqual([])
+ // Any agent in a group reaches its Lead (review of #2243).
+ expect(posted.recipients).toEqual([lead])
const reply = f.run({ type: 'message.send', conversationId: 'lead', text: 'Keep the local copy.', replyTo: posted.id, recipientIds: [] }, owner,
{ reply: { id: posted.id, conversationId: 'lead', author: posted.author, createdAt: 0 } })
expect(reply.messages[0]!.recipients).toEqual([worker])
@@ -276,23 +277,48 @@ describe('Super Tab collaboration policy', () => {
expect(inline.alsoInChat).toBeUndefined()
})
- it('delivers an assigned worker\'s own post to its Lead, and nobody else\'s', () => {
+ it('delivers an assigned worker\'s own post to its Lead, or to the agents it names instead', () => {
const f = fixture()
const lead = f.add('lead', 'owner', 'lead', true)
const group = f.group(), worker = f.add('worker', 'owner', group), other = f.add('other', 'owner', group)
const as = (agentId: string): Actor => ({ kind: 'agent', agentId, machineOwnerId: 'host-owner' })
const post = (actor: Actor, recipientIds: string[] = []) => f.run({ type: 'message.send', conversationId: group, text: 'PR is green.',
replyTo: null, recipientIds }, actor).messages[0]!.recipients
- expect(post(as(worker))).toEqual([])
f.run({ type: 'work.assign', conversationId: group, agentId: worker })
expect(post(as(worker))).toEqual([lead])
expect(post(as(worker), [other])).toEqual([other])
- expect(post(as(other))).toEqual([])
expect(post(as(lead))).toEqual([worker])
f.run({ type: 'work.status', conversationId: group, assignmentId: f.state.conversations[1]!.assignment!.id, status: 'blocked', result: 'Stuck.' }, as(worker))
expect(post(as(worker))).toEqual([lead])
})
+ // 10-29: a moved agent's alarm and answers reached no one, and the backend worker stayed down 4 h 50 min.
+ it('delivers any other agent\'s post in the group to its Lead, beside the agents it names, whether or not it held the task', () => {
+ const f = fixture()
+ const lead = f.add('lead', 'owner', 'lead', true)
+ const group = f.group(), worker = f.add('worker', 'owner', group), other = f.add('other', 'owner', group)
+ const as = (agentId: string): Actor => ({ kind: 'agent', agentId, machineOwnerId: 'host-owner' })
+ const post = (actor: Actor, recipientIds: string[] = [], agentOnly?: boolean) => f.run({ type: 'message.send', conversationId: group,
+ text: 'Severe: the worker is down.', replyTo: null, recipientIds, ...(agentOnly ? { agentOnly } : {}) }, actor).messages[0]!.recipients
+ const assignment = () => f.state.conversations.find(c => c.id === group)!.assignment!
+ expect(post(as(worker))).toEqual([lead])
+ expect(post(as(lead))).toEqual([])
+ f.run({ type: 'work.assign', conversationId: group, agentId: worker })
+ expect(post(as(other))).toEqual([lead])
+ expect(post(as(other), [worker])).toEqual([worker, lead])
+ f.run({ type: 'work.cancel', conversationId: group, assignmentId: assignment().id, reason: 'Duplicate of another task' })
+ expect(post(as(worker))).toEqual([lead])
+ expect(post(as(worker), [other])).toEqual([other, lead])
+ expect(post(as(worker), [lead])).toEqual([lead])
+ // A note to itself stays its own.
+ expect(post(as(worker), [worker], true)).toEqual([worker])
+ expect(post(as(lead))).toEqual([])
+ f.run({ type: 'work.assign', conversationId: group, agentId: other })
+ f.run({ type: 'work.status', conversationId: group, assignmentId: assignment().id, status: 'stopped', result: null }, owner, { confirmedStopped: true })
+ expect(post(as(other))).toEqual([lead])
+ expect(() => post(as(other), ['not-an-agent'])).toThrow()
+ })
+
describe('a Lead\'s message to the assigned worker (#1549)', () => {
const setup = () => {
const f = fixture()
@@ -337,12 +363,12 @@ describe('Super Tab collaboration policy', () => {
expect(post(lead, [], report.id).recipients).toEqual([worker])
})
- it('lets explicit recipients win and leaves other members, replies to others and person routing alone', () => {
+ it('lets explicit recipients win and leaves replies to others and person routing alone', () => {
const { f, lead, group, worker, other, as, post, history } = setup()
f.run({ type: 'work.assign', conversationId: group, agentId: worker })
expect(post(lead, [other]).recipients).toEqual([other])
const aside = post(other)
- expect(aside.recipients).toEqual([])
+ expect(aside.recipients).toEqual([lead])
expect(post(lead, [], aside.id).recipients).toEqual([])
const personMessage = f.run({ type: 'message.send', conversationId: group, text: 'Status?', replyTo: null, recipientIds: [] }).messages[0]!
history.push({ author: { kind: 'person', id: 'owner' }, recipients: personMessage.recipients, id: personMessage.id })
diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.ts
index 0d7912700..748ab6a1e 100644
--- a/backend/src/superTabs/commands.ts
+++ b/backend/src/superTabs/commands.ts
@@ -1944,12 +1944,19 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Comma
const workerToLead = actor.kind === 'agent' && activeWork(group) && group.assignment?.agentId === actor.agentId
&& group.leadId && group.leadId !== actor.agentId && group.agentIds.includes(group.leadId) ? group.leadId : null
const agentRecipient = leadToWorker ?? workerToLead
+ // Any other agent in the group reaches its Lead, beside anyone it names: an agent whose task here was lost, moved or
+ // cancelled has its status and results refused, so its posts are all the Lead hears from it. They reached no one until
+ // 10-29, when a moved prod-watch agent's "the backend worker is down" alarm and its answers to the Lead's questions had no
+ // delivery, and the worker stayed down 4 h 50 min. Membership, not a record of past attempts, decides (Lead's ruling on
+ // #2243): the Lead already reads the group. The assigned worker keeps its own rule above; a note to itself stays its own.
+ const memberToLead = actor.kind === 'agent' && !command.agentOnly && assigned !== actor.agentId && group.leadId
+ && group.leadId !== actor.agentId && group.agentIds.includes(group.leadId) ? [group.leadId] : []
// A Buddy chat has no Lead: its owner talks to its Buddy.
const automatic = actor.kind === 'person' ? (replyAgent && group.agentIds.includes(replyAgent) ? replyAgent
: isBuddyChat(group) ? group.agentIds[0] ?? null : group.leadId) : agentRecipient
// In the Leads conversation everything posted reaches every other Lead; naming some only adds emphasis.
const everyLead = isLeads(group) ? group.agentIds.filter(id => actor.kind !== 'agent' || id !== actor.agentId) : []
- const recipients = [...new Set([...command.recipientIds, ...everyLead,
+ const recipients = [...new Set([...command.recipientIds, ...everyLead, ...memberToLead,
...(command.recipientIds.length || everyLead.length || !automatic ? [] : [automatic])])]
for (const id of recipients) {
agent(state, id)