Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
/**
* Integration test: outbound retry worker.
*
* The retry loop's whole point is SQL the unit tests can't fake:
* - SELECT … FOR UPDATE SKIP LOCKED so concurrent replicas claim
* disjoint rows
* - next_retry_at scheduling that advances on each miss and goes NULL
* after the last backoff step
* - transport_status flipping 'failed' → 'sent' when the provider
* accepts
*
* We exercise it by:
* 1. seeding a failed row in PG via persistEmailMessage
* 2. swapping sendViaProvider via dependency injection so we can control
* the next attempt's outcome without hitting Resend
* 3. invoking runRetryTick() directly and reading PG back
*/
import { test, before, beforeEach, after } from 'node:test'
import assert from 'node:assert/strict'
import { ensureSchemaOnce, resetAllTables, seedCompanyWithAgent, teardownAll } from './_helpers.js'
import { pool } from '../db/pool.js'
import { findOrCreateEmailConversation, persistEmailMessage } from '../email.js'
before(async () => {
await ensureSchemaOnce()
})
beforeEach(async () => {
await resetAllTables()
})
after(async () => {
await teardownAll()
})
/** Seed one outbound row in 'failed' state, with next_retry_at already
* in the past so the retry worker picks it up on the next tick. */
async function seedFailedOutbound(): Promise<{
messageId: string; conversationId: string; companyId: string; agentId: string;
}> {
const { companyId, agentId, agentEmail } = await seedCompanyWithAgent()
const conv = await findOrCreateEmailConversation({
companyId, inReplyTo: null, references: [], subject: 'pending',
memberIds: [agentId],
})
const { messageId } = await persistEmailMessage({
conversationId: conv.conversationId, companyId, authorId: agentId,
direction: 'out',
transportStatus: 'failed',
transportError: 'first attempt blew up',
smtpMessageId: `retry-test-${Date.now()}@host`,
inReplyTo: null, references: [], subject: 'pending',
fromAddr: agentEmail, toAddrs: ['[email protected]'],
body: 'retry me',
autoSubmitted: true,
})
// persistEmailMessage stamps next_retry_at = NOW()+60s on failed
// inserts. Move it into the past so the retry worker considers it due.
await pool.query(
`UPDATE email_messages SET next_retry_at = NOW() - INTERVAL '1 second' WHERE message_id = $1`,
[messageId],
)
return { messageId, conversationId: conv.conversationId, companyId, agentId }
}
async function seedSendingOutbound(): Promise<{ messageId: string }> {
const { companyId, agentId, agentEmail } = await seedCompanyWithAgent()
const conv = await findOrCreateEmailConversation({
companyId, inReplyTo: null, references: [], subject: 'outbox pending',
memberIds: [agentId],
})
const { messageId } = await persistEmailMessage({
conversationId: conv.conversationId, companyId, authorId: agentId,
direction: 'out', transportStatus: 'sending', transportError: null,
smtpMessageId: `outbox-${Date.now()}@host`, inReplyTo: null, references: [],
subject: 'outbox pending', fromAddr: agentEmail,
toAddrs: ['[email protected]'], body: 'recover me', autoSubmitted: true,
})
return { messageId }
}
test('[integration] retry worker promotes a failed row to sent when the provider succeeds', async (t) => {
const { messageId } = await seedFailedOutbound()
// Swap sendViaProvider via env-mock: empty RESEND_API_KEY → mock mode
// always succeeds (no failure injection set). The retry worker should
// see ok=true and promote.
delete process.env.RESEND_API_KEY
delete process.env.EMAIL_MOCK_FAIL_RATE
const { runRetryTick } = await import('../email-retry.js')
const r = await runRetryTick(8)
assert.equal(r.attempted, 1)
const { rows } = await pool.query<{
transport_status: string; transport_error: string | null;
retry_attempts: number; next_retry_at: string | null;
}>(
`SELECT transport_status, transport_error, retry_attempts, next_retry_at
FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.equal(rows[0].transport_status, 'sent')
assert.equal(rows[0].transport_error, null)
assert.equal(rows[0].retry_attempts, 1)
assert.equal(rows[0].next_retry_at, null, 'sent rows must clear next_retry_at to stay out of the retry set')
})
test('[integration] retry worker recovers a crash-stale sending outbox row', async () => {
const { messageId } = await seedSendingOutbound()
const scheduled = await pool.query<{ next_retry_at: string | null }>(
`SELECT next_retry_at FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.ok(scheduled.rows[0].next_retry_at, 'sending row must have a recovery deadline')
assert.ok(new Date(scheduled.rows[0].next_retry_at!).getTime() > Date.now())
await pool.query(
`UPDATE email_messages SET next_retry_at = NOW() - INTERVAL '1 second' WHERE message_id = $1`,
[messageId],
)
delete process.env.RESEND_API_KEY
delete process.env.EMAIL_MOCK_FAIL_RATE
const { runRetryTick } = await import('../email-retry.js')
const result = await runRetryTick(8)
assert.equal(result.attempted, 1)
const recovered = await pool.query<{
transport_status: string; retry_attempts: number; next_retry_at: string | null;
}>(
`SELECT transport_status, retry_attempts, next_retry_at
FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.deepEqual(recovered.rows[0], {
transport_status: 'sent', retry_attempts: 1, next_retry_at: null,
})
})
test('[integration] retry worker advances backoff on a still-failing send', async () => {
const { messageId } = await seedFailedOutbound()
// Force mock mode to always fail.
delete process.env.RESEND_API_KEY
process.env.EMAIL_MOCK_FAIL_RATE = '1'
const { runRetryTick } = await import('../email-retry.js')
try {
await runRetryTick(8)
} finally {
delete process.env.EMAIL_MOCK_FAIL_RATE
}
const { rows } = await pool.query<{
transport_status: string; retry_attempts: number; next_retry_at: string | null;
}>(
`SELECT transport_status, retry_attempts, next_retry_at
FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.equal(rows[0].transport_status, 'failed', 'row stays failed after a failing retry')
assert.equal(rows[0].retry_attempts, 1)
assert.ok(rows[0].next_retry_at, 'next_retry_at should be scheduled for another attempt')
// The next scheduled time must be in the FUTURE (first backoff step is
// 60s, plus we're not at the terminal step yet).
const next = new Date(rows[0].next_retry_at!).getTime()
assert.ok(next > Date.now(), `next_retry_at should be future, got ${rows[0].next_retry_at}`)
})
test('[integration] retry worker terminates the row after exhausting backoff steps', async () => {
const { messageId } = await seedFailedOutbound()
// Pretend we've already burned 5 attempts (one short of the 6-step
// backoff curve). Next failed tick should mark terminal (next_retry_at = NULL).
await pool.query(
`UPDATE email_messages SET retry_attempts = 5 WHERE message_id = $1`,
[messageId],
)
delete process.env.RESEND_API_KEY
process.env.EMAIL_MOCK_FAIL_RATE = '1'
const { runRetryTick } = await import('../email-retry.js')
try {
await runRetryTick(8)
} finally {
delete process.env.EMAIL_MOCK_FAIL_RATE
}
const { rows } = await pool.query<{
transport_status: string; retry_attempts: number; next_retry_at: string | null;
}>(
`SELECT transport_status, retry_attempts, next_retry_at
FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.equal(rows[0].transport_status, 'failed')
assert.equal(rows[0].retry_attempts, 6)
assert.equal(rows[0].next_retry_at, null, 'after the last backoff step the row goes terminal — never picked up again')
})
test('[integration] retry worker ignores rows with next_retry_at in the future', async () => {
const { messageId } = await seedFailedOutbound()
// Push the scheduled retry into the future so the worker should NOT
// claim it this tick.
await pool.query(
`UPDATE email_messages SET next_retry_at = NOW() + INTERVAL '10 minutes' WHERE message_id = $1`,
[messageId],
)
const { runRetryTick } = await import('../email-retry.js')
const r = await runRetryTick(8)
assert.equal(r.attempted, 0)
const { rows } = await pool.query<{ transport_status: string; retry_attempts: number }>(
`SELECT transport_status, retry_attempts FROM email_messages WHERE message_id = $1`,
[messageId],
)
assert.equal(rows[0].retry_attempts, 0, 'untouched row stays at attempt 0')
assert.equal(rows[0].transport_status, 'failed')
})
test('[integration] retry worker ignores inbound rows (only out + failed are eligible)', async () => {
const { companyId, agentId, agentEmail } = await seedCompanyWithAgent()
const conv = await findOrCreateEmailConversation({
companyId, inReplyTo: null, references: [], subject: 'inbound',
memberIds: [agentId],
})
// Even setting transport_status='failed' on an INBOUND row shouldn't
// make it retry-eligible — only direction='out' rows are.
await persistEmailMessage({
conversationId: conv.conversationId, companyId, authorId: agentId,
direction: 'in', transportStatus: 'failed', transportError: 'shouldn\'t retry',
smtpMessageId: `inbound-failed-${Date.now()}@host`,
inReplyTo: null, references: [], subject: 'inbound',
fromAddr: '[email protected]', toAddrs: [agentEmail],
body: 'do not retry',
})
// Force next_retry_at into the past on every row — if the filter were
// wrong, the inbound row would get picked up.
await pool.query(`UPDATE email_messages SET next_retry_at = NOW() - INTERVAL '1 second'`)
const { runRetryTick } = await import('../email-retry.js')
const r = await runRetryTick(8)
assert.equal(r.attempted, 0)
})