Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript
59.8 KB
Raw Blame
/**
 * Wake scheduler — listens to CH_MESSAGE_NEW (Redis pubsub) and
 * routes wake events to the right agent's pod.
 *
 * Model (pod-only):
 *   1. Try deliverWake() — push SSE event to the agent's on-bus Pod
 *   2. If 0 subscribers, the Pod is `resting`. ensurePod() spins one
 *      up; the Pod, after SSE attach, runs drain() unconditionally
 *      to catch up the inbox (so we don't need to queue the wake
 *      server-side — the missed event self-heals through the
 *      inbox-read).
 *
 * Per-agent serialization is intrinsic — each agent has exactly one
 * Pod, which runs one turn at a time and collapses concurrent wakes
 * into pendingRerun internally (see runtime/pod-agent.ts).
 *
 * Idle teardown is handled by the Pod itself: after
 * CUMORA_AGENT_IDLE_MS without a wake, it sets the agent's status to
 * `resting` and exits, leaving the PVC bound for next time. The
 * server doesn't track Pod lifetimes — the next wake re-creates the
 * Pod via the orchestrator.
 */
import { pool } from '../db/pool.js'
import { env } from '../env.js'
import { CH_MESSAGE_NEW, CH_POLLS, CH_TYPING, publish, redis, sub, type MessageNewEvent, type PollUpdatedEvent } from '../redis.js'
import { notifyAlert } from '../alerting.js'
import { ensurePod } from './runtime/orchestrator.js'
import {
  resolveAgentHost,
  isByoaKind,
  type ResolvedAgentHost,
} from './computer/registry.js'
import { deliver as deliverWake, deliverSteer, type PollWakeBrief } from './runtime/wake-bus.js'
import { inprocClient, isAgentBusy } from './runtime/inproc-client.js'
import { classifyInboxTriage, type InboxTriageVerdict } from './inbox-triage.js'
import type { AgentTurnOptions } from './turn.js'
import { recipientsForRoute, routeMessage, routeUnaddressedMessage } from './routing.js'
import { electLineup, type ElectionCandidate } from './routing-election.js'
import { claimPrimary, loadElectionCandidates, startRoutingClaimSweeperIfEnabled } from './routing-claims.js'
import { BUSY_STATUS_LEASE_MS } from '../status.js'
import { Semaphore } from '../concurrency.js'

/** Bounds how many recipients the wake fan-out triages + wakes at once
 *  (per replica). See env.WAKE_FANOUT_CONCURRENCY — this is the
 *  backpressure that stops a swarm reply-storm from oversubscribing the
 *  pg pool and tripping the triage fail-open amplification loop. */
const wakeFanoutSem = new Semaphore(env.WAKE_FANOUT_CONCURRENCY)

/** Test handle on the semaphore above. Exported so a test can prove the retry
 *  drain queues behind the SAME permits as live fan-out, not just behind some
 *  bound of its own. */
export const _wakeFanoutSem = wakeFanoutSem

/** Optional payload attached to a wake that the busy-agent path should
 *  also publish as a `steer` event so the running turn can react
 *  mid-flight (instead of waiting until the next wake). Built by the
 *  message-new handler from MessageNewEvent. */
export interface SteerWakePayload {
  messageId: string
  conversationId: string
  authorName: string
  body: string
  /** Tenant tag so the steer-ack typing broadcast routes to clients instead of
   *  being dropped as "untagged". Sourced from the message-new event. Optional
   *  only so test fixtures can omit it; the production wake() path always sets it. */
  companyId?: string
}

type WakeReason = 'message.new' | 'idle' | 'manual' | 'background_scan' | 'poll.updated'
type WakeOptions = Pick<AgentTurnOptions, 'idleReason' | 'backgroundBrief' | 'pollBrief' | 'triageNote'> & {
  /** Exact durable notice to acknowledge before managed-runtime triage. */
  triageTarget?: { conversationId: string; messageId: string }
  /** Message fan-out must resolve placement before delivering to a live runtime. */
  placementTriage?: boolean
}
type WakeFailureClass = 'ensure_pod' | 'host_resolution'

interface WakeRetryJob {
  id: string
  agentId: string
  reason: WakeReason
  conversationId: string | null
  steerPayload: SteerWakePayload | null
  options: WakeOptions
  attempt: number
  lastFailure: string
}

const WAKE_RETRY_DUE_KEY = 'cumora:wake-retry:due'
const WAKE_RETRY_JOB_KEY = 'cumora:wake-retry:jobs'
const WAKE_RETRY_MAX_ATTEMPTS = 60
const WAKE_RETRY_BATCH_SIZE = 25

/** 5s, 10s, 20s, 40s, then 60s for every attempt after that. */
function wakeRetryBaseDelayMs(attempt: number): number {
  const n = Math.max(0, Math.min(4, attempt))
  return Math.min(60_000, 5_000 * 2 ** n)
}

/**
 * Backoff with decorrelating jitter (0.5x–1.5x of the base delay).
 *
 * The base curve flattens at 60s from attempt 4 on, so without jitter every job
 * that failed in the same blip is handed the SAME delay forever: they come due
 * in the same second, every minute, for all `WAKE_RETRY_MAX_ATTEMPTS`
 * attempts — an hour-long synchronized herd hitting a database that is
 * probably still recovering from whatever knocked them out. Jitter is what
 * turns that into a spread-out trickle.
 *
 * `random` is injectable so the spread is testable rather than asserted by
 * eyeball.
 */
export function _wakeRetryDelayMs(attempt: number, random: () => number = Math.random): number {
  return Math.floor(wakeRetryBaseDelayMs(attempt) * (0.5 + random()))
}

export function _shouldRetryEnsurePodFailure(reason: WakeReason, ensureReason: string): boolean {
  // message.new wakes are DURABLE because the message itself is already
  // committed to the messages table. Whenever the agent's pod next
  // attaches its wake-stream it runs drain() unconditionally, which
  // calls loadInbox — picking up everything since the agent's last
  // read cursor including any wake we "missed". Queuing a separate
  // retry on top means the pod can receive the same wake twice (once
  // via the original SSE delivery once it's healthy, once via the
  // retry), trigger an extra turn through pendingRerun, and post a
  // duplicate reply — exactly the "Nova says 3 then 1" bug.
  //
  // Synthetic wakes (idle / background_scan) don't have a backing DB
  // row, so they ARE handled here too — but via the inline poll loop
  // at the end of wakeOne, not via this scheduled-retry queue. The
  // queue is left to `manual` only (CLI/admin pokes that explicitly
  // want delivery guarantees).
  if (reason !== 'manual') return false
  if (/no such agent/i.test(ensureReason)) return false
  return true
}

/** Host lookup failed before any runtime was selected or started, so replay is
 * safe even for a durable message wake. This is deliberately separate from
 * ordinary ensurePod retries, whose message replay can duplicate a turn. */
export function _shouldRetryWakeFailure(
  reason: WakeReason,
  failureReason: string,
  failureClass: WakeFailureClass,
): boolean {
  if (failureClass === 'host_resolution') return true
  return _shouldRetryEnsurePodFailure(reason, failureReason)
}

function wakeRetryId(agentId: string, reason: WakeReason, conversationId: string | null): string {
  return `${agentId}:${reason}:${conversationId ?? '-'}`
}

async function scheduleWakeRetry(
  agentId: string,
  reason: WakeReason,
  conversationId: string | null,
  steerPayload: SteerWakePayload | null,
  options: WakeOptions,
  attempt: number,
  failureReason: string,
  failureClass: WakeFailureClass = 'ensure_pod',
): Promise<void> {
  if (!_shouldRetryWakeFailure(reason, failureReason, failureClass)) return
  const id = wakeRetryId(agentId, reason, conversationId)
  if (attempt > WAKE_RETRY_MAX_ATTEMPTS) {
    await redis.hdel(WAKE_RETRY_JOB_KEY, id).catch(() => { /* ignore */ })
    void notifyAlert({
      label: 'scheduler.wake_retry_exhausted',
      error: new Error(`wake retry exhausted for ${agentId}: ${failureReason}`),
      extras: { agentId, reason, conversationId, attempt, failureReason },
    })
    return
  }
  const dueAt = Date.now() + _wakeRetryDelayMs(attempt)
  const job: WakeRetryJob = {
    id, agentId, reason, conversationId, steerPayload, options,
    attempt,
    lastFailure: failureReason,
  }
  await redis.hset(WAKE_RETRY_JOB_KEY, id, JSON.stringify(job))
  await redis.zadd(WAKE_RETRY_DUE_KEY, dueAt, id)
  console.warn(`[scheduler] ${agentId} ${reason} wake retry scheduled in ${Math.round((dueAt - Date.now()) / 1000)}s after ${failureClass}: ${failureReason}`)
}

/** Drop a queued retry whose job has already been done.
 *
 *  The post-spawn health check and the inline replay loop at the end of
 *  `wakeOne` are two mechanisms for one thing: make sure a freshly-spawned pod
 *  actually receives this wake. Only the loop can CONFIRM it — it re-delivers
 *  until a subscriber answers. Leaving the queued job behind after that
 *  confirmation delivers the same wake a second time.
 *
 *  For a durable `message.new` that was harmless, which is why it went
 *  unnoticed: the pod's cold-start drain reads the inbox and the fingerprint
 *  absorbs the repeat. But `message.new` never enters this queue —
 *  `_shouldRetryEnsurePodFailure` admits `manual` and nothing else, the only
 *  producer of `manual` in the product is a board card wake, and that always
 *  carries a `backgroundBrief` whose fingerprint embeds `new Date()`
 *  (turn.ts). So the one wake class that reaches the queue is exactly the one
 *  the "fingerprint makes the turn no-op" premise does not hold for: every
 *  board action on a resting agent ran the whole turn twice, re-claiming the
 *  card, re-commenting, and billing a second time.
 *
 *  The id is deterministic and `scheduleWakeRetry` hsets it, so there is at
 *  most one job per (agent, reason, conversation) and this removes exactly the
 *  one whose work just completed. */
export async function _cancelWakeRetry(
  agentId: string,
  reason: WakeReason,
  conversationId: string | null,
): Promise<void> {
  const id = wakeRetryId(agentId, reason, conversationId)
  await redis.zrem(WAKE_RETRY_DUE_KEY, id).catch(() => { /* best effort */ })
  await redis.hdel(WAKE_RETRY_JOB_KEY, id).catch(() => { /* best effort */ })
}

/** The Redis side of the retry queue, behind an interface so the drain loop can
 *  be driven by a test without a live Redis or a live agent fleet. */
export interface WakeRetryQueue {
  /** Retry ids that are due, oldest first, capped at the batch size. */
  due(now: number): Promise<string[]>
  /** Atomically take ownership of one id. `null` means another replica won it,
   *  or the job body is gone or unreadable. */
  claim(id: string): Promise<WakeRetryJob | null>
  run(job: WakeRetryJob): Promise<void>
  reschedule(job: WakeRetryJob, failure: string): Promise<void>
}

const redisWakeRetryQueue: WakeRetryQueue = {
  due: (now) => redis.zrangebyscore(WAKE_RETRY_DUE_KEY, 0, now, 'LIMIT', 0, WAKE_RETRY_BATCH_SIZE),
  claim: async (id) => {
    // zrem is the claim: exactly one replica gets 1 back for a given id.
    if (await redis.zrem(WAKE_RETRY_DUE_KEY, id) !== 1) return null
    const raw = await redis.hget(WAKE_RETRY_JOB_KEY, id)
    if (!raw) return null
    await redis.hdel(WAKE_RETRY_JOB_KEY, id).catch(() => { /* ignore */ })
    try {
      return JSON.parse(raw) as WakeRetryJob
    } catch {
      return null
    }
  },
  // wakeOne's boolean says whether it woke anything; the retry loop has never
  // branched on it (its own failure paths already re-enqueue through
  // scheduleWakeRetry), so drop it rather than invent a new policy here.
  run: async (job) => {
    await wakeOne(job.agentId, job.reason, job.conversationId, job.steerPayload, job.options, job.attempt)
  },
  reschedule: (job, failure) => scheduleWakeRetry(
    job.agentId, job.reason, job.conversationId, job.steerPayload, job.options, job.attempt + 1, failure,
  ),
}

/**
 * Drain one batch of due wake retries — through `wakeFanoutSem`, the same
 * semaphore the live fan-out uses.
 *
 * This loop used to call `wakeOne(...)` without awaiting it, so a tick fired up
 * to `WAKE_RETRY_BATCH_SIZE` (25) wakes fully in parallel, every 5 seconds, and
 * none of them passed through the semaphore. Each wake costs a host-resolution
 * query, and for a managed agent a triage pass on top: persona + inbox +
 * context loads and a model call. Twenty-five of those at once is ~100
 * concurrent queries against a 20-slot pool.
 *
 * That matters most in exactly the situation this queue exists for. Any DB
 * error in `resolveAgentHost` maps to `lookup_failed`, which
 * `_shouldRetryWakeFailure` treats as retryable for every wake reason, so a
 * 30-second database blip enqueues a retry for every recipient of every message
 * sent during it. The recovery burst then oversubscribes the pool that is still
 * recovering, more wakes time out, and each timeout re-enqueues itself — the
 * triage fail-open amplification loop the semaphore was introduced for after
 * the 2026-05-27 connection-exhaustion outage. Recovery traffic has to obey the
 * same backpressure as live traffic; it is the traffic most likely to arrive
 * all at once.
 */
export async function _drainWakeRetries(queue: WakeRetryQueue = redisWakeRetryQueue): Promise<void> {
  const ids = await queue.due(Date.now())
  await Promise.all(ids.map((id) => wakeFanoutSem.run(async () => {
    const job = await queue.claim(id)
    if (!job) return
    try {
      await queue.run(job)
    } catch (err) {
      const failure = err instanceof Error ? err.message : String(err)
      console.error(`[scheduler] wake retry ${job.id} failed:`, failure)
      await queue.reschedule(job, failure).catch(() => { /* ignore */ })
    }
  })))
}

function startWakeRetryWorker(intervalMs: number = 5_000): NodeJS.Timeout {
  // One batch at a time. The drain now awaits its wakes, so a slow recovery
  // would otherwise stack a fresh 25-id batch on the semaphore queue every 5s
  // — bounded work, but an ever-growing backlog of claims that mostly lose
  // their zrem race and do nothing.
  let inFlight: Promise<void> | null = null
  const tick = (): void => {
    if (inFlight) return
    inFlight = _drainWakeRetries()
      .catch((err) =>
        console.error('[scheduler] wake retry worker failed:', err instanceof Error ? err.message : err),
      )
      .finally(() => { inFlight = null })
  }
  setImmediate(tick)
  const t = setInterval(tick, intervalMs)
  t.unref?.()
  return t
}

/** Wake one agent. Exposed so non-message-new triggers (kanban card
 *  mentions, calendar dispatches, …) can use the same SSE-or-spawn path. */
export async function wakeAgent(
  agentId: string,
  reason: WakeReason,
  conversationId: string | null = null,
  steerPayload: SteerWakePayload | null = null,
  options: WakeOptions = {},
): Promise<boolean> {
  return wakeOne(agentId, reason, conversationId, steerPayload, options)
}

// ─── low-priority wake backpressure (FUSE-cap incident 2026-05-20) ──
//
// `idle` and `background_scan` wake reasons are SYNTHETIC — they were
// generated by the server's own schedulers, not by a real user
// message. During a cumora-server crashloop, idle ticks accumulate in
// the schedulers' clock backlog, and on recovery they fire all at
// once: the idle scheduler can issue a wake per agent per tenant in
// rapid succession, the scanner the same. Each wake → ensurePod →
// new agent pod → reserves one /dev/fuse cluster slot.
//
// The fix is a hard cap on synthetic wakes per cumora-server process
// per minute. User-facing reasons (`message.new`, `manual`) are NEVER
// rate-limited — they correspond to a real user action and must go
// through promptly. The counter is per process. Idle ticks really do
// run on every replica (`idle.ts` takes no lock), so replicas multiply
// the idle budget. The scanner does not: `scanner.ts` holds
// `pg_try_advisory_lock` for the tick, so this cap is not a cluster-wide
// scanner budget.
const LOW_PRIORITY_WAKE_BUDGET_PER_MIN = 20
let lowPriWindowStart = Date.now()
let lowPriUsed = 0
let lowPriDroppedInWindow = 0

// Agent turn rate limit — a CONTENT-BLIND cost floor, not a loop decision.
// Whether a thread is "a loop" and whether to reply is the small model's call
// (it sees the per-conversation "thread heat" signal and goes quiet on an
// agent-only thread). This limit only bounds how often any one agent can be
// activated by AGENT-driven traffic, so a runaway (or a model that won't stop)
// can't burn unbounded cost. It never looks at message content, and
// human-driven wakes are never throttled. Generous enough that normal use never
// trips it; low enough to cap a ping-pong.
const AGENT_TURN_RATE_PER_MINUTE = 30

/** Consume one agent-turn token (rolling 60s window). Returns false when the
 *  agent is over its content-blind activation budget. Fail-open on Redis errors.
 *  Shared by the cloud fan-out and the BYOA triage endpoint so one agent has one
 *  budget across both paths. */
export async function consumeAgentTurnToken(agentId: string): Promise<boolean> {
  try {
    const key = `cumora:turn-rate:${agentId}`
    const count = await redis.incr(key)
    if (count === 1) await redis.expire(key, 60).catch(() => { /* best-effort */ })
    return count <= AGENT_TURN_RATE_PER_MINUTE
  } catch {
    return true // fail-open
  }
}

/** Returns true if a low-priority wake is allowed under the current
 *  rolling 60s budget. Mutates internal counters. Exported for tests. */
export function _consumeLowPriorityWakeBudget(now: number = Date.now()): boolean {
  if (now - lowPriWindowStart >= 60_000) {
    if (lowPriDroppedInWindow > 0) {
      console.warn(`[scheduler] low-priority wake budget window closing: used=${lowPriUsed} dropped=${lowPriDroppedInWindow}`)
    }
    lowPriWindowStart = now
    lowPriUsed = 0
    lowPriDroppedInWindow = 0
  }
  if (lowPriUsed >= LOW_PRIORITY_WAKE_BUDGET_PER_MIN) {
    lowPriDroppedInWindow++
    return false
  }
  lowPriUsed++
  return true
}

/** Test-only reset. Sets the window start to 0 (epoch) so the next
 *  consume() call — regardless of the `now` value the test passes —
 *  always rolls into a fresh window. In production this function is
 *  never called and `lowPriWindowStart` retains its `Date.now()`
 *  initial value, so rollovers happen on real wall-clock time. */
export function _resetLowPriorityWakeBudgetForTests(): void {
  lowPriWindowStart = 0
  lowPriUsed = 0
  lowPriDroppedInWindow = 0
}

/** Resolve the exceptional recipient on a departure notice. The pubsub payload
 * is only a hint: an attacker or buggy publisher cannot add an arbitrary wake
 * target because the exact message, conversation, tenant, persisted recipient,
 * and active-agent row must all agree in Postgres. Exported for the delivery
 * contract regression test. */
export async function resolveDurableDeliveryAgent(args: {
  conversationId: string
  messageId: string
  companyId: string | undefined
  claimedRecipientId: string | undefined
}): Promise<string | null> {
  if (!args.companyId || !args.claimedRecipientId) return null
  const { rows } = await pool.query<{ delivery_recipient_id: string }>(
    `SELECT dm.delivery_recipient_id
       FROM messages dm
       JOIN conversations c
         ON c.id = dm.conversation_id
        AND c.company_id = $3
       JOIN participants p
         ON p.id = dm.delivery_recipient_id
        AND p.company_id = c.company_id
        AND p.kind = 'agent'
        AND p.departed_at IS NULL
      WHERE dm.id = $1
        AND dm.conversation_id = $2
        AND dm.kind = 'system'
        AND dm.delivery_recipient_id = $4
      LIMIT 1`,
    [args.messageId, args.conversationId, args.companyId, args.claimedRecipientId],
  )
  return rows[0]?.delivery_recipient_id ?? null
}

async function wakeOne(
  agentId: string,
  reason: WakeReason,
  conversationId: string | null,
  steerPayload: SteerWakePayload | null = null,
  options: WakeOptions = {},
  retryAttempt: number = 0,
): Promise<boolean> {
  // Synthetic wakes can be dropped under load — the next idle tick
  // or next scanner pass will re-evaluate. Real wakes never are.
  if ((reason === 'idle' || reason === 'background_scan') && !_consumeLowPriorityWakeBudget()) {
    console.warn(`[scheduler] ${agentId} ${reason} wake dropped: budget ${LOW_PRIORITY_WAKE_BUDGET_PER_MIN}/min exceeded`)
    return false
  }

  // Execution placement is an authorization decision, not a nullable hint.
  // A connected runtime can receive a direct wake without starting anything;
  // placement becomes mandatory for managed-message triage and whenever zero
  // subscribers would send us toward ensurePod.
  const resolveHostForWake = async (): Promise<ResolvedAgentHost | null> => {
    const hostResult = await resolveAgentHost(agentId)
    if (hostResult.status === 'missing') {
      console.warn(`[scheduler] ${agentId} wake ignored: no active agent row`)
      return null
    }
    if (hostResult.status === 'error') {
      const cause = hostResult.cause instanceof Error
        ? `: ${hostResult.cause.message}`
        : ''
      console.error(`[scheduler] ${hostResult.reason}${cause}`)
      if (hostResult.code === 'lookup_failed') {
        await scheduleWakeRetry(
          agentId,
          reason,
          conversationId,
          steerPayload,
          options,
          retryAttempt + 1,
          hostResult.reason,
          'host_resolution',
        )
      } else {
        void notifyAlert({
          label: 'scheduler.invalid_agent_host_assignment',
          error: new Error(hostResult.reason),
          extras: { agentId, reason, conversationId },
        })
      }
      return null
    }
    return hostResult
  }

  let host: ResolvedAgentHost | null = null
  if (options.placementTriage) {
    host = await resolveHostForWake()
    if (!host) return false
    // BYOA daemons triage locally. Managed Agents are gated before a live-pod
    // wake or a new Pod; the flag is serialized into retries so recovery cannot
    // bypass the same placement + triage contract.
    if (!isByoaKind(host.kind) && reason === 'message.new' && !options.triageNote) {
      const verdict = await triageWakeRecipient(agentId, options.triageTarget ?? null)
      if (!verdict) return false
      options = { ...options, ...verdict }
    }
  }

  const wakePayload = {
    kind: 'wake' as const,
    reason,
    conversationId,
    ...(options.idleReason ? { idleReason: options.idleReason } : {}),
    ...(options.backgroundBrief ? { backgroundBrief: options.backgroundBrief } : {}),
    ...(options.pollBrief ? { pollBrief: options.pollBrief } : {}),
    ...(options.triageNote ? { triageNote: options.triageNote } : {}),
  }
  const delivered = await deliverWake(agentId, wakePayload)

  // Steering: if the agent's pod is currently mid-turn (busy lease in
  // Redis), ALSO publish the message body as a steer event so the
  // running turn can inject it at the next hop boundary. Steer is
  // additive to wake — the wake gets the pod's drain loop spinning,
  // and the steer carries the actual content. If the pod is idle the
  // steer payload is harmless (queue gets discarded by the new turn's
  // resetSteerForAgent at start).
  //
  // STEER_ENABLED kill-switch: if env says false, never publish a
  // steer. Wake still fires; the message is in the DB; the agent's
  // next turn picks it up via loadInbox normally. Use this knob to
  // disable steering during an incident without a redeploy.
  if (env.STEER_ENABLED && steerPayload && delivered > 0) {
    const busy = await isAgentBusy(agentId).catch(() => false)
    if (busy) {
      // Per-agent rate limit: at most STEER_RATE_PER_MINUTE steers
      // delivered in any rolling 60s window. Above that, fall through
      // to wake-only (message is still in DB → next wake picks it up).
      // Without this, a spam loop could saturate MAX_BATCHES_PER_TURN
      // within seconds and the agent's runtime budget burns down.
      const allowed = await consumeSteerRateToken(agentId)
      if (!allowed) {
        console.warn(`[scheduler] steer rate-limited for ${agentId}; falling back to wake-only`)
      } else {
        await deliverSteer(agentId, steerPayload).catch((err) =>
          console.warn(`[scheduler] deliverSteer(${agentId}) failed:`,
            err instanceof Error ? err.message : err),
        )
        // Visible acknowledgement: the steer payload won't actually be
        // injected into the LLM context until the next hop boundary
        // (turn.ts drains between LLM iterations, not mid-tool). For a
        // long-running tool call — yt-dlp / ffmpeg / opencli browser —
        // that boundary could be minutes away, during which the user
        // sees nothing at all and the agent reads as "silent."
        //
        // Fire a typing indicator on the source conversation so the
        // renderer immediately shows "<agent> is typing…" — proof to
        // the human that their steer was received and is queued. The
        // typing event auto-expires in the renderer (no `done:true`
        // teardown needed); when the agent eventually replies, the
        // typing indicator gets replaced by the new message naturally.
        if (steerPayload.conversationId) {
          await publish(CH_TYPING, {
            type: 'typing',
            conversationId: steerPayload.conversationId,
            agentId,
            done: false,
            companyId: steerPayload.companyId,
          }).catch((err) =>
            console.warn(`[scheduler] steer-ack publishTyping(${agentId}) failed:`,
              err instanceof Error ? err.message : err),
          )
        }
      }
    }
  }

  if (delivered > 0) return true

  if (!host) {
    host = await resolveHostForWake()
    if (!host) return false
  }

  // Is there anything left to catch up ON? `message.new` is backed by the
  // message row, so a runtime that was offline finds it on its next drain and
  // an undelivered wake still comes true. Every other reason carries its whole
  // content in the payload — an idle nudge, the scanner's brief, a manual poke
  // — and that payload is gone the moment nobody is subscribed. The managed-pod
  // path below already draws this line (it replays a synthetic wake until the
  // pod attaches, and returns false if it never does); these two early returns
  // are the same wake taking a different exit, and they were reporting success
  // for a brief that reached no one.
  //
  // The scanner is the caller that notices: on `false` it declines to spend the
  // activity fingerprint, so the next pass re-evaluates once the daemon is back.
  // On `true` it records "background scan wake queued", claims the fingerprint,
  // and that scan is never offered again.
  const durableWithoutDelivery = reason === 'message.new'

  // BYOA agents run on a user-paired Computer (the `cumora agent computer`
  // daemon), never a server-managed pod. If delivered === 0 the daemon
  // simply isn't subscribed right now (host offline / asleep) — there's
  // nothing to spin up. Skip the pod path entirely; do NOT ensurePod /
  // wake-retry kubectl.
  if (isByoaKind(host.kind)) {
    console.log(`[scheduler] ${agentId} is BYOA (${host.kind}); daemon offline — ${
      durableWithoutDelivery ? 'wake deferred to reconnect' : `${reason} wake not delivered`}`)
    return durableWithoutDelivery
  }

  // Free tier is BYOA-only: it must NEVER spin a managed Cumora Cloud pod. A free
  // agent only runs when its own paired daemon is connected (delivered>0, handled
  // above) or reconnects (isByoaKind, handled above). Reaching here for a free
  // company means the agent is unassigned/managed with no live daemon — defer, do
  // NOT ensurePod. Previously this was ungated ("grandfathered free on cloud"),
  // which silently ran thousands of free agents on managed cloud — a real cost
  // leak; the polluted legacy data (cloud computers + managed engines) was cleaned
  // up separately. The wake stays durable in the inbox for whenever they pair.
  if (host.tier === 'free') {
    console.log(`[scheduler] ${agentId} is free-tier (BYOA-only); no managed pod — ${
      durableWithoutDelivery ? 'wake deferred until paired' : `${reason} wake not delivered`}`)
    return durableWithoutDelivery
  }

  // Paid (pro/max) managed agent — spin up a Pod. The Pod will catch up on first
  // connect via its initial drain(); we don't need to deliver the
  // event explicitly afterwards because the inbox IS the source of
  // truth.
  const r = await ensurePod(agentId)
  if (r.created) {
    console.log(`[scheduler] ${agentId} resting → spinning up pod (${reason})`)
    // A successful kubectl apply does not guarantee the pod will ever
    // reach the runtime wake stream: kubelet can reject allocation
    // immediately after scheduling (for example a transient unhealthy
    // devic.es/fuse device). The call below defaults failureClass to
    // `ensure_pod`, and that class is queued only for `manual` wakes.
    // For `message.new` it is a no-op. The durable inbox is the recovery
    // path, and replaying the wake can duplicate a turn. The inline replay
    // loop further down is also skipped for `message.new`, so a durable
    // wake whose new pod never attaches is not retried here.
    await scheduleWakeRetry(agentId, reason, conversationId, steerPayload, options, Math.max(1, retryAttempt + 1), 'post-spawn health check')
  } else if (!r.ok) {
    console.error(`[scheduler] ${agentId} ensurePod failed: ${r.reason}`)
    await scheduleWakeRetry(
      agentId,
      reason,
      conversationId,
      steerPayload,
      options,
      retryAttempt + 1,
      r.reason,
      r.code === 'placement_lookup_failed' ? 'host_resolution' : 'ensure_pod',
    )
    return false
  } else if (r.reason === 'already pending' || r.reason === 'already running') {
    await scheduleWakeRetry(agentId, reason, conversationId, steerPayload, options, retryAttempt + 1, r.reason)
  }

  // Message wakes are durable because the message is already in the inbox,
  // and the pod's cold-start drain reads that source of truth. Synthetic
  // idle/background wakes have no inbox row, so replay the wake briefly after
  // the pod is created until its SSE stream attaches.
  if (reason !== 'message.new') {
    const deadline = Date.now() + 20_000
    while (Date.now() < deadline) {
      await new Promise((resolve) => setTimeout(resolve, 500))
      const replayed = await deliverWake(agentId, wakePayload).catch(() => 0)
      if (replayed > 0) {
        // Confirmed: the pod attached and took this wake. The health-check
        // retry was the net for "it never attached" — drop it before it
        // delivers the same brief again.
        await _cancelWakeRetry(agentId, reason, conversationId)
        return true
      }
    }
    console.warn(`[scheduler] ${agentId} synthetic wake (${reason}) was not delivered after pod start`)
    return false
  }
  return true
}

/** Multi-instance dedup: every cumora-server replica subscribes to
 *  CH_MESSAGE_NEW, so a single message.new fan-outs to N servers.
 *  Without this guard each replica would call wake() → ensurePod →
 *  deliverWake (Redis publish) independently — agent Pod ends up
 *  receiving N copies of the wake event (drain folds via
 *  pendingRerun, so it's correctness-safe but wasteful), and N
 *  parallel kubectl-applies fight at the K8s API server.
 *
 *  Fix: SETNX a key per message id; only the first replica that
 *  claims it proceeds. TTL=60s reaps the key automatically. */
async function claimAndWake(payload: MessageNewEvent): Promise<void> {
  const key = `cumora:wake-claim:${payload.message.id}`
  const claimed = await redis.set(key, '1', 'EX', 60, 'NX').catch(() => null)
  if (claimed === null) return     // another replica owns this wake
  await wake(payload)
}

// ─── author-name cache (Fix #10) ─────────────────────────────────────
//
// Phase 4 originally did `SELECT name FROM participants WHERE id = $1`
// for every new message to resolve the display name for the steer
// prefix. That's N+1 per group-chat message (one query per agent
// member that gets a steer). participants.name changes rarely, so a
// small LRU with a 5-minute TTL collapses the N+1 into ~0 queries in
// steady state. Cache miss is still correct — falls back to authorId.

interface CachedName { name: string; expiresAt: number }
const AUTHOR_NAME_CACHE_TTL_MS = 5 * 60_000
const AUTHOR_NAME_CACHE_MAX = 5_000 // rough cap on a single-process Map
const authorNameCache = new Map<string, CachedName>()

/** Exported for testability. Lookup happens at most once per
 *  authorId per AUTHOR_NAME_CACHE_TTL_MS, with a capacity-trim pass
 *  on overflow. Returns authorId on any DB error so the steer
 *  prefix never breaks. */
export async function resolveAuthorName(authorId: string): Promise<string> {
  const now = Date.now()
  const hit = authorNameCache.get(authorId)
  if (hit && hit.expiresAt > now) return hit.name
  // Best-effort DB read. On error we cache nothing (so the next
  // message retries) and return the id as fallback.
  try {
    const { rows } = await pool.query<{ name: string | null }>(
      `SELECT name FROM participants WHERE id = $1 LIMIT 1`, [authorId],
    )
    const name = rows[0]?.name || authorId
    // Opportunistic capacity trim — when the map exceeds the cap,
    // drop everything that's expired; if still over, drop the
    // OLDEST entries (Map iteration is insertion-ordered, so the
    // first entries are the oldest).
    if (authorNameCache.size >= AUTHOR_NAME_CACHE_MAX) {
      for (const [k, v] of authorNameCache) {
        if (v.expiresAt <= now) authorNameCache.delete(k)
      }
      while (authorNameCache.size >= AUTHOR_NAME_CACHE_MAX) {
        const firstKey = authorNameCache.keys().next().value
        if (firstKey === undefined) break
        authorNameCache.delete(firstKey)
      }
    }
    authorNameCache.set(authorId, { name, expiresAt: now + AUTHOR_NAME_CACHE_TTL_MS })
    return name
  } catch {
    return authorId
  }
}

/** Test-only: clear the author-name cache so test cases don't poison
 *  each other. Production code never calls this. */
export function _resetAuthorNameCacheForTests(): void {
  authorNameCache.clear()
}

// ─── steer rate limit (Fix #8) ───────────────────────────────────────
//
// A spam loop (compromised account, runaway script, etc.) could fire
// hundreds of messages/sec at one agent. Even with MAX_BATCHES_PER_TURN
// and MAX_BYTES_PER_TURN limiting in-pod consumption, the network +
// Redis pub/sub bandwidth + the agent's hop budget would still burn.
// Rate limit at the publish edge: at most STEER_RATE_PER_MINUTE steers
// dispatched per agent in any rolling 60s window. Beyond that, fall
// through to wake-only (message stays in DB → next wake handles).

const STEER_RATE_PER_MINUTE = 30

/** Attempts to consume one steer-rate token for the agent. Returns
 *  true if allowed, false if the rate limit is exceeded. Uses an
 *  atomic Lua-like INCR-and-check, which on first call sets a 60s
 *  TTL — so the counter naturally resets. Fail-open on Redis errors
 *  (steers continue; the wake path is correct anyway). */
async function consumeSteerRateToken(agentId: string): Promise<boolean> {
  try {
    const key = `cumora:steer-rate:${agentId}`
    // INCR is atomic; we set the TTL only on the FIRST increment so
    // the window doesn't get reset on every call. EXPIRE returns 1
    // if the TTL was applied, 0 if the key already had one or didn't
    // exist. Race-safe enough at the per-second granularity we need.
    const count = await redis.incr(key)
    if (count === 1) {
      await redis.expire(key, 60).catch(() => { /* best-effort */ })
    }
    return count <= STEER_RATE_PER_MINUTE
  } catch {
    return true // fail-open
  }
}

/** Test-only: clear the rate-limit counter for an agent so tests
 *  don't poison each other. Production never calls this. */
export async function _resetSteerRateForTests(agentId: string): Promise<void> {
  try { await redis.del(`cumora:steer-rate:${agentId}`) } catch { /* ignore */ }
}

async function wake(payload: MessageNewEvent): Promise<void> {
  const conversationId = payload.conversationId
  const authorId = payload.message.authorId
  const messageKind = payload.message.kind
  const messageBody = payload.message.body
  const messageId = payload.message.id

  // Steering: build a steer payload only for user-visible message
  // kinds — system notices (quota exhausted, etc.) are noise the
  // model doesn't need re-injected mid-turn. Resolve the author's
  // display name via the cache above so a group chat with N agents
  // doesn't trigger N participants-lookups per message.
  let steerPayload: SteerWakePayload | null = null
  if (messageKind !== 'system' && messageBody && messageBody.length > 0) {
    const authorName = await resolveAuthorName(authorId)
    steerPayload = { messageId, conversationId, authorName, body: messageBody, companyId: payload.companyId ?? '' }
  }

  const { rows: convoRows } = await pool.query<{
    members: string[]
    kind: string
    company_id: string
    muted_agent_ids: string[]
  }>(
    `SELECT COALESCE((
              SELECT array_agg(cm.participant_id ORDER BY cm.ordinal, cm.participant_id)
                FROM conversation_members cm
               WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
            ), ARRAY[]::text[]) AS members,
            c.kind, c.company_id,
            COALESCE((
              SELECT array_agg(mu.user_id)
                FROM conversation_mutes mu
               WHERE mu.conversation_id = c.id
                 AND (mu.muted_until IS NULL OR mu.muted_until > NOW())
            ), ARRAY[]::text[]) AS muted_agent_ids
       FROM conversations c
      WHERE c.id = $1`,
    [conversationId],
  )
  const conversation = convoRows[0]
  if (!conversation) return
  const members = conversation?.members ?? []
  if (steerPayload) steerPayload.companyId = conversation.company_id
  const mutedAgentIds = new Set(conversation?.muted_agent_ids ?? [])
  let quotedAuthorId = payload.message.quoted?.authorId ?? null
  if (!quotedAuthorId && payload.message.quotedMessageId && mutedAgentIds.size > 0) {
    const { rows } = await pool.query<{ author_id: string }>(
      `SELECT author_id FROM messages WHERE id = $1 AND conversation_id = $2`,
      [payload.message.quotedMessageId, conversationId],
    )
    quotedAuthorId = rows[0]?.author_id ?? null
  }
  const { rows: currentAgentRows } = await pool.query<{ id: string }>(
    `SELECT id FROM participants
      WHERE company_id = $1 AND kind = 'agent' AND departed_at IS NULL
        AND id = ANY($2::text[])`,
    [conversation.company_id, members],
  )
  const currentAgents = new Set(currentAgentRows.map((row) => row.id))
  const agentRecipients: string[] = []
  for (const m of members) {
    if (m === authorId) continue
    // The normalized membership FK is tenant-constrained; the active
    // participant filter additionally excludes offboarded agents.
    if (!currentAgents.has(m)) continue
    if (mutedAgentIds.has(m) && !shouldDeliverToMutedAgent({
      agentId: m,
      conversationKind: conversation?.kind ?? 'group',
      body: messageBody,
      quotedAuthorId,
    })) continue
    agentRecipients.push(m)
  }

  // Whether to reply into an agent-only thread is the small model's decision
  // (it sees the per-conversation "thread heat" signal and goes quiet). The
  // scheduler no longer makes that call — it only applies a content-blind cost
  // floor: when an AGENT's message would wake peers, drop the wake for any
  // recipient that is over its activation budget, so a runaway can't burn
  // unbounded cost. Human-driven wakes are NEVER throttled.
  const authorIsAgent = currentAgents.has(authorId) || Boolean((await pool.query(
    `SELECT 1 FROM participants
      WHERE id = $1 AND company_id = $2
        AND kind = 'agent' AND departed_at IS NULL`,
    [authorId, conversation.company_id],
  )).rowCount)
  let recipients = agentRecipients
  if (authorIsAgent) {
    const allowed = await Promise.all(
      agentRecipients.map(async (m) => (await consumeAgentTurnToken(m)) ? m : null),
    )
    recipients = allowed.filter((m): m is string => m !== null)
    const dropped = agentRecipients.length - recipients.length
    if (dropped > 0) {
      console.warn(`[scheduler] turn-rate floor: dropped ${dropped} agent-driven wake(s) in ${conversationId} (over ${AGENT_TURN_RATE_PER_MINUTE}/min)`)
    }
  }

  // A leave/kick commits the membership removal before this event is
  // published, so the departing agent no longer appears in `members`. The
  // persisted message may name one durable delivery recipient; the query
  // above validates that it is an active agent in this conversation's tenant
  // before we restore it to fan-out. Departure explanations bypass mute and
  // the agent-authored cost floor because this is the final access-revocation
  // notice, not ordinary conversation chatter.
  const deliveryAgentId = await resolveDurableDeliveryAgent({
    conversationId,
    messageId,
    companyId: conversation.company_id,
    claimedRecipientId: payload.message.deliveryRecipientId,
  })
  if (deliveryAgentId && !recipients.includes(deliveryAgentId)) {
    recipients = [...recipients, deliveryAgentId]
  }

  // ROUTING (#70). A human message that explicitly NAMES agents is usually for
  // them, and waking the rest of the room costs a full big-brain turn each to
  // reach "not mine" — production measures ~26% of group wakes replying with
  // nothing. Ask the cerebellum ONCE per message (not once per agent) whether the
  // named agents are the intended audience, and narrow only then.
  //
  // Deliberately conservative, because narrowing is the one mistake that is
  // SILENT — a agent that should have answered and was never woken leaves no
  // reply, no typing indicator, no agent_runs row. So this only ever runs for a
  // human-authored group message with a real named subset, and every uncertainty
  // (@all, no targets, a model error, an unparseable answer) keeps today's full
  // fan-out. See routing.ts.
  //
  // The UNADDRESSED case (names nobody) gets the same treatment when
  // env.ROUTING_ONE_OF_US is on: the router may elect ONE agent to take the
  // turn instead of waking the room. The election is bounded by a lease row
  // (routing-claims.ts) — a primary that never starts a turn is replaced by
  // the next candidate — and every uncertainty fails open to the fan-out
  // below. It deliberately does NOT resurrect the daemon-side `claimReply`
  // that was removed for breaking chains: this happens once per message,
  // before waking, and the woken agent still runs its own glance/yield.
  if (!authorIsAgent && (conversation?.kind ?? 'group') !== 'direct' && recipients.length > 1 && messageKind !== 'system' && !deliveryAgentId) {
    const targets = [
      ...mentionedAgentIds(messageBody, recipients),
      ...(quotedAuthorId && recipients.includes(quotedAuthorId) ? [quotedAuthorId] : []),
    ]
    const uniqueTargets = [...new Set(targets)]
    if (uniqueTargets.length > 0) {
      const mode = await routeMessage({
        companyId: conversation.company_id,
        body: messageBody,
        conversationKind: conversation?.kind ?? 'group',
        candidates: recipients,
        targets: uniqueTargets,
      })
      const routed = recipientsForRoute(mode, recipients, uniqueTargets)
      if (routed.length !== recipients.length) {
        console.log(`[scheduler] routed ${conversationId} to ${routed.join(', ')} (mode=${mode}, ${recipients.length - routed.length} wake(s) avoided)`)
      }
      recipients = routed
    } else if (env.ROUTING_ONE_OF_US) {
      const roster = await loadElectionCandidates(recipients).catch(() => [] as ElectionCandidate[])
      const route = await routeUnaddressedMessage({
        companyId: conversation.company_id,
        body: messageBody,
        conversationKind: conversation?.kind ?? 'group',
        candidates: roster.map((c) => ({ id: c.id, role: c.role })),
      })
      if (route.mode === 'one-of-us') {
        const election = electLineup(route.primary, roster, { leaseMs: BUSY_STATUS_LEASE_MS })
        if (election) {
          // Fail open: if the lease row cannot be written, the no-show sweep
          // has nothing to advance, so narrowing would be silent-room with no
          // safety net — keep the full fan-out instead.
          // An existing row means the message re-delivered after the wake
          // claim TTL lapsed: honor the recorded primary instead of
          // re-electing, and skip entirely when the lease already resolved.
          const claim = await claimPrimary({
            messageId: payload.message.id,
            companyId: conversation.company_id,
            conversationId,
            orderedCandidates: election.lineup,
          }).catch(() => null)
          if (claim?.status === 'served' || claim?.status === 'exhausted') {
            console.log(`[scheduler] claim for message ${payload.message.id} is already ${claim.status} — skipping wake on re-delivery`)
            recipients = []
          } else if (claim?.status === 'pending') {
            const primary = claim.candidates[claim.cursor]
            if (primary && recipients.includes(primary)) {
              console.log(`[scheduler] routed ${conversationId} to ${primary} (mode=one-of-us, ${recipients.length - 1} wake(s) avoided)`)
              recipients = [primary]
            }
          }
        }
      }
    }
  }

  // Always parallel fan-out. We tried a scheduler-side serial queue
  // for @all (stagger → event-driven Redis state machine) — both
  // worked mechanically but felt like nothing a real team would do.
  // Real coordination happens at the AGENT level: see the room,
  // glance once more before committing, defer when a peer is on it.
  // Those affordances live in pod-agent + the `cumora glance` tool +
  // the broadcast etiquette section of the persona prompt — the
  // scheduler wakes every subscribed, non-muted agent at the same time,
  // like a Slack room. A muted agent only passes through for an exact
  // mention or quoted reply.
  await fanOutWake(
    recipients,
    conversationId,
    steerPayload,
    deliveryAgentId ? { recipientId: deliveryAgentId, messageId } : null,
  )
}

function escapeRegex(value: string): string {
  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}

/** Muted rooms are genuinely absent from an agent's delivery stream. The
 * only intentional escape hatches are a direct conversation, an explicit
 * @agent-id token, or a reply to one of that agent's messages. Exported so
 * the delivery contract stays regression-testable without Redis/Postgres. */
export function shouldDeliverToMutedAgent(args: {
  agentId: string
  conversationKind: string
  body: string
  quotedAuthorId: string | null
}): boolean {
  if (args.conversationKind === 'direct') return true
  if (args.quotedAuthorId === args.agentId) return true
  const mention = new RegExp(`(^|[^\\w@])@${escapeRegex(args.agentId)}(?![\\w-])`, 'i')
  return mention.test(args.body)
}

/** Which of `agentMemberIds` this message explicitly @-mentions.
 *
 *  Same token rule as {@link shouldDeliverToMutedAgent} — an exact `@id`, not a
 *  prefix and not an email — so "addressed to" means one thing everywhere.
 *
 *  This is a PROMPT signal, never a delivery decision. Narrowing the wake to
 *  the mentioned agents looks tempting and is not safe: a mentioned BYOA agent
 *  whose laptop is closed has its wake deferred, and the peers who would have
 *  covered were never woken — so the room stays silent. Worse, an excluded peer
 *  who posts anything else in that room auto-acks their read cursor to NOW
 *  (`cumora reply`), which puts the human's message permanently BEHIND the
 *  cursor: not deferred, gone. So every member still wakes, and the mention only
 *  tells them who it was for.
 *
 *  Exported for tests. */
export function mentionedAgentIds(body: string, agentMemberIds: readonly string[]): string[] {
  if (!body) return []
  return agentMemberIds.filter((id) =>
    new RegExp(`(^|[^\\w@])@${escapeRegex(id)}(?![\\w-])`, 'i').test(body))
}

/** Render a small-brain inbox verdict into the note attached to the wake. */
function renderTriageNote(verdict: InboxTriageVerdict): string {
  const state = verdict.actionable ? 'relevant' : 'not relevant'
  const reason = verdict.reason.trim() ? `\nReason: ${verdict.reason.trim().slice(0, 500)}` : ''
  return `Small-brain inbox triage (${state}, ${verdict.source}): ${verdict.promptNote.trim()}${reason}`
}

export async function triageWakeRecipient(
  agentId: string,
  durableDelivery: { conversationId: string; messageId: string } | null = null,
): Promise<WakeOptions | null> {
  try {
    const persona = await inprocClient.loadPersona(agentId)
    if (!persona) return null
    let inbox = await inprocClient.loadInbox(agentId)
    if (inbox.length === 0) return null
    if (durableDelivery) {
      const terminal = inbox.find((row) =>
        row.id === durableDelivery.messageId &&
        row.conversation_id === durableDelivery.conversationId)
      if (terminal) {
        // Managed/cloud agents do not need an expensive main-brain turn for an
        // informational departure. Consume the exact persisted row here, which
        // mirrors the BYOA daemon's system-only snapshot+ack path and prevents
        // the terminal notice from replaying forever or resurfacing after a
        // later re-invite. Other unread work remains eligible for normal triage.
        await inprocClient.markConversationRead({
          agentId,
          conversationId: terminal.conversation_id,
          upToMessageId: terminal.id,
        })
        console.log(`[scheduler] ${agentId} acknowledged durable departure notice ${terminal.id}`)
        inbox = inbox.filter((row) => row.id !== terminal.id)
        if (inbox.length === 0) return null
      }
    }
    const convoIds = [...new Set(inbox.map((m) => m.conversation_id))]
    const context = await inprocClient.loadContext(agentId, persona.companyId, convoIds)
    const verdict = await classifyInboxTriage({
      agentId,
      companyId: persona.companyId,
      persona,
      inbox,
      context,
    })
    if (!verdict.actionable) {
      console.log(`[scheduler] ${agentId} message.new skipped by inbox triage: ${verdict.reason}`)
      return null
    }
    return { triageNote: renderTriageNote(verdict) }
  } catch (err) {
    const reason = err instanceof Error ? err.message : String(err)
    console.warn(`[scheduler] ${agentId} inbox triage unavailable; waking fail-open: ${reason}`)
    return {
      triageNote:
        `Small-brain inbox triage failed in the scheduler, so this wake is fail-open. ` +
        `Read the inbox/context yourself and do not reply unless the new messages concern you. Reason: ${reason.slice(0, 500)}`,
    }
  }
}

export async function fanOutWake(
  recipients: string[],
  conversationId: string,
  steerPayload: SteerWakePayload | null,
  durableDelivery: { recipientId: string; messageId: string } | null = null,
): Promise<void> {
  // Bounded fan-out (env.WAKE_FANOUT_CONCURRENCY). Each recipient's work
  // — host resolve, the support-model triage (3 DB reads + a model call)
  // and wakeOne (Redis publish + ensurePod/kubectl) — runs under one
  // semaphore so a large group / agent reply-storm can't stampede the pg
  // pool and trip the triage fail-open amplification loop (the
  // 2026-05-27 connection-exhaustion outage). Excess recipients queue
  // and drain at a sustainable rate; a delayed wake still self-heals —
  // the message is durable in the inbox, the pod drains on attach, and
  // the per-message wake-claim TTLs out in 60s.
  await Promise.all(recipients.map((m) => wakeFanoutSem.run(async () => {
    // One recipient failing (triage throw, kubectl flake) must not stop
    // the others or escape into the Redis on-message handler.
    try {
      // wakeOne owns the single placement decision and managed triage. Keeping
      // both inside the retryable operation prevents an earlier lookup failure
      // from being reinterpreted as cloud and preserves the durable departure
      // target across a retry.
      const options: WakeOptions = durableDelivery?.recipientId === m
        ? {
            placementTriage: true,
            triageTarget: { conversationId, messageId: durableDelivery.messageId },
          }
        : { placementTriage: true }
      // Await INSIDE the slot so it's held across host resolution, triage and
      // ensurePod (kubectl). That's what bounds DB, model and child-process
      // pressure together.
      await wakeOne(m, 'message.new', conversationId, steerPayload, options)
    } catch (err) {
      console.error(`[scheduler] wakeOne(${m}) failed:`, err instanceof Error ? err.message : err)
    }
  })))
}


let started = false
export function startScheduler(): void {
  if (started) return
  started = true
  void sub.subscribe(CH_MESSAGE_NEW, (err) => {
    if (err) console.error('[scheduler] subscribe failed', err)
  })
  // Wake the poll author on every state change. Without this an agent
  // who posts a poll sees the original message land in everyone's
  // inbox but never finds out when votes actually come in — and they
  // can neither nudge laggards mid-poll nor summarize when it closes.
  void sub.subscribe(CH_POLLS, (err) => {
    if (err) console.error('[scheduler] CH_POLLS subscribe failed', err)
  })
  sub.on('message', (channel, raw) => {
    if (channel === CH_MESSAGE_NEW) {
      let payload: MessageNewEvent
      try { payload = JSON.parse(raw) as MessageNewEvent } catch { return }
      if (payload.type !== 'message.new') return
      // fire-and-forget — pool.query inside wake() or ensurePod's kubectl
      // shell-out can transiently reject. We don't want an unhandled
      // rejection here; just log and let the next wake retry. The
      // wake-claim Redis key TTLs out in 60s so a missed wake recovers
      // naturally on the next message in the same convo.
      claimAndWake(payload).catch((err) => {
        console.error(`[scheduler] claimAndWake failed for ${payload.message?.id}:`,
          err instanceof Error ? err.message : err)
      })
      return
    }
    if (channel === CH_POLLS) {
      let payload: PollUpdatedEvent
      try { payload = JSON.parse(raw) as PollUpdatedEvent } catch { return }
      if (payload.type !== 'poll.updated') return
      handlePollUpdated(payload).catch((err) => {
        console.error(`[scheduler] handlePollUpdated failed for ${payload.messageId}:`,
          err instanceof Error ? err.message : err)
      })
      return
    }
  })
  startWakeRetryWorker()
  startRoutingClaimSweeperIfEnabled()
  console.log(`[scheduler] mailbox scheduler listening on ${CH_MESSAGE_NEW}, ${CH_POLLS} · runtime=pod-only`)
}

// ─── poll author wake (real-time vote watching) ──────────────────────
//
// When a poll moves (vote cast, vote changed, poll closed), wake the
// agent who AUTHORED that poll so it can:
//   • watch the tally come in
//   • nudge laggards mid-poll
//   • summarize on close
//
// Filters:
//   • author must be an active agent (humans get UI updates already)
//   • skip when the actor IS the author (we don't self-wake an agent
//     who just voted on their own poll — would also infinite-loop)
//
// Debounce:
//   • vote events claim cumora:poll-vote-wake-claim:<msg> with TTL
//     POLL_VOTE_WAKE_DEBOUNCE_SECONDS. Only the first replica/event
//     within the window wakes; subsequent votes silently drop. The
//     agent's turn reads fresh state via the brief built RIGHT BEFORE
//     the wake (and can call `cumora poll show` for newer state).
//   • close events claim a separate cumora:poll-close-wake-claim:<msg>
//     with a long TTL so cross-replica idempotency holds; closePoll
//     is itself idempotent so a duplicate event is harmless apart
//     from the wasted wake.
const POLL_VOTE_WAKE_DEBOUNCE_SECONDS = 8
const POLL_CLOSE_WAKE_CLAIM_SECONDS = 600

export async function handlePollUpdated(event: PollUpdatedEvent): Promise<boolean> {
  if (!event.companyId) return false
  const { rows } = await pool.query<{ author_id: string; members: string[] }>(
    `SELECT m.author_id,
            COALESCE((
              SELECT array_agg(cm.participant_id ORDER BY cm.ordinal, cm.participant_id)
                FROM conversation_members cm
               WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
            ), ARRAY[]::text[]) AS members
       FROM messages m
       JOIN conversations c ON c.id = m.conversation_id
       JOIN participants author
         ON author.id = m.author_id
        AND author.company_id = c.company_id
        AND author.kind = 'agent'
        AND author.departed_at IS NULL
      WHERE m.id = $1 AND m.company_id = $2
        AND m.conversation_id = $3
        AND c.company_id = $2
        AND EXISTS (
          SELECT 1 FROM conversation_members cm
           WHERE cm.conversation_id = c.id
             AND cm.company_id = c.company_id
             AND cm.participant_id = m.author_id
        )`,
    [event.messageId, event.companyId, event.conversationId],
  )
  const row = rows[0]
  if (!row) return false
  const authorId = row.author_id
  // The join above is the disclosure boundary: a departed, moved, or kicked
  // poll author must not receive a later tally containing room membership,
  // voter identities, or option text.
  // Skip self-wake: if the agent voted on (or closed) their own poll,
  // they already know.
  if (event.actorId && event.actorId === authorId) return false

  const isClose = Boolean(event.poll.closedAt)
  const claimKey = isClose
    ? `cumora:poll-close-wake-claim:${event.messageId}`
    : `cumora:poll-vote-wake-claim:${event.messageId}`
  const claimTtl = isClose ? POLL_CLOSE_WAKE_CLAIM_SECONDS : POLL_VOTE_WAKE_DEBOUNCE_SECONDS
  const claimed = await redis.set(claimKey, '1', 'EX', claimTtl, 'NX').catch(() => null)
  if (claimed === null) return false // another replica owned this wake, or debounce window still open

  // Resolve display names for everyone we plan to surface in the
  // brief — author already excluded, voters from the tallies, and
  // pending voters (members − {author} − voters).
  const voterSet = new Set<string>()
  for (const t of event.tallies) {
    for (const v of t.voterIds) voterSet.add(v)
  }
  const idsToResolve = new Set<string>([...row.members, ...voterSet])
  if (event.actorId) idsToResolve.add(event.actorId)
  const names = await resolveCurrentParticipantNames([...idsToResolve], event.companyId)
  const currentIds = new Set(names.keys())
  const currentVoters = new Set([...voterSet].filter((id) => currentIds.has(id)))
  const pendingIds = row.members.filter((id) =>
    id !== authorId && currentIds.has(id) && !currentVoters.has(id))

  const brief: PollWakeBrief = {
    messageId: event.messageId,
    conversationId: event.conversationId,
    question: event.poll.question,
    mode: event.poll.mode,
    status: isClose ? 'closed' : 'open',
    closedReason: event.poll.closedReason,
    expiresAt: event.poll.expiresAt,
    totalVotes: event.tallies.reduce(
      (sum, tally) => sum + tally.voterIds.filter((id) => currentIds.has(id)).length,
      0,
    ),
    tallies: event.poll.options.map((opt) => {
      const tally = event.tallies.find((t) => t.optionId === opt.id)
      const voters = (tally?.voterIds ?? []).filter((id) => currentIds.has(id)).map((id) => ({
        id, name: names.get(id) ?? id,
      }))
      return { optionId: opt.id, text: opt.text, count: voters.length, voters }
    }),
    pending: pendingIds.map((id) => ({ id, name: names.get(id) ?? id })),
    actor: {
      id: event.actorId && currentIds.has(event.actorId) ? event.actorId : null,
      name: event.actorId && currentIds.has(event.actorId) ? (names.get(event.actorId) ?? event.actorId) : null,
    },
    phase: isClose ? 'close' : 'vote',
  }

  await wakeAgent(authorId, 'poll.updated', event.conversationId, null, { pollBrief: brief })
  return true
}

/** Resolve only current participants in the poll conversation's tenant.
 * Conversation member arrays and historical vote rows are denormalized and
 * may contain legacy foreign/departed ids; never disclose those through a
 * poll wake. */
async function resolveCurrentParticipantNames(
  ids: string[],
  companyId: string,
): Promise<Map<string, string>> {
  const out = new Map<string, string>()
  if (ids.length === 0) return out
  try {
    const { rows } = await pool.query<{ id: string; name: string | null }>(
      `SELECT id, name FROM participants
        WHERE company_id = $1 AND id = ANY($2::text[])
          AND kind IN ('agent', 'human') AND departed_at IS NULL`,
      [companyId, ids],
    )
    for (const r of rows) {
      const name = r.name || r.id
      out.set(r.id, name)
    }
  } catch { /* fail closed: an empty map yields no identity disclosure */ }
  return out
}