Multiplayer agent harness for work.
Fix cron contention, approval delivery loops, and slow reports (#1639)
Joshua France5be07724b654
diff --git a/src/core/approval-store.ts b/src/core/approval-store.tsindex f446a0b13..13a81a2a3 100644--- a/src/core/approval-store.ts+++ b/src/core/approval-store.ts@@ -8,13 +8,19 @@ export function approvalDeliveryKey(id: string, record: Pick<PendingApprovalRecoreturn `command-approval:${id}:${record.createdAt ?? 0}`;}+export function approvalDeliveryRecipient(actor: { externalId?: string } | undefined): string | undefined {+ const id = actor?.externalId;+ return id && !id.startsWith("system:") ? id : undefined;+}+export function createApprovalStore(backing: DurableMap<PendingApprovalRecord>,deliveries: Pick<DeliveryStore, "enqueue">,) {async function deliver(id: string, record: PendingApprovalRecord): Promise<void> {if (record.request?.surface !== "slack") return;- const actorId = record.request.actor.externalId;+ const actorId = approvalDeliveryRecipient(record.request.actor);+ if (!actorId) return;await deliveries.enqueue({destination: { ...principalDestination(actorId, actorId), commandApprovalId: id },text: `Approval needed: ${record.summary ?? record.command}`,diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.tsindex 62c5f7832..b1a91c285 100644--- a/src/cron/scheduler.ts+++ b/src/cron/scheduler.ts@@ -201,23 +201,29 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {const maxFiresPerTick = deps.maxFiresPerTick ?? 100;const leaderLease = deps.leaderLease ?? createNoopLeaderLease();+ async function withCronLifecycle<T>(cronId: string, scheduled: boolean, run: () => Promise<T>): Promise<T | null> {+ if (scheduled && deps.lock?.tryWithLock) {+ const result = await deps.lock.tryWithLock(`cron-lifecycle:${cronId}`, run);+ if (result === null) await deps.crons.defer(cronId, now() + BUSY_DEFER_MS);+ return result;+ }+ return deps.lock ? deps.lock.withLock(`cron-lifecycle:${cronId}`, run) : run();+ }+async function fire(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {- const run = async (): Promise<FireResult> => {- const current = await deps.crons.get(cron.id);- if (!current || current.archived || !current.enabled) {- await deps.crons.recordFire(cron.id, {- fireKey,- threadRef: cronFireThreadRef(cron.id, fireKey),- firedAt: t,- endedAt: now(),- status: "silent",- note: "Cron is no longer enabled",- });- return { authzFailed: true };- }- return fireEnabled(current, t, fireKey, scheduledAt);- };- return deps.lock ? deps.lock.withLock(`cron-lifecycle:${cron.id}`, run) : run();+ const current = await deps.crons.get(cron.id);+ if (!current || current.archived || !current.enabled) {+ await deps.crons.recordFire(cron.id, {+ fireKey,+ threadRef: cronFireThreadRef(cron.id, fireKey),+ firedAt: t,+ endedAt: now(),+ status: "silent",+ note: "Cron is no longer enabled",+ });+ return { authzFailed: true };+ }+ return fireEnabled(current, t, fireKey, scheduledAt);}async function fireEnabled(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {@@ -375,8 +381,16 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {for (const cron of batch) {if (stopped || observed !== epoch) break;try {- const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);- if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);+ await withCronLifecycle(cron.id, true, async () => {+ if (stopped || observed !== epoch) return;+ const { authzFailed, deferred } = await fire(+ cron,+ t,+ `cron:${cron.id}:${cron.scheduledAt}`,+ cron.scheduledAt,+ );+ if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);+ });} catch (e) {reportFailure("scheduler: fire", e);}@@ -440,24 +454,28 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {return;}if (stopped || observed !== epoch) return;- if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;- if (stopped || observed !== epoch) {- await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);- return;- }- try {- const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);- if (authzFailed || deferred) {+ const result = await withCronLifecycle(job.cronId, true, async () => {+ if (stopped || observed !== epoch) return;+ if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;+ if (stopped || observed !== epoch) {await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);- if (deferred) await enqueueNext(job.cronId);return;}- } catch (e) {- reportFailure("scheduler: fire", e);- await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);- return;- }- await enqueueNext(job.cronId);+ try {+ const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);+ if (authzFailed || deferred) {+ await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);+ if (deferred) await enqueueNext(job.cronId);+ return;+ }+ } catch (e) {+ reportFailure("scheduler: fire", e);+ await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);+ return;+ }+ await enqueueNext(job.cronId);+ });+ if (result === null) await enqueueNext(job.cronId);}async function reconcile(): Promise<void> {@@ -533,7 +551,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {? { started: false, reason: "already_running", running: begin.running }: { started: false, reason: "unavailable" };}- const settled = withWork(() => fire(cron, t, fireKey)).then(+ const settled = withWork(() => withCronLifecycle(cron.id, false, () => fire(cron, t, fireKey))).then(() => undefined,reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),);diff --git a/src/persistence/durable-map.ts b/src/persistence/durable-map.tsindex 94390817b..6743251c5 100644--- a/src/persistence/durable-map.ts+++ b/src/persistence/durable-map.ts@@ -158,7 +158,11 @@ export function createMemoryMap<T>(): DurableMap<T> {const VERSIONS_TABLE = "durable_map_versions";-export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {+export function createPostgresMap<T>(+ pg: PgPool,+ table: string,+ indexedFields: readonly Extract<keyof T, string>[] = [],+): DurableMap<T> {if (!/^[a-z_][a-z0-9_]*$/i.test(table)) throw new Error(`invalid table name: ${table}`);const migration = {id: `durable-map/${table}/0001`,@@ -167,11 +171,41 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {`CREATE TABLE IF NOT EXISTS ${VERSIONS_TABLE} (tbl TEXT PRIMARY KEY, v BIGINT NOT NULL)`,],};- pg.registerMigration(migration);+ const migrations = [+ migration,+ ...indexedFields.flatMap((field) => {+ if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid index field: ${field}`);+ return [+ {+ id: `durable-map/${table}/lookup-${field.toLowerCase()}-drop`,+ statements: [`SET LOCAL lock_timeout = '3s'`, `DROP INDEX IF EXISTS ${table}_${field.toLowerCase()}_fold`],+ },+ {+ id: `durable-map/${table}/lookup-${field.toLowerCase()}-hash`,+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold+ ON ${table} USING hash (lower(json->>'${field}'))`,+ ],+ },+ {+ id: `durable-map/${table}/select-${field.toLowerCase()}`,+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold+ ON ${table} (lower(json->>'${field}'))`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_unicode+ ON ${table} (id) WHERE json->>'${field}' ~ '[^\\x01-\\x7f]'`,+ ],+ },+ ];+ }),+ ];+ for (const definition of migrations) pg.registerMigration(definition);let readyP: Promise<void> | null = null;function ready(): Promise<void> {if (!readyP) {- readyP = pg.migrate(migration).catch((e) => {+ readyP = (async () => {+ for (const definition of migrations) await pg.migrate(definition);+ })().catch((e) => {readyP = null;throw e;});@@ -224,7 +258,10 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {query.where.field,query.where.anyOfFold.map((v) => v.toLowerCase()),);- sql += ` WHERE (lower(json->>$2) = ANY($3::text[]) OR json->>$2 ~ '[^\\x01-\\x7f]')`;+ sql += ` WHERE id IN (+ SELECT id FROM ${table} WHERE lower(json->>$2) = ANY($3::text[])+ UNION+ SELECT id FROM ${table} WHERE json->>$2 ~ '[^\\x01-\\x7f]')`;}if (query.afterId !== undefined) {params.push(query.afterId);@@ -314,11 +351,15 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {}export interface PostgresArtifactMaps {- map<T>(table: string): DurableMap<T>;+ map<T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T>;pool: PgPool;}export function createPostgresMapFactory(connectionString: string): PostgresArtifactMaps {const pg = createPgPool(connectionString);- return { map: <T>(table: string): DurableMap<T> => createPostgresMap<T>(pg, table), pool: pg };+ return {+ map: <T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T> =>+ createPostgresMap<T>(pg, table, indexedFields),+ pool: pg,+ };}diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex f25f2456e..93c7e9b53 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -190,6 +190,8 @@ function rowToSpendRow(r: Record<string, unknown>): SpendRow {const LAST_ACTIVITY_DEBOUNCE_MS = 60_000;const SEARCH_TIMEOUT_MS = 10_000;+const SPEND_INDEXABLE = `(spend_usage_json(usage_json) IS NOT NULL+ AND octet_length(session_id) + octet_length(model) + octet_length(usage_json) <= 2000)`;export function createPostgresSessionStore(connectionString: string, opts: StoreOptions = {}): SessionStore {const now = opts.now ?? (() => Date.now());@@ -206,14 +208,13 @@ export function createPostgresSessionStore(connectionString: string, opts: StoreOR (${participant}.valid_to_seq IS NULL AND (${participant}.valid_to IS NULL OR ${entry}.created_at < ${participant}.valid_to))))`;const participantSessionsSql = (extraWhere: string): string =>`SELECT s.*, p.title AS p_title, p.archived AS p_archived, p.pinned AS p_pinned, p.color AS p_color,- COALESCE(MAX(e.created_at), s.created_at) AS user_last_activity,+ COALESCE((SELECT MAX(e.created_at) FROM session_entries e+ WHERE e.session_id = s.id AND e.type = 'user'), s.created_at) AS user_last_activity,EXISTS (SELECT 1 FROM session_entries x WHERE x.session_id = s.idAND ${withinParticipantWindow("x", "p")}) AS has_entriesFROM sessions sJOIN participants p ON p.session_id = s.id- LEFT JOIN session_entries e ON e.session_id = s.id AND e.type = 'user'- WHERE p.principal_id = $1${extraWhere}- GROUP BY s.id, p.title, p.archived, p.pinned, p.color, p.valid_from, p.valid_to, p.valid_from_seq, p.valid_to_seq`;+ WHERE p.principal_id = $1${extraWhere}`;const participantSessions = async (principalId: string, opts?: { limit: number }): Promise<Session[]> => {const limit = opts ? Math.max(0, Math.floor(opts.limit)) : undefined;const rows = await q(@@ -595,6 +596,61 @@ export function createPostgresSessionStore(connectionString: string, opts: StoreON session_llm_requests(created_at)`,],},+ {+ id: "sessions/store/0019-participant-activity-indexes",+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS participants_by_principal+ ON participants(principal_id, session_id)`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_entries_user_activity+ ON session_entries(session_id, created_at DESC) WHERE type = 'user'`,+ ],+ },+ {+ id: "sessions/store/0020-spend-usage-json",+ statements: [+ `CREATE OR REPLACE FUNCTION spend_usage_json(t text) RETURNS jsonb+ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $spend_usage_json$+ SELECT CASE WHEN pg_input_is_valid(j ->> 'costUsd', 'double precision') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'input', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'output', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'cacheRead', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'cacheWrite', 'bigint') IS NOT FALSE THEN j END+ FROM (SELECT CASE WHEN pg_input_is_valid(t, 'jsonb') THEN t::jsonb END AS j OFFSET 0) parsed+ $spend_usage_json$`,+ ],+ },+ {+ id: "sessions/store/0020-spend-usage-json-size",+ statements: [+ `CREATE OR REPLACE FUNCTION spend_usage_json(t text) RETURNS jsonb+ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $spend_usage_json$+ SELECT CASE WHEN pg_input_is_valid(j ->> 'costUsd', 'double precision') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'input', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'output', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'cacheRead', 'bigint') IS NOT FALSE+ AND pg_input_is_valid(j ->> 'cacheWrite', 'bigint') IS NOT FALSE THEN j END+ FROM (SELECT CASE WHEN octet_length(t) > 2000 THEN NULL+ WHEN pg_input_is_valid(t, 'jsonb') THEN t::jsonb END AS j OFFSET 0) parsed+ $spend_usage_json$`,+ ],+ },+ {+ id: "sessions/store/0021-spend-covering-index",+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_llm_requests_spend+ ON session_llm_requests(created_at,+ ((spend_usage_json(usage_json) ->> 'costUsd')::double precision),+ ((spend_usage_json(usage_json) ->> 'input')::bigint),+ ((spend_usage_json(usage_json) ->> 'output')::bigint),+ ((spend_usage_json(usage_json) ->> 'cacheRead')::bigint),+ ((spend_usage_json(usage_json) ->> 'cacheWrite')::bigint))+ INCLUDE (session_id, model, usage_json)+ WHERE usage_json IS NOT NULL AND ${SPEND_INDEXABLE}`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_llm_requests_spend_wide+ ON session_llm_requests(created_at)+ WHERE usage_json IS NOT NULL AND NOT ${SPEND_INDEXABLE}`,+ ],+ },],[{@@ -1632,12 +1688,13 @@ export function createPostgresSessionStore(connectionString: string, opts: Storeasync spendRollup(range): Promise<SpendRow[]> {const rows = await q(- `WITH RECURSIVE spending AS (- SELECT DISTINCT session_id FROM session_llm_requests- WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL- ), ancestry AS (+ `WITH RECURSIVE ancestry AS (SELECT s.id AS session_id, s.parent_session_id, ${originExpr("s")} AS origin, ARRAY[s.id] AS path- FROM sessions s JOIN spending r ON r.session_id = s.id+ FROM sessions s+ WHERE s.parent_session_id IS NOT NULL AND ${originExpr("s")} = 'conversation'+ AND EXISTS (SELECT 1 FROM session_llm_requests r+ WHERE r.session_id = s.id AND r.created_at >= $1 AND r.created_at < $2+ AND r.usage_json IS NOT NULL)UNION ALLSELECT a.session_id, p.parent_session_id, ${originExpr("p")}, a.path || p.idFROM ancestry a JOIN sessions p ON p.id = a.parent_session_id@@ -1645,30 +1702,41 @@ export function createPostgresSessionStore(connectionString: string, opts: Store), origins AS (SELECT DISTINCT ON (session_id) session_id, originFROM ancestry ORDER BY session_id, cardinality(path) DESC- )- SELECT day, scope_id, origin, model,+ ), rollup AS MATERIALIZED (+ SELECT (r.created_at / 86400000)::bigint AS day, s.scope_id,+ COALESCE(o.origin, ${originExpr("s")}) AS origin, r.model,COUNT(*) AS calls,COALESCE(SUM(cost_usd), 0) AS cost_usd,COALESCE(SUM(input), 0) AS input,COALESCE(SUM(output), 0) AS output,COALESCE(SUM(cache_read), 0) AS cache_read,COALESCE(SUM(cache_write), 0) AS cache_write- FROM (SELECT (r.created_at / 86400000)::bigint AS day,- s.scope_id,- r.model,- o.origin,- (u.j ->> 'costUsd')::double precision AS cost_usd,- (u.j ->> 'input')::bigint AS input,- (u.j ->> 'output')::bigint AS output,- (u.j ->> 'cacheRead')::bigint AS cache_read,- (u.j ->> 'cacheWrite')::bigint AS cache_write- FROM session_llm_requests r- JOIN sessions s ON s.id = r.session_id- JOIN origins o ON o.session_id = s.id- CROSS JOIN LATERAL (SELECT r.usage_json::jsonb AS j) u- WHERE r.created_at >= $1 AND r.created_at < $2- AND r.usage_json IS NOT NULL) t- GROUP BY day, scope_id, origin, model`,+ FROM (SELECT created_at, session_id, model,+ (spend_usage_json(usage_json) ->> 'costUsd')::double precision AS cost_usd,+ (spend_usage_json(usage_json) ->> 'input')::bigint AS input,+ (spend_usage_json(usage_json) ->> 'output')::bigint AS output,+ (spend_usage_json(usage_json) ->> 'cacheRead')::bigint AS cache_read,+ (spend_usage_json(usage_json) ->> 'cacheWrite')::bigint AS cache_write+ FROM session_llm_requests+ WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL+ AND ${SPEND_INDEXABLE}+ UNION ALL+ SELECT created_at, session_id, model,+ (usage_json::jsonb ->> 'costUsd')::double precision AS cost_usd,+ (usage_json::jsonb ->> 'input')::bigint AS input,+ (usage_json::jsonb ->> 'output')::bigint AS output,+ (usage_json::jsonb ->> 'cacheRead')::bigint AS cache_read,+ (usage_json::jsonb ->> 'cacheWrite')::bigint AS cache_write+ FROM session_llm_requests+ WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL+ AND NOT ${SPEND_INDEXABLE}+ AND EXISTS (SELECT 1 FROM sessions WHERE id = session_llm_requests.session_id)+ OFFSET 0) r+ JOIN sessions s ON s.id = r.session_id+ LEFT JOIN origins o ON o.session_id = s.id+ GROUP BY day, s.scope_id, COALESCE(o.origin, ${originExpr("s")}), r.model+ )+ SELECT * FROM rollup ORDER BY day, scope_id, origin, model`,[range.from, range.to],);return rows.map(rowToSpendRow);diff --git a/src/slack/deliveries.ts b/src/slack/deliveries.tsindex 7e6926e0d..9eeff21e4 100644--- a/src/slack/deliveries.ts+++ b/src/slack/deliveries.ts@@ -1,5 +1,5 @@import { deployAccessMessage } from "./deploy-access.ts";-import { approvalDeliveryKey } from "../core/approval-store.ts";+import { approvalDeliveryKey, approvalDeliveryRecipient } from "../core/approval-store.ts";import { samePerson } from "../directory/person.ts";import { approvalMessage } from "./approval-cards.ts";import { keychainApprovalMessage, keychainApprovalOrigin } from "./keychain-approvals.ts";@@ -374,12 +374,14 @@ export function createDeliveryPoller(deps: {const commandApproval = d.destination.commandApprovalId? await core.getApproval(d.destination.commandApprovalId): null;- const requester = commandApproval?.request?.actor as { externalId?: string } | undefined;+ const requester = approvalDeliveryRecipient(+ commandApproval?.request?.actor as { externalId?: string } | undefined,+ );if (d.destination.commandApprovalId &&(!commandApproval ||- !requester?.externalId ||- !samePerson(requester.externalId, d.destination.target) ||+ !requester ||+ !samePerson(requester, d.destination.target) ||d.idempotencyKey !== approvalDeliveryKey(d.destination.commandApprovalId, commandApproval)))return undefined;diff --git a/src/wiring.ts b/src/wiring.tsindex 6700da051..7825a0ff3 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -606,8 +606,8 @@ export function buildApp(membership.managesArtifactHome!(scopeId, authoredBy ?? "", principalId),});const pgArtifactMap = config.databaseUrl ? createPostgresMapFactory(config.databaseUrl) : null;- const artifactMap = <T>(table: string): DurableMap<T> =>- pgArtifactMap ? pgArtifactMap.map<T>(table) : createMemoryMap<T>();+ const artifactMap = <T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T> =>+ pgArtifactMap ? pgArtifactMap.map<T>(table, indexedFields) : createMemoryMap<T>();setProviderBaseUrls(config.providerBaseUrls);const unknownGatewayModels = Object.keys(config.modelGateway?.models ?? {}).filter((id) => !resolveModel(id));if (unknownGatewayModels.length) {@@ -1201,8 +1201,8 @@ export function buildApp(...(legacyCredentialKey ? { fallbacks: [legacyCredentialKey] } : {}),};const credentialStore: Keychain = createKeychain({- creds: artifactMap<KeychainCredential>("keychain_credentials"),- grants: artifactMap<KeychainGrant>("keychain_grants"),+ creds: artifactMap<KeychainCredential>("keychain_credentials", ["ownerId"]),+ grants: artifactMap<KeychainGrant>("keychain_grants", ["ownerId"]),asks: artifactMap<KeychainAsk>("keychain_asks"),key: credentialKey,lock: advisoryLock,@@ -1490,7 +1490,7 @@ export function buildApp(config.swarmsEnabled !== false && config.sessionStore === swarmStoreKind && runStoreKind === swarmStoreKind? createSwarmService({defaults: config.swarmDefaults,- store: createSwarmStore(artifactMap<SwarmStorage>("swarms"), {+ store: createSwarmStore(artifactMap<SwarmStorage>("swarms", ["pending"]), {runs,sessions,...(pgArtifactMap && runStoreKind === "postgres" ? { pg: pgArtifactMap.pool } : {}),@@ -1739,7 +1739,7 @@ export function buildApp(: createMemoryEnvironmentStore();const monitors = createMonitorStore(artifactMap<Monitor>("monitors"));const loopStore = createLoopStore(artifactMap<Loop>("loops"));- const loopItemsMap = artifactMap<LoopItem>("loop_items");+ const loopItemsMap = artifactMap<LoopItem>("loop_items", ["loopId"]);const loopOwnerCache = new Map<string, string>();const publishLoopEvent = (event: import("./loops/ledger-events.ts").LedgerEvent): void => {void (async () => {@@ -1758,7 +1758,7 @@ export function buildApp(return Boolean(loop && (loop.surface !== "inbox" || loop.state === "enabled"));},});- const loopOutputs = createLoopOutputStore(artifactMap<LoopOutput>("loop_outputs"), (output) =>+ const loopOutputs = createLoopOutputStore(artifactMap<LoopOutput>("loop_outputs", ["itemId"]), (output) =>publishLoopEvent({ loopId: output.loopId, itemId: output.itemId, op: "ready", at: Date.now() }),);const loopGrants = createShipGrantStore(artifactMap<ShipGrant>("loop_ship_grants"));@@ -1838,7 +1838,7 @@ export function buildApp(conversation: { ...request.conversation, audience, publishMembers: audience },};};- const sessionMailbox = createSessionMailbox(artifactMap<SessionMessage>("session_mailbox"));+ const sessionMailbox = createSessionMailbox(artifactMap<SessionMessage>("session_mailbox", ["recipientId"]));const sessionSyscalls = createSessionSyscalls({mailbox: sessionMailbox,enabled: async (actorId) =>@@ -2353,8 +2353,8 @@ export function buildApp(});const loopIngress = createLoopIngress({enabledFor: (owner) => featureFlags.enabled("inbox_loops", scopeId("personal", owner)),- sources: artifactMap<LoopIngress>("loop_ingress"),- deliveries: artifactMap<IngressDelivery>("loop_ingress_deliveries"),+ sources: artifactMap<LoopIngress>("loop_ingress", ["loopId"]),+ deliveries: artifactMap<IngressDelivery>("loop_ingress_deliveries", ["queueKey"]),loops: loopStore,items: loopItems,outputs: loopOutputs,diff --git a/test/approval-store.test.ts b/test/approval-store.test.tsindex 12f31356b..46cf3b0d9 100644--- a/test/approval-store.test.ts+++ b/test/approval-store.test.ts@@ -66,3 +66,29 @@ test("web approvals remain on their native surface", async () => {await approvals.deliverPending();assert.equal((await deliveries.pending("principal")).length, 0);});++test("system approvals stay pending without enqueuing or reviving an impossible Slack DM", async () => {+ const backing = createMemoryMap<PendingApprovalRecord>();+ const deliveries = createDeliveryStore({ maxAgeMs: 0 });+ const approvals = createApprovalStore(backing, deliveries);+ const approval = record("slack:C1:ambient:1.0");+ approval.request!.actor.externalId = "system:ambient:acme";+ const stale = await deliveries.enqueue({+ destination: { type: "principal", target: "system:ambient:acme", commandApprovalId: "A1" },+ text: "Approval needed: publish",+ idempotencyKey: "command-approval:A1:1",+ });+ await deliveries.claimPending("principal", 1);+ const expiredAt = (await deliveries.get(stale.id))?.expiredAt;+ assert.ok(expiredAt);++ await approvals.put("A1", approval);+ await approvals.put("A2", approval);+ await approvals.deliverPending();+ await createApprovalStore(backing, deliveries).deliverPending();++ assert.deepEqual(await backing.get("A1"), approval, "the protected action still requires approval");+ assert.deepEqual(await backing.get("A2"), approval);+ assert.deepEqual(await deliveries.pending("principal"), []);+ assert.equal((await deliveries.get(stale.id))?.expiredAt, expiredAt, "a sweep must not revive the stale DM");+});diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.tsindex 069033b45..6afeeb2ea 100644--- a/test/cron-scheduler.test.ts+++ b/test/cron-scheduler.test.ts@@ -13,6 +13,8 @@ import { isPollSurface, isSilentPollReply } from "../src/triggers/run-trigger.tsimport { createDirectoryStore, type DirectoryStore } from "../src/directory/directory-store.ts";import { createMemoryMap } from "../src/persistence/durable-map.ts";import type { Cron } from "../src/types.ts";+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";+import type { CronFireJob, CronJobQueue } from "../src/cron/job-queue.ts";function fakeLease(isLeader: () => boolean): LeaderLease {return {@@ -1024,6 +1026,193 @@ test("queue mode: fires claim the slot before running, and stale or lost claimsscheduler.stop();});+for (const queued of [false, true]) {+ test(`${queued ? "queue" : "interval"} mode: a busy cron defers without blocking other crons or losing its slot`, async () => {+ const crons = createCronStore();+ const gate = Promise.withResolvers<TurnResult>();+ const started = Promise.withResolvers<void>();+ const calls: TurnRequest[] = [];+ const enqueued: CronFireJob[] = [];+ let clock = 1000;+ let onFire: ((job: CronFireJob) => Promise<void>) | undefined;+ const jobQueue: CronJobQueue = {+ async start(handlers) {+ onFire = handlers.onFire;+ },+ async enqueueFire(job) {+ enqueued.push(job);+ },+ healthy: () => true,+ async stop() {},+ };+ const scheduler = createScheduler({+ crons,+ deliveries: createDeliveryStore(),+ idempotency: createIdempotencyStore(),+ identity: createIdentityService(),+ lock: createMemoryAdvisoryLock(),+ now: () => clock,+ ...(queued ? { jobQueue } : {}),+ run: async (req) => {+ calls.push(req);+ if (calls.length === 1) {+ started.resolve();+ return gate.promise;+ }+ return { status: "ok", reply: "done" };+ },+ });+ const cron = await crons.create({+ schedule: { everyMs: 1000, firstFireAt: 1000 },+ action: "slow recurring job",+ owner: "U1",+ createdBy: "U1",+ ownerScopeId: scopeId("personal", "U1"),+ });+ if (queued) {+ scheduler.start(1000);+ await scheduler.ready();+ }+ let first: Promise<void>;+ if (queued) first = onFire!({ cronId: cron.id, scheduledAt: 1000 });+ else {+ const manual = await scheduler.runNow(cron.id);+ assert.ok(manual.started);+ first = manual.settled;+ }+ await started.promise;+ clock = 2000;+ const other = await crons.create({+ schedule: { firstFireAt: clock },+ message: "unrelated reminder",+ owner: "U1",+ createdBy: "U1",+ ownerScopeId: scopeId("personal", "U1"),+ });+ const owedSlot = (await crons.get(cron.id))!.nextFireAt!;+ let deferred = false;+ const second = (queued ? onFire!({ cronId: cron.id, scheduledAt: owedSlot }) : scheduler.tick(clock)).then(+ () => (deferred = true),+ );+ try {+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(deferred, true, "an active fire must not make the scheduler wait for its lifecycle lock");+ assert.equal(calls.length, 1, "the busy cron did not start another agent");+ const pending = (await crons.get(cron.id))!;+ assert.equal(pending.nextFireAt, owedSlot, "the unstarted slot remains owed");+ assert.equal(pending.deferUntil, 32_000, "the retry is durable and delayed");+ if (queued) {+ assert.deepEqual(enqueued.at(-1), { cronId: cron.id, scheduledAt: owedSlot, notBefore: 32_000 });+ await onFire!({ cronId: other.id, scheduledAt: 2000 });+ }+ assert.equal((await crons.get(other.id))!.enabled, false, "an unrelated cron can finish while this one runs");+ gate.resolve({ status: "ok", reply: "done" });+ await first;+ clock = 32_000;+ if (queued) await onFire!({ cronId: cron.id, scheduledAt: owedSlot });+ else await scheduler.tick(clock);+ assert.equal(calls.length, 2, "the deferred slot runs once after the active fire completes");+ assert.equal((await crons.listFires(cron.id)).total, 2, "no phantom running fire was created for the deferral");+ } finally {+ gate.resolve({ status: "ok", reply: "done" });+ await Promise.all([first, second]);+ await scheduler.stop();+ }+ });+}++test("queue mode: concurrent reconciliation cannot consume an overdue slot while its lifecycle lock is held", async () => {+ const crons = createCronStore();+ const cron = await crons.create({+ schedule: { cron: "* * * * *", timezone: "UTC" },+ action: "overdue calendar task",+ owner: "U1",+ createdBy: "U1",+ ownerScopeId: scopeId("personal", "U1"),+ });+ const slot = cron.nextFireAt!;+ let clock = slot + 120_000;+ const mutex = createMemoryAdvisoryLock();+ const releaseHeld = Promise.withResolvers<void>();+ const heldEntered = Promise.withResolvers<void>();+ const holding = mutex.withLock(`cron-lifecycle:${cron.id}`, async () => {+ heldEntered.resolve();+ await releaseHeld.promise;+ });+ await heldEntered.promise;+ const gates = [0, 1].map(() => ({ entered: Promise.withResolvers<void>(), release: Promise.withResolvers<void>() }));+ let attempts = 0;+ let onFire!: (job: CronFireJob) => Promise<void>;+ let onTick!: () => Promise<void>;+ const enqueued: CronFireJob[] = [];+ const calls: TurnRequest[] = [];+ const scheduler = createScheduler({+ crons,+ deliveries: createDeliveryStore(),+ idempotency: createIdempotencyStore(),+ identity: createIdentityService(),+ now: () => clock,+ lock: {+ ...mutex,+ async tryWithLock(key, fn) {+ const gate = gates[attempts++];+ if (gate) {+ gate.entered.resolve();+ await gate.release.promise;+ }+ return mutex.tryWithLock!(key, fn);+ },+ },+ run: async (req) => {+ calls.push(req);+ return { status: "ok", reply: "done" };+ },+ jobQueue: {+ async start(handlers) {+ onFire = handlers.onFire;+ onTick = handlers.onTick;+ },+ async enqueueFire(job) {+ enqueued.push(job);+ },+ healthy: () => true,+ async stop() {},+ },+ });+ scheduler.start(1000);+ await scheduler.ready();+ const first = onFire({ cronId: cron.id, scheduledAt: slot });+ let second: Promise<void> | undefined;+ try {+ await gates[0]!.entered.promise;+ await onTick();+ clock++;+ second = onFire(enqueued.at(-1)!);+ await gates[1]!.entered.promise;+ gates[0]!.release.resolve();+ await first;+ gates[1]!.release.resolve();+ await second;+ const deferred = (await crons.get(cron.id))!;+ assert.equal(deferred.nextFireAt, slot, "neither claimant can advance past an unstarted calendar slot");+ assert.equal(deferred.lastFiredAt, undefined);+ assert.equal(deferred.deferUntil, clock + 30_000);+ assert.equal(calls.length, 0);+ releaseHeld.resolve();+ await holding;+ clock = deferred.deferUntil!;+ await onFire({ cronId: cron.id, scheduledAt: slot });+ assert.equal(calls.length, 1);+ assert.equal(calls[0]!.idempotencyKey, `cron:${cron.id}:${slot}`);+ assert.equal((await crons.get(cron.id))!.nextFireAt, slot + 60_000);+ } finally {+ for (const gate of gates) gate.release.resolve();+ releaseHeld.resolve();+ await Promise.all([holding, first, second]);+ await scheduler.stop();+ }+});+test("queue mode: while the queue runs, the interval scheduler's leader lease is held as a guard", async (t) => {t.mock.timers.enable({ apis: ["setInterval"] });const heldKeys: string[] = [];diff --git a/test/durable-stores.test.ts b/test/durable-stores.test.tsindex 1226897df..a9e26b709 100644--- a/test/durable-stores.test.ts+++ b/test/durable-stores.test.ts@@ -179,7 +179,7 @@ test("the Postgres map select projects and filters inside SQL, never fetching thassert.equal(reads.length, 1);assert.match(reads[0]!.sql,- /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE \(lower\(json->>\$2\) = ANY\(\$3::text\[\]\) OR json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,+ /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>\$2\) = ANY\(\$3::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,);assert.deepEqual(reads[0]!.params, [["secretEnc"], "owner", ["u1", "[email protected]"]]);diff --git a/test/postgres-map.test.ts b/test/postgres-map.test.tsindex e7af74dfe..a124cd95b 100644--- a/test/postgres-map.test.ts+++ b/test/postgres-map.test.ts@@ -1,5 +1,6 @@import { test, before } from "node:test";import assert from "node:assert/strict";+import { randomBytes } from "node:crypto";import { createMemoryMap, createPostgresMapFactory } from "../src/persistence/durable-map.ts";import { createCronStore } from "../src/cron/cron-store.ts";import { createWebhookStore, type WebhookHistory } from "../src/webhooks/webhook-store.ts";@@ -12,6 +13,7 @@ import {type KeychainGrant,} from "../src/credentials/keychain.ts";import { deriveConnectorKey } from "../src/connectors/connector-client-store.ts";+import { applyPgMigrations, definePgMigration } from "../src/persistence/pg-pool.ts";const URL = process.env.DATABASE_URL;const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres map tests";@@ -22,7 +24,7 @@ before(async () => {const p = new pg.Pool({ connectionString: URL });await p.query("DROP TABLE IF EXISTS qm_schema_migrations CASCADE");await p.query(- "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",+ "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",);await p.end();});@@ -103,6 +105,7 @@ test("pg map: rejects an unsafe table name (the DDL/DML interpolation guard)", (const factory = createPostgresMapFactory("postgres://unused");assert.throws(() => factory.map("bad name"), /invalid table name/);assert.throws(() => factory.map("crons;--"), /invalid table name/);+ assert.throws(() => factory.map<{ owner: string }>("map_widgets", ["bad field" as "owner"]), /invalid index field/);});test(@@ -179,12 +182,13 @@ test("pg map: select mirrors the memory map — folded field filter, projection,["sel-c", { name: "sel-c", tags: [], nested: { n: 3 }, owner: "[email protected]", secretEnc: "enc-c" }],];const factory = createPostgresMapFactory(URL!);- const pgMap = factory.map<Owned>("map_widgets");+ let pgMap = factory.map<Owned>("map_widgets");const memMap = createMemoryMap<Owned>();for (const [id, row] of rows) {await pgMap.put(id, row);await memMap.put(id, row);}+ pgMap = factory.map<Owned>("map_widgets", ["owner"]);try {const mine = await pgMap.select({ omit: ["secretEnc"], where: { field: "owner", anyOfFold: ["U7"] } });assert.deepEqual(mine, await memMap.select({ omit: ["secretEnc"], where: { field: "owner", anyOfFold: ["U7"] } }));@@ -221,22 +225,92 @@ test("pg map: select mirrors the memory map — folded field filter, projection,await memMap.put("sel-d", turkish);const swept = await pgMap.select({ where: { field: "owner", anyOfFold: ["no-such-owner"] } });assert.deepEqual(swept, await memMap.select({ where: { field: "owner", anyOfFold: ["no-such-owner"] } }));+ const indexes = await factory.pool.q(+ "SELECT indexrelid::regclass::text AS name, indisvalid FROM pg_index WHERE indrelid = 'map_widgets'::regclass",+ );+ for (const name of ["map_widgets_owner_fold", "map_widgets_owner_unicode"]) {+ assert.ok(indexes.some((index) => index.name === name && index.indisvalid));+ }+ await pgMap.merge("sel-a", { owner: "Moved" });+ assert.deepEqual(+ (await pgMap.select({ where: { field: "owner", anyOfFold: ["MOVED"] } })).map((w) => w.name),+ ["sel-a", "sel-d"],+ );assert.deepEqual(swept.map((w) => w.name),["sel-d"],"a non-ASCII field value is always a candidate — SQL lower() and JS toLowerCase() disagree there",);+ await pgMap.merge("sel-d", { owner: "josé" });+ assert.deepEqual(+ (await pgMap.select({ where: { field: "owner", anyOfFold: ["josé"] } })).map((row) => row.name),+ ["sel-d"],+ "a row matching both indexed branches is returned once",+ );} finally {for (const id of [...rows.map(([id]) => id), "sel-d"]) await pgMap.delete(id);await factory.pool.close();}});+test("pg map: indexed fields accept wide values through migration and later writes", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const first = randomBytes(3000).toString("hex");+ const second = randomBytes(3000).toString("hex");+ const unindexed = factory.map<{ owner: string }>("map_wide_widgets");+ try {+ await unindexed.put("existing", { owner: first });+ const indexed = factory.map<{ owner: string }>("map_wide_widgets", ["owner"]);+ const select = (owner: string) => indexed.select({ where: { field: "owner", anyOfFold: [owner.toUpperCase()] } });+ assert.deepEqual(await select(first), [{ owner: first }]);+ await indexed.put("later", { owner: second });+ await indexed.merge("existing", { owner: second });+ assert.deepEqual(await select(first), []);+ assert.deepEqual(await select(second), [{ owner: second }, { owner: second }]);+ } finally {+ await unindexed.delete("existing");+ await unindexed.delete("later");+ await factory.pool.close();+ }+});++test("pg map: original folded-index migration upgrades without changing its ledger", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const unindexed = factory.map<{ owner: string }>("map_wide_widgets");+ await unindexed.put("legacy", { owner: "short" });+ const legacy = definePgMigration("durable-map/map_wide_widgets/select-owner", [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS map_wide_widgets_owner_fold+ ON map_wide_widgets (lower(json->>'owner'))`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS map_wide_widgets_owner_unicode+ ON map_wide_widgets (id) WHERE json->>'owner' ~ '[^\\x01-\\x7f]'`,+ ]);+ try {+ await factory.pool.q("DROP INDEX IF EXISTS map_wide_widgets_owner_fold, map_wide_widgets_owner_unicode");+ await factory.pool.q(+ "DELETE FROM qm_schema_migrations WHERE id LIKE 'durable-map/map_wide_widgets/%' AND id <> 'durable-map/map_wide_widgets/0001'",+ );+ await applyPgMigrations(await factory.pool.pool(), [legacy]);+ const ledger = await factory.pool.q("SELECT * FROM qm_schema_migrations WHERE id = $1", [legacy.id]);+ const indexed = factory.map<{ owner: string }>("map_wide_widgets", ["owner"]);+ const owner = randomBytes(3000).toString("hex");+ await indexed.put("legacy", { owner });+ assert.deepEqual(await indexed.select({ where: { field: "owner", anyOfFold: [owner] } }), [{ owner }]);+ assert.deepEqual(await factory.pool.q("SELECT * FROM qm_schema_migrations WHERE id = $1", [legacy.id]), ledger);+ const [index] = await factory.pool.q(+ "SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid = c.relam WHERE c.oid = 'map_wide_widgets_owner_fold'::regclass",+ );+ assert.equal(index!.amname, "hash");+ } finally {+ await unindexed.delete("legacy");+ await factory.pool.close();+ }+});+test("pg keychain: listByOwner is a per-owner projected read with no secret material", { skip }, async () => {const factory = createPostgresMapFactory(URL!);const keychain = createKeychain({- creds: factory.map<KeychainCredential>("map_keychain_creds"),- grants: factory.map<KeychainGrant>("map_keychain_grants"),+ creds: factory.map<KeychainCredential>("map_keychain_creds", ["ownerId"]),+ grants: factory.map<KeychainGrant>("map_keychain_grants", ["ownerId"]),asks: factory.map<KeychainAsk>("map_keychain_asks"),key: deriveConnectorKey("postgres-keychain-test-key"),});@@ -272,8 +346,8 @@ test("pg map: concurrent keychain instances claim a once grant exactly once", {const key = deriveConnectorKey("postgres-keychain-test-key");const build = (factory: ReturnType<typeof createPostgresMapFactory>) =>createKeychain({- creds: factory.map<KeychainCredential>("map_keychain_creds"),- grants: factory.map<KeychainGrant>("map_keychain_grants"),+ creds: factory.map<KeychainCredential>("map_keychain_creds", ["ownerId"]),+ grants: factory.map<KeychainGrant>("map_keychain_grants", ["ownerId"]),asks: factory.map<KeychainAsk>("map_keychain_asks"),key,});diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex b6b70fd16..9a0f04fbf 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -4,7 +4,7 @@ import { migrateTranscriptPage } from "../scripts/lib/transcript-tape-migration.import { test, before } from "node:test";import { randomUUID } from "node:crypto";import assert from "node:assert/strict";-import { migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";+import { applyPgMigrations, migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";import { PARALLEL_EXCEPTION_QUERY } from "../src/deployment/postdeploy-smoke.ts";import {backfillSessionOriginBatch,@@ -73,10 +73,224 @@ test("pg session store: getForParticipant returns exactly the row listByParticipawait assertParticipantSessionParity(createPostgresSessionStore(URL!), `pg-parity-${randomUUID()}`);});+test("pg participant activity uses the latest user entry, including overheard entries", { skip }, async () => {+ let at = 1_000;+ const store = createPostgresSessionStore(URL!, { now: () => at });+ const owner = `activity-${randomUUID()}`;+ const scope = scopeId("personal", owner);+ const session = await store.getOrCreateByThread(owner, "dm", scope);+ await store.addParticipant(session.id, owner);+ const { lease } = await store.acquireLease(session.id);+ assert.ok(lease);+ at = 300_000;+ await store.append(lease, { type: "user", payload: { text: "overheard", overheard: true }, scopeLabel: scope });+ at = 500_000;+ await store.append(lease, { type: "assistant", payload: { text: "later reply" }, scopeLabel: scope });+ await store.releaseLease(lease);+ at = 400_000;+ const empty = await store.getOrCreateByThread(`${owner}-empty`, "dm", scope);+ await store.addParticipant(empty.id, owner);+ const listed = await store.listByParticipant(owner);+ assert.equal(listed.find((row) => row.id === session.id)?.lastActivityAt, 300_000);+ assert.equal(listed.find((row) => row.id === empty.id)?.lastActivityAt, 400_000);+ assert.equal((await store.getForParticipant(session.id, owner))?.lastActivityAt, 300_000);+ assert.deepEqual(+ (await store.listByParticipant(owner, { limit: 1 })).map((row) => row.id),+ [session.id],+ );+ assert.deepEqual(await store.listByParticipant(owner, { limit: 0 }), []);+});+test("pg session store: spendRollup matches the memory rollup row for row", { skip }, async () => {await assertSpendRollupParity((now) => createPostgresSessionStore(URL!, { now }), `pg-spend-${randomUUID()}`);});+test("pg spend indexes preserve legacy and unusual requests through migration and later writes", { skip }, async () => {+ const at = Date.UTC(2022, 0, 1);+ const store = createPostgresSessionStore(URL!, { now: () => at });+ const session = await store.getOrCreateByThread("spend-index", "dm", scopeId("personal", "USPENDINDEX"));+ const pg = (await import("pg")).default;+ const raw = new pg.Pool({ connectionString: URL });+ const usage = { input: 10, output: 2, cacheRead: 3, cacheWrite: 4, totalTokens: 19, costUsd: 0.5 };+ const wideModel = Array.from({ length: 100 }, () => randomUUID()).join("");+ const wideUsage = { ...usage, legacyDetail: wideModel };+ const deeplyNestedUsage =+ JSON.stringify(usage).slice(0, -1) + ',"detail":' + "[".repeat(20_000) + "0" + "]".repeat(20_000) + "}";+ const insert = async (id: string, model: string, json: string | null, when = at, sessionId = session.id) => {+ await raw.query(+ "INSERT INTO session_llm_requests(id, session_id, step, model, scope_label, created_at, usage_json) VALUES ($1,$2,0,$3,$4,$5,$6)",+ [id, sessionId, model, session.scopeId, when, json],+ );+ };+ const range = { from: at, to: at + 1 };+ try {+ await raw.query("DROP INDEX session_llm_requests_spend, session_llm_requests_spend_wide");+ await raw.query(+ "DELETE FROM qm_schema_migrations WHERE id IN ('sessions/store/0020-spend-usage-json', 'sessions/store/0020-spend-usage-json-size', 'sessions/store/0021-spend-covering-index')",+ );+ await insert("spend-index-normal", "normal", JSON.stringify(usage));+ await insert("spend-index-wide", wideModel, JSON.stringify(wideUsage));+ await insert("spend-index-null", "null", null);+ await insert("spend-index-empty", "empty", "{}");+ await insert("spend-index-bad-history", "bad-history", "not json", at - 1);+ await insert("spend-index-bad-orphan", "bad-orphan", "not json", at, "missing-spend-session");+ await insert(+ "spend-index-fraction-orphan",+ "fraction-orphan",+ JSON.stringify({ ...usage, input: 0.5 }),+ at,+ "missing-spend-session",+ );+ await insert("spend-index-unicode", "bad-unicode", JSON.stringify({ ...usage, detail: "\u0000" }), at - 1);+ await insert("spend-index-fraction", "fraction", JSON.stringify({ ...usage, input: 0.5 }), at - 1);+ await insert("spend-index-overflow", "overflow", JSON.stringify({ ...usage, input: 1e308 }), at - 1);+ await insert("spend-index-deep-history", "deep-history", deeplyNestedUsage, at - 2);+ await migrateRegisteredPgSchemas(URL!);+ const indexes = await raw.query(+ "SELECT indexrelid::regclass::text AS name, indisvalid, indisready FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass)",+ );+ assert.equal(indexes.rows.length, 2);+ assert.ok(indexes.rows.every((r) => r.indisvalid && r.indisready));+ const read = async () => (await store.spendRollup(range)).filter((r) => r.scopeId === session.scopeId);+ const before = await read();+ assert.equal(before.length, 3);+ assert.equal(+ before.reduce((sum, row) => sum + row.calls, 0),+ 3,+ );+ assert.equal(+ before.reduce((sum, row) => sum + row.costUsd, 0),+ 1,+ );+ await store.recordLlmRequest(session.id, {+ turnSeq: null,+ step: 0,+ model: wideModel,+ scopeLabel: session.scopeId,+ usage: wideUsage,+ });+ await store.recordLlmRequest(session.id, {+ turnSeq: null,+ step: 0,+ model: "fraction-after-index",+ scopeLabel: session.scopeId,+ usage: { ...usage, input: 0.5 },+ });+ await raw.query("UPDATE session_llm_requests SET created_at = $1 WHERE model = 'fraction-after-index'", [at - 1]);+ await insert("spend-index-bad-later", "bad-later", "not json", at - 1);+ await insert("spend-index-deep-later", "deep-later", deeplyNestedUsage, at - 2);+ await assert.rejects(store.spendRollup({ from: at - 2, to: at - 1 }), { code: "54001" });+ await raw.query("UPDATE session_llm_requests SET usage_json = $1 WHERE id = 'spend-index-normal'", [+ JSON.stringify({ ...usage, costUsd: 2 }),+ ]);+ assert.equal(+ (await read()).reduce((sum, row) => sum + row.costUsd, 0),+ 3,+ );+ await raw.query("UPDATE sessions SET scope_id = 'personal:USPENDMOVED' WHERE id = $1", [session.id]);+ const moved = (await store.spendRollup(range)).filter((r) => r.scopeId === "personal:USPENDMOVED");+ assert.equal(+ moved.reduce((sum, row) => sum + row.calls, 0),+ 4,+ );+ await raw.query("UPDATE session_llm_requests SET created_at = $1 WHERE id = 'spend-index-bad-later'", [at]);+ await assert.rejects(store.spendRollup(range), /invalid input syntax for type json/);+ } finally {+ await raw.query("DELETE FROM session_llm_requests WHERE session_id IN ($1, 'missing-spend-session')", [session.id]);+ await store.deleteSession(session.id);+ await raw.end();+ }+});++test("pg spend usage guard upgrades the original migration without rewriting indexes or ledger", { skip }, async () => {+ const pg = (await import("pg")).default;+ const admin = new pg.Pool({ connectionString: URL });+ const schema = `spend_upgrade_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new globalThis.URL(URL!);+ url.searchParams.set("options", `-c search_path=${schema}`);+ const raw = new pg.Pool({ connectionString: url.toString() });+ const store = createPostgresSessionStore(url.toString());+ const migrations = registeredPgMigrations(url.toString());+ const original = migrations.find((migration) => migration.id === "sessions/store/0020-spend-usage-json")!;+ try {+ assert.equal(original.checksum, "febf07cde8b9ffe09a497d44a0e524a5c994f7708a0fab6d245315e4ab1e717a");+ await applyPgMigrations(+ raw,+ migrations.filter((migration) => migration.id !== "sessions/store/0020-spend-usage-json-size"),+ );+ const ledger = (await raw.query("SELECT * FROM qm_schema_migrations ORDER BY id")).rows;+ const indexes = (+ await raw.query(+ "SELECT indexrelid FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass) ORDER BY indexrelid",+ )+ ).rows;+ await store.countSessions();+ assert.deepEqual(+ (+ await raw.query(+ "SELECT * FROM qm_schema_migrations WHERE id <> 'sessions/store/0020-spend-usage-json-size' ORDER BY id",+ )+ ).rows,+ ledger,+ );+ assert.deepEqual(+ (+ await raw.query(+ "SELECT indexrelid FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass) ORDER BY indexrelid",+ )+ ).rows,+ indexes,+ );+ const deep = "[".repeat(20_000) + "0" + "]".repeat(20_000);+ assert.equal((await raw.query("SELECT spend_usage_json($1) AS usage", [deep])).rows[0].usage, null);+ } finally {+ await raw.end();+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ await admin.end();+ }+});++test("pg spend ancestry only visits sessions with usage inside the requested range", { skip }, async (t) => {+ const at = Date.UTC(2023, 0, 1);+ const store = createPostgresSessionStore(URL!, { now: () => at });+ const scope = scopeId("personal", "USPENDANCESTRY");+ const parent = await store.getOrCreateByThread("cron:spend-ancestry", "dm", scope);+ const current = await store.getOrCreateByThread("spend-ancestry-current", "dm", scope);+ const historical = await store.getOrCreateByThread("spend-ancestry-historical", "dm", scope);+ await store.setParentSession(current.id, parent.id);+ await store.setParentSession(historical.id, parent.id);+ await store.recordLlmRequest(current.id, {+ turnSeq: null,+ step: 0,+ model: "ancestry",+ scopeLabel: scope,+ usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, costUsd: 1 },+ });+ const pg = (await import("pg")).default;+ const raw = new pg.Pool({ connectionString: URL });+ const execute = pg.Pool.prototype.query;+ let query = "";+ t.mock.method(pg.Pool.prototype, "query", function (this: InstanceType<typeof pg.Pool>, ...args: unknown[]) {+ if (typeof args[0] === "string" && args[0].startsWith("WITH RECURSIVE")) query = args[0];+ return Reflect.apply(execute, this, args);+ });+ try {+ assert.equal((await store.spendRollup({ from: at, to: at + 1 }))[0]!.origin, "cron");+ assert.ok(query);+ const explained = await raw.query(`EXPLAIN (ANALYZE, FORMAT JSON) ${query}`, [at, at + 1]);+ type Plan = { Plans?: Plan[]; "Subplan Name"?: string; "Actual Rows"?: number };+ const plans = (plan: Plan): Plan[] => [plan, ...(plan.Plans ?? []).flatMap(plans)];+ const ancestry = plans(explained.rows[0]["QUERY PLAN"][0].Plan).find(+ (plan) => plan["Subplan Name"] === "CTE ancestry",+ );+ assert.equal(ancestry?.["Actual Rows"], 2);+ } finally {+ for (const session of [current, historical, parent]) await store.deleteSession(session.id);+ await raw.end();+ }+});+test("pg session store: a bare failed acquire means the session is gone, not a lease race", { skip }, async () => {const s = createPostgresSessionStore(URL!);const scope = scopeId("personal", "U1");diff --git a/test/slack-deliveries.test.ts b/test/slack-deliveries.test.tsindex 4394be7ae..e02383cc1 100644--- a/test/slack-deliveries.test.ts+++ b/test/slack-deliveries.test.ts@@ -37,6 +37,7 @@ async function deliver(const marks: Array<{ channel: string; ts: string }> = [];const probes: Record<string, unknown>[] = [];const history = row.history ?? [];+ const conversationsOpened: Record<string, unknown>[] = [];const probe = async (args: Record<string, unknown>) => {probes.push(args);return {@@ -54,7 +55,14 @@ async function deliver(},};const client = {- conversations: { open: async () => ({ channel: { id: "C1" } }), replies: probe, history: probe },+ conversations: {+ open: async (args: Record<string, unknown>) => {+ conversationsOpened.push(args);+ return { channel: { id: "C1" } };+ },+ replies: probe,+ history: probe,+ },files: {uploadV2: async (args: Record<string, unknown>) => {uploads.push(args);@@ -84,7 +92,7 @@ async function deliver(});await poller.pollDeliveries(client);- return { acknowledgements, uploads, posts, marks, probes };+ return { acknowledgements, uploads, posts, marks, probes, conversationsOpened };}for (const type of ["slack", "group", "principal"]) {@@ -344,6 +352,22 @@ test("a stale queued approval cannot render a newer request for the same commandassert.deepEqual(out.acknowledgements, ["D1"]);});+test("a queued system approval notification is retired without opening a Slack DM", async () => {+ const actorId = "system:ambient:acme";+ const approval = { requestId: "A1", command: "publish", request: { actor: { externalId: actorId } } };+ const out = await deliver(+ { type: "principal", target: actorId, commandApprovalId: "A1" },+ undefined,+ undefined,+ "Approval needed",+ { approval },+ );+ assert.deepEqual(out.conversationsOpened, []);+ assert.deepEqual(out.posts, []);+ assert.deepEqual(out.uploads, []);+ assert.deepEqual(out.acknowledgements, ["D1"]);+});+for (const type of ["slack", "group", "principal"]) {test(`${type} file-only deliveries survive a restart after a lost acknowledgement`, async () => {const history: Record<string, unknown>[] = [];diff --git a/test/support/spend-rollup-parity.ts b/test/support/spend-rollup-parity.tsindex 152311923..35c308b0a 100644--- a/test/support/spend-rollup-parity.ts+++ b/test/support/spend-rollup-parity.ts@@ -182,4 +182,25 @@ export async function assertSpendRollupParity(assert.equal(row.calls, 1);assert.equal(row.costUsd, 0.01);}++ clock.at = BASE + 123;+ const changing = await record(`${prefix}:changing`, "dm", alice, 0, `${prefix}-changing`);+ const range = { from: clock.at, to: clock.at + 1 };+ const readChanging = async () => (await store.spendRollup(range)).filter((r) => r.model === `${prefix}-changing`);+ assert.equal((await readChanging())[0]!.calls, 1);+ assert.equal((await readChanging())[0]!.costUsd, 0);+ assert.equal(+ (await store.spendRollup({ from: BASE, to: clock.at })).some((r) => r.model === `${prefix}-changing`),+ false,+ );+ await record(`${prefix}:changing`, "dm", alice, 2, `${prefix}-changing`);+ assert.equal((await readChanging())[0]!.calls, 2);+ assert.equal((await readChanging())[0]!.costUsd, 2);+ const cronParent = await store.getOrCreateByThread(`cron:${prefix}-changing:fire:aaa`, "dm", bob);+ await store.setParentSession(changing.id, cronParent.id);+ assert.equal((await readChanging())[0]!.origin, "cron");+ await store.deleteSession(cronParent.id);+ assert.equal((await readChanging())[0]!.origin, "conversation");+ await store.deleteSession(changing.id);+ assert.deepEqual(await readChanging(), []);}