Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fix cron contention, approval delivery loops, and slow reports (#1639)

Joshua France5be07724b654
diff --git a/src/core/approval-store.ts b/src/core/approval-store.ts
index f446a0b13..13a81a2a3 100644
--- a/src/core/approval-store.ts
+++ b/src/core/approval-store.ts
@@ -8,13 +8,19 @@ export function approvalDeliveryKey(id: string, record: Pick<PendingApprovalReco
return `command-approval:${id}:${record.createdAt ?? 0}`;
}
+export function approvalDeliveryRecipient(actor: { externalId?: string } | undefined): string | undefined {
+ const id = actor?.externalId;
+ return id && !id.startsWith("system:") ? id : undefined;
+}
+
export function createApprovalStore(
backing: DurableMap<PendingApprovalRecord>,
deliveries: Pick<DeliveryStore, "enqueue">,
) {
async function deliver(id: string, record: PendingApprovalRecord): Promise<void> {
if (record.request?.surface !== "slack") return;
- const actorId = record.request.actor.externalId;
+ const actorId = approvalDeliveryRecipient(record.request.actor);
+ if (!actorId) return;
await deliveries.enqueue({
destination: { ...principalDestination(actorId, actorId), commandApprovalId: id },
text: `Approval needed: ${record.summary ?? record.command}`,
diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts
index 62c5f7832..b1a91c285 100644
--- a/src/cron/scheduler.ts
+++ b/src/cron/scheduler.ts
@@ -201,23 +201,29 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
const maxFiresPerTick = deps.maxFiresPerTick ?? 100;
const leaderLease = deps.leaderLease ?? createNoopLeaderLease();
+ async function withCronLifecycle<T>(cronId: string, scheduled: boolean, run: () => Promise<T>): Promise<T | null> {
+ if (scheduled && deps.lock?.tryWithLock) {
+ const result = await deps.lock.tryWithLock(`cron-lifecycle:${cronId}`, run);
+ if (result === null) await deps.crons.defer(cronId, now() + BUSY_DEFER_MS);
+ return result;
+ }
+ return deps.lock ? deps.lock.withLock(`cron-lifecycle:${cronId}`, run) : run();
+ }
+
async function fire(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
- const run = async (): Promise<FireResult> => {
- const current = await deps.crons.get(cron.id);
- if (!current || current.archived || !current.enabled) {
- await deps.crons.recordFire(cron.id, {
- fireKey,
- threadRef: cronFireThreadRef(cron.id, fireKey),
- firedAt: t,
- endedAt: now(),
- status: "silent",
- note: "Cron is no longer enabled",
- });
- return { authzFailed: true };
- }
- return fireEnabled(current, t, fireKey, scheduledAt);
- };
- return deps.lock ? deps.lock.withLock(`cron-lifecycle:${cron.id}`, run) : run();
+ const current = await deps.crons.get(cron.id);
+ if (!current || current.archived || !current.enabled) {
+ await deps.crons.recordFire(cron.id, {
+ fireKey,
+ threadRef: cronFireThreadRef(cron.id, fireKey),
+ firedAt: t,
+ endedAt: now(),
+ status: "silent",
+ note: "Cron is no longer enabled",
+ });
+ return { authzFailed: true };
+ }
+ return fireEnabled(current, t, fireKey, scheduledAt);
}
async function fireEnabled(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
@@ -375,8 +381,16 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
for (const cron of batch) {
if (stopped || observed !== epoch) break;
try {
- const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);
- if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
+ await withCronLifecycle(cron.id, true, async () => {
+ if (stopped || observed !== epoch) return;
+ const { authzFailed, deferred } = await fire(
+ cron,
+ t,
+ `cron:${cron.id}:${cron.scheduledAt}`,
+ cron.scheduledAt,
+ );
+ if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
+ });
} catch (e) {
reportFailure("scheduler: fire", e);
}
@@ -440,24 +454,28 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
return;
}
if (stopped || observed !== epoch) return;
- if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;
- if (stopped || observed !== epoch) {
- await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
- return;
- }
- try {
- const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);
- if (authzFailed || deferred) {
+ const result = await withCronLifecycle(job.cronId, true, async () => {
+ if (stopped || observed !== epoch) return;
+ if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;
+ if (stopped || observed !== epoch) {
await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
- if (deferred) await enqueueNext(job.cronId);
return;
}
- } catch (e) {
- reportFailure("scheduler: fire", e);
- await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
- return;
- }
- await enqueueNext(job.cronId);
+ try {
+ const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);
+ if (authzFailed || deferred) {
+ await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
+ if (deferred) await enqueueNext(job.cronId);
+ return;
+ }
+ } catch (e) {
+ reportFailure("scheduler: fire", e);
+ await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
+ return;
+ }
+ await enqueueNext(job.cronId);
+ });
+ if (result === null) await enqueueNext(job.cronId);
}
async function reconcile(): Promise<void> {
@@ -533,7 +551,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
? { started: false, reason: "already_running", running: begin.running }
: { started: false, reason: "unavailable" };
}
- const settled = withWork(() => fire(cron, t, fireKey)).then(
+ const settled = withWork(() => withCronLifecycle(cron.id, false, () => fire(cron, t, fireKey))).then(
() => undefined,
reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),
);
diff --git a/src/persistence/durable-map.ts b/src/persistence/durable-map.ts
index 94390817b..6743251c5 100644
--- a/src/persistence/durable-map.ts
+++ b/src/persistence/durable-map.ts
@@ -158,7 +158,11 @@ export function createMemoryMap<T>(): DurableMap<T> {
const VERSIONS_TABLE = "durable_map_versions";
-export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
+export function createPostgresMap<T>(
+ pg: PgPool,
+ table: string,
+ indexedFields: readonly Extract<keyof T, string>[] = [],
+): DurableMap<T> {
if (!/^[a-z_][a-z0-9_]*$/i.test(table)) throw new Error(`invalid table name: ${table}`);
const migration = {
id: `durable-map/${table}/0001`,
@@ -167,11 +171,41 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
`CREATE TABLE IF NOT EXISTS ${VERSIONS_TABLE} (tbl TEXT PRIMARY KEY, v BIGINT NOT NULL)`,
],
};
- pg.registerMigration(migration);
+ const migrations = [
+ migration,
+ ...indexedFields.flatMap((field) => {
+ if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid index field: ${field}`);
+ return [
+ {
+ id: `durable-map/${table}/lookup-${field.toLowerCase()}-drop`,
+ statements: [`SET LOCAL lock_timeout = '3s'`, `DROP INDEX IF EXISTS ${table}_${field.toLowerCase()}_fold`],
+ },
+ {
+ id: `durable-map/${table}/lookup-${field.toLowerCase()}-hash`,
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold
+ ON ${table} USING hash (lower(json->>'${field}'))`,
+ ],
+ },
+ {
+ id: `durable-map/${table}/select-${field.toLowerCase()}`,
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold
+ ON ${table} (lower(json->>'${field}'))`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_unicode
+ ON ${table} (id) WHERE json->>'${field}' ~ '[^\\x01-\\x7f]'`,
+ ],
+ },
+ ];
+ }),
+ ];
+ for (const definition of migrations) pg.registerMigration(definition);
let readyP: Promise<void> | null = null;
function ready(): Promise<void> {
if (!readyP) {
- readyP = pg.migrate(migration).catch((e) => {
+ readyP = (async () => {
+ for (const definition of migrations) await pg.migrate(definition);
+ })().catch((e) => {
readyP = null;
throw e;
});
@@ -224,7 +258,10 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
query.where.field,
query.where.anyOfFold.map((v) => v.toLowerCase()),
);
- sql += ` WHERE (lower(json->>$2) = ANY($3::text[]) OR json->>$2 ~ '[^\\x01-\\x7f]')`;
+ sql += ` WHERE id IN (
+ SELECT id FROM ${table} WHERE lower(json->>$2) = ANY($3::text[])
+ UNION
+ SELECT id FROM ${table} WHERE json->>$2 ~ '[^\\x01-\\x7f]')`;
}
if (query.afterId !== undefined) {
params.push(query.afterId);
@@ -314,11 +351,15 @@ export function createPostgresMap<T>(pg: PgPool, table: string): DurableMap<T> {
}
export interface PostgresArtifactMaps {
- map<T>(table: string): DurableMap<T>;
+ map<T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T>;
pool: PgPool;
}
export function createPostgresMapFactory(connectionString: string): PostgresArtifactMaps {
const pg = createPgPool(connectionString);
- return { map: <T>(table: string): DurableMap<T> => createPostgresMap<T>(pg, table), pool: pg };
+ return {
+ map: <T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T> =>
+ createPostgresMap<T>(pg, table, indexedFields),
+ pool: pg,
+ };
}
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts
index f25f2456e..93c7e9b53 100644
--- a/src/sessions/postgres-session-store.ts
+++ b/src/sessions/postgres-session-store.ts
@@ -190,6 +190,8 @@ function rowToSpendRow(r: Record<string, unknown>): SpendRow {
const LAST_ACTIVITY_DEBOUNCE_MS = 60_000;
const SEARCH_TIMEOUT_MS = 10_000;
+const SPEND_INDEXABLE = `(spend_usage_json(usage_json) IS NOT NULL
+ AND octet_length(session_id) + octet_length(model) + octet_length(usage_json) <= 2000)`;
export function createPostgresSessionStore(connectionString: string, opts: StoreOptions = {}): SessionStore {
const now = opts.now ?? (() => Date.now());
@@ -206,14 +208,13 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
OR (${participant}.valid_to_seq IS NULL AND (${participant}.valid_to IS NULL OR ${entry}.created_at < ${participant}.valid_to))))`;
const participantSessionsSql = (extraWhere: string): string =>
`SELECT s.*, p.title AS p_title, p.archived AS p_archived, p.pinned AS p_pinned, p.color AS p_color,
- COALESCE(MAX(e.created_at), s.created_at) AS user_last_activity,
+ COALESCE((SELECT MAX(e.created_at) FROM session_entries e
+ WHERE e.session_id = s.id AND e.type = 'user'), s.created_at) AS user_last_activity,
EXISTS (SELECT 1 FROM session_entries x WHERE x.session_id = s.id
AND ${withinParticipantWindow("x", "p")}) AS has_entries
FROM sessions s
JOIN participants p ON p.session_id = s.id
- LEFT JOIN session_entries e ON e.session_id = s.id AND e.type = 'user'
- WHERE p.principal_id = $1${extraWhere}
- GROUP BY s.id, p.title, p.archived, p.pinned, p.color, p.valid_from, p.valid_to, p.valid_from_seq, p.valid_to_seq`;
+ WHERE p.principal_id = $1${extraWhere}`;
const participantSessions = async (principalId: string, opts?: { limit: number }): Promise<Session[]> => {
const limit = opts ? Math.max(0, Math.floor(opts.limit)) : undefined;
const rows = await q(
@@ -595,6 +596,61 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
ON session_llm_requests(created_at)`,
],
},
+ {
+ id: "sessions/store/0019-participant-activity-indexes",
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS participants_by_principal
+ ON participants(principal_id, session_id)`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_entries_user_activity
+ ON session_entries(session_id, created_at DESC) WHERE type = 'user'`,
+ ],
+ },
+ {
+ id: "sessions/store/0020-spend-usage-json",
+ statements: [
+ `CREATE OR REPLACE FUNCTION spend_usage_json(t text) RETURNS jsonb
+ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $spend_usage_json$
+ SELECT CASE WHEN pg_input_is_valid(j ->> 'costUsd', 'double precision') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'input', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'output', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'cacheRead', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'cacheWrite', 'bigint') IS NOT FALSE THEN j END
+ FROM (SELECT CASE WHEN pg_input_is_valid(t, 'jsonb') THEN t::jsonb END AS j OFFSET 0) parsed
+ $spend_usage_json$`,
+ ],
+ },
+ {
+ id: "sessions/store/0020-spend-usage-json-size",
+ statements: [
+ `CREATE OR REPLACE FUNCTION spend_usage_json(t text) RETURNS jsonb
+ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $spend_usage_json$
+ SELECT CASE WHEN pg_input_is_valid(j ->> 'costUsd', 'double precision') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'input', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'output', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'cacheRead', 'bigint') IS NOT FALSE
+ AND pg_input_is_valid(j ->> 'cacheWrite', 'bigint') IS NOT FALSE THEN j END
+ FROM (SELECT CASE WHEN octet_length(t) > 2000 THEN NULL
+ WHEN pg_input_is_valid(t, 'jsonb') THEN t::jsonb END AS j OFFSET 0) parsed
+ $spend_usage_json$`,
+ ],
+ },
+ {
+ id: "sessions/store/0021-spend-covering-index",
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_llm_requests_spend
+ ON session_llm_requests(created_at,
+ ((spend_usage_json(usage_json) ->> 'costUsd')::double precision),
+ ((spend_usage_json(usage_json) ->> 'input')::bigint),
+ ((spend_usage_json(usage_json) ->> 'output')::bigint),
+ ((spend_usage_json(usage_json) ->> 'cacheRead')::bigint),
+ ((spend_usage_json(usage_json) ->> 'cacheWrite')::bigint))
+ INCLUDE (session_id, model, usage_json)
+ WHERE usage_json IS NOT NULL AND ${SPEND_INDEXABLE}`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS session_llm_requests_spend_wide
+ ON session_llm_requests(created_at)
+ WHERE usage_json IS NOT NULL AND NOT ${SPEND_INDEXABLE}`,
+ ],
+ },
],
[
{
@@ -1632,12 +1688,13 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
async spendRollup(range): Promise<SpendRow[]> {
const rows = await q(
- `WITH RECURSIVE spending AS (
- SELECT DISTINCT session_id FROM session_llm_requests
- WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL
- ), ancestry AS (
+ `WITH RECURSIVE ancestry AS (
SELECT s.id AS session_id, s.parent_session_id, ${originExpr("s")} AS origin, ARRAY[s.id] AS path
- FROM sessions s JOIN spending r ON r.session_id = s.id
+ FROM sessions s
+ WHERE s.parent_session_id IS NOT NULL AND ${originExpr("s")} = 'conversation'
+ AND EXISTS (SELECT 1 FROM session_llm_requests r
+ WHERE r.session_id = s.id AND r.created_at >= $1 AND r.created_at < $2
+ AND r.usage_json IS NOT NULL)
UNION ALL
SELECT a.session_id, p.parent_session_id, ${originExpr("p")}, a.path || p.id
FROM ancestry a JOIN sessions p ON p.id = a.parent_session_id
@@ -1645,30 +1702,41 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
), origins AS (
SELECT DISTINCT ON (session_id) session_id, origin
FROM ancestry ORDER BY session_id, cardinality(path) DESC
- )
- SELECT day, scope_id, origin, model,
+ ), rollup AS MATERIALIZED (
+ SELECT (r.created_at / 86400000)::bigint AS day, s.scope_id,
+ COALESCE(o.origin, ${originExpr("s")}) AS origin, r.model,
COUNT(*) AS calls,
COALESCE(SUM(cost_usd), 0) AS cost_usd,
COALESCE(SUM(input), 0) AS input,
COALESCE(SUM(output), 0) AS output,
COALESCE(SUM(cache_read), 0) AS cache_read,
COALESCE(SUM(cache_write), 0) AS cache_write
- FROM (SELECT (r.created_at / 86400000)::bigint AS day,
- s.scope_id,
- r.model,
- o.origin,
- (u.j ->> 'costUsd')::double precision AS cost_usd,
- (u.j ->> 'input')::bigint AS input,
- (u.j ->> 'output')::bigint AS output,
- (u.j ->> 'cacheRead')::bigint AS cache_read,
- (u.j ->> 'cacheWrite')::bigint AS cache_write
- FROM session_llm_requests r
- JOIN sessions s ON s.id = r.session_id
- JOIN origins o ON o.session_id = s.id
- CROSS JOIN LATERAL (SELECT r.usage_json::jsonb AS j) u
- WHERE r.created_at >= $1 AND r.created_at < $2
- AND r.usage_json IS NOT NULL) t
- GROUP BY day, scope_id, origin, model`,
+ FROM (SELECT created_at, session_id, model,
+ (spend_usage_json(usage_json) ->> 'costUsd')::double precision AS cost_usd,
+ (spend_usage_json(usage_json) ->> 'input')::bigint AS input,
+ (spend_usage_json(usage_json) ->> 'output')::bigint AS output,
+ (spend_usage_json(usage_json) ->> 'cacheRead')::bigint AS cache_read,
+ (spend_usage_json(usage_json) ->> 'cacheWrite')::bigint AS cache_write
+ FROM session_llm_requests
+ WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL
+ AND ${SPEND_INDEXABLE}
+ UNION ALL
+ SELECT created_at, session_id, model,
+ (usage_json::jsonb ->> 'costUsd')::double precision AS cost_usd,
+ (usage_json::jsonb ->> 'input')::bigint AS input,
+ (usage_json::jsonb ->> 'output')::bigint AS output,
+ (usage_json::jsonb ->> 'cacheRead')::bigint AS cache_read,
+ (usage_json::jsonb ->> 'cacheWrite')::bigint AS cache_write
+ FROM session_llm_requests
+ WHERE created_at >= $1 AND created_at < $2 AND usage_json IS NOT NULL
+ AND NOT ${SPEND_INDEXABLE}
+ AND EXISTS (SELECT 1 FROM sessions WHERE id = session_llm_requests.session_id)
+ OFFSET 0) r
+ JOIN sessions s ON s.id = r.session_id
+ LEFT JOIN origins o ON o.session_id = s.id
+ GROUP BY day, s.scope_id, COALESCE(o.origin, ${originExpr("s")}), r.model
+ )
+ SELECT * FROM rollup ORDER BY day, scope_id, origin, model`,
[range.from, range.to],
);
return rows.map(rowToSpendRow);
diff --git a/src/slack/deliveries.ts b/src/slack/deliveries.ts
index 7e6926e0d..9eeff21e4 100644
--- a/src/slack/deliveries.ts
+++ b/src/slack/deliveries.ts
@@ -1,5 +1,5 @@
import { deployAccessMessage } from "./deploy-access.ts";
-import { approvalDeliveryKey } from "../core/approval-store.ts";
+import { approvalDeliveryKey, approvalDeliveryRecipient } from "../core/approval-store.ts";
import { samePerson } from "../directory/person.ts";
import { approvalMessage } from "./approval-cards.ts";
import { keychainApprovalMessage, keychainApprovalOrigin } from "./keychain-approvals.ts";
@@ -374,12 +374,14 @@ export function createDeliveryPoller(deps: {
const commandApproval = d.destination.commandApprovalId
? await core.getApproval(d.destination.commandApprovalId)
: null;
- const requester = commandApproval?.request?.actor as { externalId?: string } | undefined;
+ const requester = approvalDeliveryRecipient(
+ commandApproval?.request?.actor as { externalId?: string } | undefined,
+ );
if (
d.destination.commandApprovalId &&
(!commandApproval ||
- !requester?.externalId ||
- !samePerson(requester.externalId, d.destination.target) ||
+ !requester ||
+ !samePerson(requester, d.destination.target) ||
d.idempotencyKey !== approvalDeliveryKey(d.destination.commandApprovalId, commandApproval))
)
return undefined;
diff --git a/src/wiring.ts b/src/wiring.ts
index 6700da051..7825a0ff3 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -606,8 +606,8 @@ export function buildApp(
membership.managesArtifactHome!(scopeId, authoredBy ?? "", principalId),
});
const pgArtifactMap = config.databaseUrl ? createPostgresMapFactory(config.databaseUrl) : null;
- const artifactMap = <T>(table: string): DurableMap<T> =>
- pgArtifactMap ? pgArtifactMap.map<T>(table) : createMemoryMap<T>();
+ const artifactMap = <T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T> =>
+ pgArtifactMap ? pgArtifactMap.map<T>(table, indexedFields) : createMemoryMap<T>();
setProviderBaseUrls(config.providerBaseUrls);
const unknownGatewayModels = Object.keys(config.modelGateway?.models ?? {}).filter((id) => !resolveModel(id));
if (unknownGatewayModels.length) {
@@ -1201,8 +1201,8 @@ export function buildApp(
...(legacyCredentialKey ? { fallbacks: [legacyCredentialKey] } : {}),
};
const credentialStore: Keychain = createKeychain({
- creds: artifactMap<KeychainCredential>("keychain_credentials"),
- grants: artifactMap<KeychainGrant>("keychain_grants"),
+ creds: artifactMap<KeychainCredential>("keychain_credentials", ["ownerId"]),
+ grants: artifactMap<KeychainGrant>("keychain_grants", ["ownerId"]),
asks: artifactMap<KeychainAsk>("keychain_asks"),
key: credentialKey,
lock: advisoryLock,
@@ -1490,7 +1490,7 @@ export function buildApp(
config.swarmsEnabled !== false && config.sessionStore === swarmStoreKind && runStoreKind === swarmStoreKind
? createSwarmService({
defaults: config.swarmDefaults,
- store: createSwarmStore(artifactMap<SwarmStorage>("swarms"), {
+ store: createSwarmStore(artifactMap<SwarmStorage>("swarms", ["pending"]), {
runs,
sessions,
...(pgArtifactMap && runStoreKind === "postgres" ? { pg: pgArtifactMap.pool } : {}),
@@ -1739,7 +1739,7 @@ export function buildApp(
: createMemoryEnvironmentStore();
const monitors = createMonitorStore(artifactMap<Monitor>("monitors"));
const loopStore = createLoopStore(artifactMap<Loop>("loops"));
- const loopItemsMap = artifactMap<LoopItem>("loop_items");
+ const loopItemsMap = artifactMap<LoopItem>("loop_items", ["loopId"]);
const loopOwnerCache = new Map<string, string>();
const publishLoopEvent = (event: import("./loops/ledger-events.ts").LedgerEvent): void => {
void (async () => {
@@ -1758,7 +1758,7 @@ export function buildApp(
return Boolean(loop && (loop.surface !== "inbox" || loop.state === "enabled"));
},
});
- const loopOutputs = createLoopOutputStore(artifactMap<LoopOutput>("loop_outputs"), (output) =>
+ const loopOutputs = createLoopOutputStore(artifactMap<LoopOutput>("loop_outputs", ["itemId"]), (output) =>
publishLoopEvent({ loopId: output.loopId, itemId: output.itemId, op: "ready", at: Date.now() }),
);
const loopGrants = createShipGrantStore(artifactMap<ShipGrant>("loop_ship_grants"));
@@ -1838,7 +1838,7 @@ export function buildApp(
conversation: { ...request.conversation, audience, publishMembers: audience },
};
};
- const sessionMailbox = createSessionMailbox(artifactMap<SessionMessage>("session_mailbox"));
+ const sessionMailbox = createSessionMailbox(artifactMap<SessionMessage>("session_mailbox", ["recipientId"]));
const sessionSyscalls = createSessionSyscalls({
mailbox: sessionMailbox,
enabled: async (actorId) =>
@@ -2353,8 +2353,8 @@ export function buildApp(
});
const loopIngress = createLoopIngress({
enabledFor: (owner) => featureFlags.enabled("inbox_loops", scopeId("personal", owner)),
- sources: artifactMap<LoopIngress>("loop_ingress"),
- deliveries: artifactMap<IngressDelivery>("loop_ingress_deliveries"),
+ sources: artifactMap<LoopIngress>("loop_ingress", ["loopId"]),
+ deliveries: artifactMap<IngressDelivery>("loop_ingress_deliveries", ["queueKey"]),
loops: loopStore,
items: loopItems,
outputs: loopOutputs,
diff --git a/test/approval-store.test.ts b/test/approval-store.test.ts
index 12f31356b..46cf3b0d9 100644
--- a/test/approval-store.test.ts
+++ b/test/approval-store.test.ts
@@ -66,3 +66,29 @@ test("web approvals remain on their native surface", async () => {
await approvals.deliverPending();
assert.equal((await deliveries.pending("principal")).length, 0);
});
+
+test("system approvals stay pending without enqueuing or reviving an impossible Slack DM", async () => {
+ const backing = createMemoryMap<PendingApprovalRecord>();
+ const deliveries = createDeliveryStore({ maxAgeMs: 0 });
+ const approvals = createApprovalStore(backing, deliveries);
+ const approval = record("slack:C1:ambient:1.0");
+ approval.request!.actor.externalId = "system:ambient:acme";
+ const stale = await deliveries.enqueue({
+ destination: { type: "principal", target: "system:ambient:acme", commandApprovalId: "A1" },
+ text: "Approval needed: publish",
+ idempotencyKey: "command-approval:A1:1",
+ });
+ await deliveries.claimPending("principal", 1);
+ const expiredAt = (await deliveries.get(stale.id))?.expiredAt;
+ assert.ok(expiredAt);
+
+ await approvals.put("A1", approval);
+ await approvals.put("A2", approval);
+ await approvals.deliverPending();
+ await createApprovalStore(backing, deliveries).deliverPending();
+
+ assert.deepEqual(await backing.get("A1"), approval, "the protected action still requires approval");
+ assert.deepEqual(await backing.get("A2"), approval);
+ assert.deepEqual(await deliveries.pending("principal"), []);
+ assert.equal((await deliveries.get(stale.id))?.expiredAt, expiredAt, "a sweep must not revive the stale DM");
+});
diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.ts
index 069033b45..6afeeb2ea 100644
--- a/test/cron-scheduler.test.ts
+++ b/test/cron-scheduler.test.ts
@@ -13,6 +13,8 @@ import { isPollSurface, isSilentPollReply } from "../src/triggers/run-trigger.ts
import { createDirectoryStore, type DirectoryStore } from "../src/directory/directory-store.ts";
import { createMemoryMap } from "../src/persistence/durable-map.ts";
import type { Cron } from "../src/types.ts";
+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
+import type { CronFireJob, CronJobQueue } from "../src/cron/job-queue.ts";
function fakeLease(isLeader: () => boolean): LeaderLease {
return {
@@ -1024,6 +1026,193 @@ test("queue mode: fires claim the slot before running, and stale or lost claims
scheduler.stop();
});
+for (const queued of [false, true]) {
+ test(`${queued ? "queue" : "interval"} mode: a busy cron defers without blocking other crons or losing its slot`, async () => {
+ const crons = createCronStore();
+ const gate = Promise.withResolvers<TurnResult>();
+ const started = Promise.withResolvers<void>();
+ const calls: TurnRequest[] = [];
+ const enqueued: CronFireJob[] = [];
+ let clock = 1000;
+ let onFire: ((job: CronFireJob) => Promise<void>) | undefined;
+ const jobQueue: CronJobQueue = {
+ async start(handlers) {
+ onFire = handlers.onFire;
+ },
+ async enqueueFire(job) {
+ enqueued.push(job);
+ },
+ healthy: () => true,
+ async stop() {},
+ };
+ const scheduler = createScheduler({
+ crons,
+ deliveries: createDeliveryStore(),
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ lock: createMemoryAdvisoryLock(),
+ now: () => clock,
+ ...(queued ? { jobQueue } : {}),
+ run: async (req) => {
+ calls.push(req);
+ if (calls.length === 1) {
+ started.resolve();
+ return gate.promise;
+ }
+ return { status: "ok", reply: "done" };
+ },
+ });
+ const cron = await crons.create({
+ schedule: { everyMs: 1000, firstFireAt: 1000 },
+ action: "slow recurring job",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+ if (queued) {
+ scheduler.start(1000);
+ await scheduler.ready();
+ }
+ let first: Promise<void>;
+ if (queued) first = onFire!({ cronId: cron.id, scheduledAt: 1000 });
+ else {
+ const manual = await scheduler.runNow(cron.id);
+ assert.ok(manual.started);
+ first = manual.settled;
+ }
+ await started.promise;
+ clock = 2000;
+ const other = await crons.create({
+ schedule: { firstFireAt: clock },
+ message: "unrelated reminder",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+ const owedSlot = (await crons.get(cron.id))!.nextFireAt!;
+ let deferred = false;
+ const second = (queued ? onFire!({ cronId: cron.id, scheduledAt: owedSlot }) : scheduler.tick(clock)).then(
+ () => (deferred = true),
+ );
+ try {
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(deferred, true, "an active fire must not make the scheduler wait for its lifecycle lock");
+ assert.equal(calls.length, 1, "the busy cron did not start another agent");
+ const pending = (await crons.get(cron.id))!;
+ assert.equal(pending.nextFireAt, owedSlot, "the unstarted slot remains owed");
+ assert.equal(pending.deferUntil, 32_000, "the retry is durable and delayed");
+ if (queued) {
+ assert.deepEqual(enqueued.at(-1), { cronId: cron.id, scheduledAt: owedSlot, notBefore: 32_000 });
+ await onFire!({ cronId: other.id, scheduledAt: 2000 });
+ }
+ assert.equal((await crons.get(other.id))!.enabled, false, "an unrelated cron can finish while this one runs");
+ gate.resolve({ status: "ok", reply: "done" });
+ await first;
+ clock = 32_000;
+ if (queued) await onFire!({ cronId: cron.id, scheduledAt: owedSlot });
+ else await scheduler.tick(clock);
+ assert.equal(calls.length, 2, "the deferred slot runs once after the active fire completes");
+ assert.equal((await crons.listFires(cron.id)).total, 2, "no phantom running fire was created for the deferral");
+ } finally {
+ gate.resolve({ status: "ok", reply: "done" });
+ await Promise.all([first, second]);
+ await scheduler.stop();
+ }
+ });
+}
+
+test("queue mode: concurrent reconciliation cannot consume an overdue slot while its lifecycle lock is held", async () => {
+ const crons = createCronStore();
+ const cron = await crons.create({
+ schedule: { cron: "* * * * *", timezone: "UTC" },
+ action: "overdue calendar task",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+ const slot = cron.nextFireAt!;
+ let clock = slot + 120_000;
+ const mutex = createMemoryAdvisoryLock();
+ const releaseHeld = Promise.withResolvers<void>();
+ const heldEntered = Promise.withResolvers<void>();
+ const holding = mutex.withLock(`cron-lifecycle:${cron.id}`, async () => {
+ heldEntered.resolve();
+ await releaseHeld.promise;
+ });
+ await heldEntered.promise;
+ const gates = [0, 1].map(() => ({ entered: Promise.withResolvers<void>(), release: Promise.withResolvers<void>() }));
+ let attempts = 0;
+ let onFire!: (job: CronFireJob) => Promise<void>;
+ let onTick!: () => Promise<void>;
+ const enqueued: CronFireJob[] = [];
+ const calls: TurnRequest[] = [];
+ const scheduler = createScheduler({
+ crons,
+ deliveries: createDeliveryStore(),
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ now: () => clock,
+ lock: {
+ ...mutex,
+ async tryWithLock(key, fn) {
+ const gate = gates[attempts++];
+ if (gate) {
+ gate.entered.resolve();
+ await gate.release.promise;
+ }
+ return mutex.tryWithLock!(key, fn);
+ },
+ },
+ run: async (req) => {
+ calls.push(req);
+ return { status: "ok", reply: "done" };
+ },
+ jobQueue: {
+ async start(handlers) {
+ onFire = handlers.onFire;
+ onTick = handlers.onTick;
+ },
+ async enqueueFire(job) {
+ enqueued.push(job);
+ },
+ healthy: () => true,
+ async stop() {},
+ },
+ });
+ scheduler.start(1000);
+ await scheduler.ready();
+ const first = onFire({ cronId: cron.id, scheduledAt: slot });
+ let second: Promise<void> | undefined;
+ try {
+ await gates[0]!.entered.promise;
+ await onTick();
+ clock++;
+ second = onFire(enqueued.at(-1)!);
+ await gates[1]!.entered.promise;
+ gates[0]!.release.resolve();
+ await first;
+ gates[1]!.release.resolve();
+ await second;
+ const deferred = (await crons.get(cron.id))!;
+ assert.equal(deferred.nextFireAt, slot, "neither claimant can advance past an unstarted calendar slot");
+ assert.equal(deferred.lastFiredAt, undefined);
+ assert.equal(deferred.deferUntil, clock + 30_000);
+ assert.equal(calls.length, 0);
+ releaseHeld.resolve();
+ await holding;
+ clock = deferred.deferUntil!;
+ await onFire({ cronId: cron.id, scheduledAt: slot });
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0]!.idempotencyKey, `cron:${cron.id}:${slot}`);
+ assert.equal((await crons.get(cron.id))!.nextFireAt, slot + 60_000);
+ } finally {
+ for (const gate of gates) gate.release.resolve();
+ releaseHeld.resolve();
+ await Promise.all([holding, first, second]);
+ await scheduler.stop();
+ }
+});
+
test("queue mode: while the queue runs, the interval scheduler's leader lease is held as a guard", async (t) => {
t.mock.timers.enable({ apis: ["setInterval"] });
const heldKeys: string[] = [];
diff --git a/test/durable-stores.test.ts b/test/durable-stores.test.ts
index 1226897df..a9e26b709 100644
--- a/test/durable-stores.test.ts
+++ b/test/durable-stores.test.ts
@@ -179,7 +179,7 @@ test("the Postgres map select projects and filters inside SQL, never fetching th
assert.equal(reads.length, 1);
assert.match(
reads[0]!.sql,
- /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE \(lower\(json->>\$2\) = ANY\(\$3::text\[\]\) OR json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,
+ /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>\$2\) = ANY\(\$3::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,
);
assert.deepEqual(reads[0]!.params, [["secretEnc"], "owner", ["u1", "[email protected]"]]);
diff --git a/test/postgres-map.test.ts b/test/postgres-map.test.ts
index e7af74dfe..a124cd95b 100644
--- a/test/postgres-map.test.ts
+++ b/test/postgres-map.test.ts
@@ -1,5 +1,6 @@
import { test, before } from "node:test";
import assert from "node:assert/strict";
+import { randomBytes } from "node:crypto";
import { createMemoryMap, createPostgresMapFactory } from "../src/persistence/durable-map.ts";
import { createCronStore } from "../src/cron/cron-store.ts";
import { createWebhookStore, type WebhookHistory } from "../src/webhooks/webhook-store.ts";
@@ -12,6 +13,7 @@ import {
type KeychainGrant,
} from "../src/credentials/keychain.ts";
import { deriveConnectorKey } from "../src/connectors/connector-client-store.ts";
+import { applyPgMigrations, definePgMigration } from "../src/persistence/pg-pool.ts";
const URL = process.env.DATABASE_URL;
const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres map tests";
@@ -22,7 +24,7 @@ before(async () => {
const p = new pg.Pool({ connectionString: URL });
await p.query("DROP TABLE IF EXISTS qm_schema_migrations CASCADE");
await p.query(
- "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",
+ "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",
);
await p.end();
});
@@ -103,6 +105,7 @@ test("pg map: rejects an unsafe table name (the DDL/DML interpolation guard)", (
const factory = createPostgresMapFactory("postgres://unused");
assert.throws(() => factory.map("bad name"), /invalid table name/);
assert.throws(() => factory.map("crons;--"), /invalid table name/);
+ assert.throws(() => factory.map<{ owner: string }>("map_widgets", ["bad field" as "owner"]), /invalid index field/);
});
test(
@@ -179,12 +182,13 @@ test("pg map: select mirrors the memory map — folded field filter, projection,
["sel-c", { name: "sel-c", tags: [], nested: { n: 3 }, owner: "[email protected]", secretEnc: "enc-c" }],
];
const factory = createPostgresMapFactory(URL!);
- const pgMap = factory.map<Owned>("map_widgets");
+ let pgMap = factory.map<Owned>("map_widgets");
const memMap = createMemoryMap<Owned>();
for (const [id, row] of rows) {
await pgMap.put(id, row);
await memMap.put(id, row);
}
+ pgMap = factory.map<Owned>("map_widgets", ["owner"]);
try {
const mine = await pgMap.select({ omit: ["secretEnc"], where: { field: "owner", anyOfFold: ["U7"] } });
assert.deepEqual(mine, await memMap.select({ omit: ["secretEnc"], where: { field: "owner", anyOfFold: ["U7"] } }));
@@ -221,22 +225,92 @@ test("pg map: select mirrors the memory map — folded field filter, projection,
await memMap.put("sel-d", turkish);
const swept = await pgMap.select({ where: { field: "owner", anyOfFold: ["no-such-owner"] } });
assert.deepEqual(swept, await memMap.select({ where: { field: "owner", anyOfFold: ["no-such-owner"] } }));
+ const indexes = await factory.pool.q(
+ "SELECT indexrelid::regclass::text AS name, indisvalid FROM pg_index WHERE indrelid = 'map_widgets'::regclass",
+ );
+ for (const name of ["map_widgets_owner_fold", "map_widgets_owner_unicode"]) {
+ assert.ok(indexes.some((index) => index.name === name && index.indisvalid));
+ }
+ await pgMap.merge("sel-a", { owner: "Moved" });
+ assert.deepEqual(
+ (await pgMap.select({ where: { field: "owner", anyOfFold: ["MOVED"] } })).map((w) => w.name),
+ ["sel-a", "sel-d"],
+ );
assert.deepEqual(
swept.map((w) => w.name),
["sel-d"],
"a non-ASCII field value is always a candidate — SQL lower() and JS toLowerCase() disagree there",
);
+ await pgMap.merge("sel-d", { owner: "josé" });
+ assert.deepEqual(
+ (await pgMap.select({ where: { field: "owner", anyOfFold: ["josé"] } })).map((row) => row.name),
+ ["sel-d"],
+ "a row matching both indexed branches is returned once",
+ );
} finally {
for (const id of [...rows.map(([id]) => id), "sel-d"]) await pgMap.delete(id);
await factory.pool.close();
}
});
+test("pg map: indexed fields accept wide values through migration and later writes", { skip }, async () => {
+ const factory = createPostgresMapFactory(URL!);
+ const first = randomBytes(3000).toString("hex");
+ const second = randomBytes(3000).toString("hex");
+ const unindexed = factory.map<{ owner: string }>("map_wide_widgets");
+ try {
+ await unindexed.put("existing", { owner: first });
+ const indexed = factory.map<{ owner: string }>("map_wide_widgets", ["owner"]);
+ const select = (owner: string) => indexed.select({ where: { field: "owner", anyOfFold: [owner.toUpperCase()] } });
+ assert.deepEqual(await select(first), [{ owner: first }]);
+ await indexed.put("later", { owner: second });
+ await indexed.merge("existing", { owner: second });
+ assert.deepEqual(await select(first), []);
+ assert.deepEqual(await select(second), [{ owner: second }, { owner: second }]);
+ } finally {
+ await unindexed.delete("existing");
+ await unindexed.delete("later");
+ await factory.pool.close();
+ }
+});
+
+test("pg map: original folded-index migration upgrades without changing its ledger", { skip }, async () => {
+ const factory = createPostgresMapFactory(URL!);
+ const unindexed = factory.map<{ owner: string }>("map_wide_widgets");
+ await unindexed.put("legacy", { owner: "short" });
+ const legacy = definePgMigration("durable-map/map_wide_widgets/select-owner", [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS map_wide_widgets_owner_fold
+ ON map_wide_widgets (lower(json->>'owner'))`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS map_wide_widgets_owner_unicode
+ ON map_wide_widgets (id) WHERE json->>'owner' ~ '[^\\x01-\\x7f]'`,
+ ]);
+ try {
+ await factory.pool.q("DROP INDEX IF EXISTS map_wide_widgets_owner_fold, map_wide_widgets_owner_unicode");
+ await factory.pool.q(
+ "DELETE FROM qm_schema_migrations WHERE id LIKE 'durable-map/map_wide_widgets/%' AND id <> 'durable-map/map_wide_widgets/0001'",
+ );
+ await applyPgMigrations(await factory.pool.pool(), [legacy]);
+ const ledger = await factory.pool.q("SELECT * FROM qm_schema_migrations WHERE id = $1", [legacy.id]);
+ const indexed = factory.map<{ owner: string }>("map_wide_widgets", ["owner"]);
+ const owner = randomBytes(3000).toString("hex");
+ await indexed.put("legacy", { owner });
+ assert.deepEqual(await indexed.select({ where: { field: "owner", anyOfFold: [owner] } }), [{ owner }]);
+ assert.deepEqual(await factory.pool.q("SELECT * FROM qm_schema_migrations WHERE id = $1", [legacy.id]), ledger);
+ const [index] = await factory.pool.q(
+ "SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid = c.relam WHERE c.oid = 'map_wide_widgets_owner_fold'::regclass",
+ );
+ assert.equal(index!.amname, "hash");
+ } finally {
+ await unindexed.delete("legacy");
+ await factory.pool.close();
+ }
+});
+
test("pg keychain: listByOwner is a per-owner projected read with no secret material", { skip }, async () => {
const factory = createPostgresMapFactory(URL!);
const keychain = createKeychain({
- creds: factory.map<KeychainCredential>("map_keychain_creds"),
- grants: factory.map<KeychainGrant>("map_keychain_grants"),
+ creds: factory.map<KeychainCredential>("map_keychain_creds", ["ownerId"]),
+ grants: factory.map<KeychainGrant>("map_keychain_grants", ["ownerId"]),
asks: factory.map<KeychainAsk>("map_keychain_asks"),
key: deriveConnectorKey("postgres-keychain-test-key"),
});
@@ -272,8 +346,8 @@ test("pg map: concurrent keychain instances claim a once grant exactly once", {
const key = deriveConnectorKey("postgres-keychain-test-key");
const build = (factory: ReturnType<typeof createPostgresMapFactory>) =>
createKeychain({
- creds: factory.map<KeychainCredential>("map_keychain_creds"),
- grants: factory.map<KeychainGrant>("map_keychain_grants"),
+ creds: factory.map<KeychainCredential>("map_keychain_creds", ["ownerId"]),
+ grants: factory.map<KeychainGrant>("map_keychain_grants", ["ownerId"]),
asks: factory.map<KeychainAsk>("map_keychain_asks"),
key,
});
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index b6b70fd16..9a0f04fbf 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -4,7 +4,7 @@ import { migrateTranscriptPage } from "../scripts/lib/transcript-tape-migration.
import { test, before } from "node:test";
import { randomUUID } from "node:crypto";
import assert from "node:assert/strict";
-import { migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
+import { applyPgMigrations, migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
import { PARALLEL_EXCEPTION_QUERY } from "../src/deployment/postdeploy-smoke.ts";
import {
backfillSessionOriginBatch,
@@ -73,10 +73,224 @@ test("pg session store: getForParticipant returns exactly the row listByParticip
await assertParticipantSessionParity(createPostgresSessionStore(URL!), `pg-parity-${randomUUID()}`);
});
+test("pg participant activity uses the latest user entry, including overheard entries", { skip }, async () => {
+ let at = 1_000;
+ const store = createPostgresSessionStore(URL!, { now: () => at });
+ const owner = `activity-${randomUUID()}`;
+ const scope = scopeId("personal", owner);
+ const session = await store.getOrCreateByThread(owner, "dm", scope);
+ await store.addParticipant(session.id, owner);
+ const { lease } = await store.acquireLease(session.id);
+ assert.ok(lease);
+ at = 300_000;
+ await store.append(lease, { type: "user", payload: { text: "overheard", overheard: true }, scopeLabel: scope });
+ at = 500_000;
+ await store.append(lease, { type: "assistant", payload: { text: "later reply" }, scopeLabel: scope });
+ await store.releaseLease(lease);
+ at = 400_000;
+ const empty = await store.getOrCreateByThread(`${owner}-empty`, "dm", scope);
+ await store.addParticipant(empty.id, owner);
+ const listed = await store.listByParticipant(owner);
+ assert.equal(listed.find((row) => row.id === session.id)?.lastActivityAt, 300_000);
+ assert.equal(listed.find((row) => row.id === empty.id)?.lastActivityAt, 400_000);
+ assert.equal((await store.getForParticipant(session.id, owner))?.lastActivityAt, 300_000);
+ assert.deepEqual(
+ (await store.listByParticipant(owner, { limit: 1 })).map((row) => row.id),
+ [session.id],
+ );
+ assert.deepEqual(await store.listByParticipant(owner, { limit: 0 }), []);
+});
+
test("pg session store: spendRollup matches the memory rollup row for row", { skip }, async () => {
await assertSpendRollupParity((now) => createPostgresSessionStore(URL!, { now }), `pg-spend-${randomUUID()}`);
});
+test("pg spend indexes preserve legacy and unusual requests through migration and later writes", { skip }, async () => {
+ const at = Date.UTC(2022, 0, 1);
+ const store = createPostgresSessionStore(URL!, { now: () => at });
+ const session = await store.getOrCreateByThread("spend-index", "dm", scopeId("personal", "USPENDINDEX"));
+ const pg = (await import("pg")).default;
+ const raw = new pg.Pool({ connectionString: URL });
+ const usage = { input: 10, output: 2, cacheRead: 3, cacheWrite: 4, totalTokens: 19, costUsd: 0.5 };
+ const wideModel = Array.from({ length: 100 }, () => randomUUID()).join("");
+ const wideUsage = { ...usage, legacyDetail: wideModel };
+ const deeplyNestedUsage =
+ JSON.stringify(usage).slice(0, -1) + ',"detail":' + "[".repeat(20_000) + "0" + "]".repeat(20_000) + "}";
+ const insert = async (id: string, model: string, json: string | null, when = at, sessionId = session.id) => {
+ await raw.query(
+ "INSERT INTO session_llm_requests(id, session_id, step, model, scope_label, created_at, usage_json) VALUES ($1,$2,0,$3,$4,$5,$6)",
+ [id, sessionId, model, session.scopeId, when, json],
+ );
+ };
+ const range = { from: at, to: at + 1 };
+ try {
+ await raw.query("DROP INDEX session_llm_requests_spend, session_llm_requests_spend_wide");
+ await raw.query(
+ "DELETE FROM qm_schema_migrations WHERE id IN ('sessions/store/0020-spend-usage-json', 'sessions/store/0020-spend-usage-json-size', 'sessions/store/0021-spend-covering-index')",
+ );
+ await insert("spend-index-normal", "normal", JSON.stringify(usage));
+ await insert("spend-index-wide", wideModel, JSON.stringify(wideUsage));
+ await insert("spend-index-null", "null", null);
+ await insert("spend-index-empty", "empty", "{}");
+ await insert("spend-index-bad-history", "bad-history", "not json", at - 1);
+ await insert("spend-index-bad-orphan", "bad-orphan", "not json", at, "missing-spend-session");
+ await insert(
+ "spend-index-fraction-orphan",
+ "fraction-orphan",
+ JSON.stringify({ ...usage, input: 0.5 }),
+ at,
+ "missing-spend-session",
+ );
+ await insert("spend-index-unicode", "bad-unicode", JSON.stringify({ ...usage, detail: "\u0000" }), at - 1);
+ await insert("spend-index-fraction", "fraction", JSON.stringify({ ...usage, input: 0.5 }), at - 1);
+ await insert("spend-index-overflow", "overflow", JSON.stringify({ ...usage, input: 1e308 }), at - 1);
+ await insert("spend-index-deep-history", "deep-history", deeplyNestedUsage, at - 2);
+ await migrateRegisteredPgSchemas(URL!);
+ const indexes = await raw.query(
+ "SELECT indexrelid::regclass::text AS name, indisvalid, indisready FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass)",
+ );
+ assert.equal(indexes.rows.length, 2);
+ assert.ok(indexes.rows.every((r) => r.indisvalid && r.indisready));
+ const read = async () => (await store.spendRollup(range)).filter((r) => r.scopeId === session.scopeId);
+ const before = await read();
+ assert.equal(before.length, 3);
+ assert.equal(
+ before.reduce((sum, row) => sum + row.calls, 0),
+ 3,
+ );
+ assert.equal(
+ before.reduce((sum, row) => sum + row.costUsd, 0),
+ 1,
+ );
+ await store.recordLlmRequest(session.id, {
+ turnSeq: null,
+ step: 0,
+ model: wideModel,
+ scopeLabel: session.scopeId,
+ usage: wideUsage,
+ });
+ await store.recordLlmRequest(session.id, {
+ turnSeq: null,
+ step: 0,
+ model: "fraction-after-index",
+ scopeLabel: session.scopeId,
+ usage: { ...usage, input: 0.5 },
+ });
+ await raw.query("UPDATE session_llm_requests SET created_at = $1 WHERE model = 'fraction-after-index'", [at - 1]);
+ await insert("spend-index-bad-later", "bad-later", "not json", at - 1);
+ await insert("spend-index-deep-later", "deep-later", deeplyNestedUsage, at - 2);
+ await assert.rejects(store.spendRollup({ from: at - 2, to: at - 1 }), { code: "54001" });
+ await raw.query("UPDATE session_llm_requests SET usage_json = $1 WHERE id = 'spend-index-normal'", [
+ JSON.stringify({ ...usage, costUsd: 2 }),
+ ]);
+ assert.equal(
+ (await read()).reduce((sum, row) => sum + row.costUsd, 0),
+ 3,
+ );
+ await raw.query("UPDATE sessions SET scope_id = 'personal:USPENDMOVED' WHERE id = $1", [session.id]);
+ const moved = (await store.spendRollup(range)).filter((r) => r.scopeId === "personal:USPENDMOVED");
+ assert.equal(
+ moved.reduce((sum, row) => sum + row.calls, 0),
+ 4,
+ );
+ await raw.query("UPDATE session_llm_requests SET created_at = $1 WHERE id = 'spend-index-bad-later'", [at]);
+ await assert.rejects(store.spendRollup(range), /invalid input syntax for type json/);
+ } finally {
+ await raw.query("DELETE FROM session_llm_requests WHERE session_id IN ($1, 'missing-spend-session')", [session.id]);
+ await store.deleteSession(session.id);
+ await raw.end();
+ }
+});
+
+test("pg spend usage guard upgrades the original migration without rewriting indexes or ledger", { skip }, async () => {
+ const pg = (await import("pg")).default;
+ const admin = new pg.Pool({ connectionString: URL });
+ const schema = `spend_upgrade_${randomUUID().replaceAll("-", "")}`;
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new globalThis.URL(URL!);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ const raw = new pg.Pool({ connectionString: url.toString() });
+ const store = createPostgresSessionStore(url.toString());
+ const migrations = registeredPgMigrations(url.toString());
+ const original = migrations.find((migration) => migration.id === "sessions/store/0020-spend-usage-json")!;
+ try {
+ assert.equal(original.checksum, "febf07cde8b9ffe09a497d44a0e524a5c994f7708a0fab6d245315e4ab1e717a");
+ await applyPgMigrations(
+ raw,
+ migrations.filter((migration) => migration.id !== "sessions/store/0020-spend-usage-json-size"),
+ );
+ const ledger = (await raw.query("SELECT * FROM qm_schema_migrations ORDER BY id")).rows;
+ const indexes = (
+ await raw.query(
+ "SELECT indexrelid FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass) ORDER BY indexrelid",
+ )
+ ).rows;
+ await store.countSessions();
+ assert.deepEqual(
+ (
+ await raw.query(
+ "SELECT * FROM qm_schema_migrations WHERE id <> 'sessions/store/0020-spend-usage-json-size' ORDER BY id",
+ )
+ ).rows,
+ ledger,
+ );
+ assert.deepEqual(
+ (
+ await raw.query(
+ "SELECT indexrelid FROM pg_index WHERE indexrelid IN ('session_llm_requests_spend'::regclass, 'session_llm_requests_spend_wide'::regclass) ORDER BY indexrelid",
+ )
+ ).rows,
+ indexes,
+ );
+ const deep = "[".repeat(20_000) + "0" + "]".repeat(20_000);
+ assert.equal((await raw.query("SELECT spend_usage_json($1) AS usage", [deep])).rows[0].usage, null);
+ } finally {
+ await raw.end();
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ await admin.end();
+ }
+});
+
+test("pg spend ancestry only visits sessions with usage inside the requested range", { skip }, async (t) => {
+ const at = Date.UTC(2023, 0, 1);
+ const store = createPostgresSessionStore(URL!, { now: () => at });
+ const scope = scopeId("personal", "USPENDANCESTRY");
+ const parent = await store.getOrCreateByThread("cron:spend-ancestry", "dm", scope);
+ const current = await store.getOrCreateByThread("spend-ancestry-current", "dm", scope);
+ const historical = await store.getOrCreateByThread("spend-ancestry-historical", "dm", scope);
+ await store.setParentSession(current.id, parent.id);
+ await store.setParentSession(historical.id, parent.id);
+ await store.recordLlmRequest(current.id, {
+ turnSeq: null,
+ step: 0,
+ model: "ancestry",
+ scopeLabel: scope,
+ usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, costUsd: 1 },
+ });
+ const pg = (await import("pg")).default;
+ const raw = new pg.Pool({ connectionString: URL });
+ const execute = pg.Pool.prototype.query;
+ let query = "";
+ t.mock.method(pg.Pool.prototype, "query", function (this: InstanceType<typeof pg.Pool>, ...args: unknown[]) {
+ if (typeof args[0] === "string" && args[0].startsWith("WITH RECURSIVE")) query = args[0];
+ return Reflect.apply(execute, this, args);
+ });
+ try {
+ assert.equal((await store.spendRollup({ from: at, to: at + 1 }))[0]!.origin, "cron");
+ assert.ok(query);
+ const explained = await raw.query(`EXPLAIN (ANALYZE, FORMAT JSON) ${query}`, [at, at + 1]);
+ type Plan = { Plans?: Plan[]; "Subplan Name"?: string; "Actual Rows"?: number };
+ const plans = (plan: Plan): Plan[] => [plan, ...(plan.Plans ?? []).flatMap(plans)];
+ const ancestry = plans(explained.rows[0]["QUERY PLAN"][0].Plan).find(
+ (plan) => plan["Subplan Name"] === "CTE ancestry",
+ );
+ assert.equal(ancestry?.["Actual Rows"], 2);
+ } finally {
+ for (const session of [current, historical, parent]) await store.deleteSession(session.id);
+ await raw.end();
+ }
+});
+
test("pg session store: a bare failed acquire means the session is gone, not a lease race", { skip }, async () => {
const s = createPostgresSessionStore(URL!);
const scope = scopeId("personal", "U1");
diff --git a/test/slack-deliveries.test.ts b/test/slack-deliveries.test.ts
index 4394be7ae..e02383cc1 100644
--- a/test/slack-deliveries.test.ts
+++ b/test/slack-deliveries.test.ts
@@ -37,6 +37,7 @@ async function deliver(
const marks: Array<{ channel: string; ts: string }> = [];
const probes: Record<string, unknown>[] = [];
const history = row.history ?? [];
+ const conversationsOpened: Record<string, unknown>[] = [];
const probe = async (args: Record<string, unknown>) => {
probes.push(args);
return {
@@ -54,7 +55,14 @@ async function deliver(
},
};
const client = {
- conversations: { open: async () => ({ channel: { id: "C1" } }), replies: probe, history: probe },
+ conversations: {
+ open: async (args: Record<string, unknown>) => {
+ conversationsOpened.push(args);
+ return { channel: { id: "C1" } };
+ },
+ replies: probe,
+ history: probe,
+ },
files: {
uploadV2: async (args: Record<string, unknown>) => {
uploads.push(args);
@@ -84,7 +92,7 @@ async function deliver(
});
await poller.pollDeliveries(client);
- return { acknowledgements, uploads, posts, marks, probes };
+ return { acknowledgements, uploads, posts, marks, probes, conversationsOpened };
}
for (const type of ["slack", "group", "principal"]) {
@@ -344,6 +352,22 @@ test("a stale queued approval cannot render a newer request for the same command
assert.deepEqual(out.acknowledgements, ["D1"]);
});
+test("a queued system approval notification is retired without opening a Slack DM", async () => {
+ const actorId = "system:ambient:acme";
+ const approval = { requestId: "A1", command: "publish", request: { actor: { externalId: actorId } } };
+ const out = await deliver(
+ { type: "principal", target: actorId, commandApprovalId: "A1" },
+ undefined,
+ undefined,
+ "Approval needed",
+ { approval },
+ );
+ assert.deepEqual(out.conversationsOpened, []);
+ assert.deepEqual(out.posts, []);
+ assert.deepEqual(out.uploads, []);
+ assert.deepEqual(out.acknowledgements, ["D1"]);
+});
+
for (const type of ["slack", "group", "principal"]) {
test(`${type} file-only deliveries survive a restart after a lost acknowledgement`, async () => {
const history: Record<string, unknown>[] = [];
diff --git a/test/support/spend-rollup-parity.ts b/test/support/spend-rollup-parity.ts
index 152311923..35c308b0a 100644
--- a/test/support/spend-rollup-parity.ts
+++ b/test/support/spend-rollup-parity.ts
@@ -182,4 +182,25 @@ export async function assertSpendRollupParity(
assert.equal(row.calls, 1);
assert.equal(row.costUsd, 0.01);
}
+
+ clock.at = BASE + 123;
+ const changing = await record(`${prefix}:changing`, "dm", alice, 0, `${prefix}-changing`);
+ const range = { from: clock.at, to: clock.at + 1 };
+ const readChanging = async () => (await store.spendRollup(range)).filter((r) => r.model === `${prefix}-changing`);
+ assert.equal((await readChanging())[0]!.calls, 1);
+ assert.equal((await readChanging())[0]!.costUsd, 0);
+ assert.equal(
+ (await store.spendRollup({ from: BASE, to: clock.at })).some((r) => r.model === `${prefix}-changing`),
+ false,
+ );
+ await record(`${prefix}:changing`, "dm", alice, 2, `${prefix}-changing`);
+ assert.equal((await readChanging())[0]!.calls, 2);
+ assert.equal((await readChanging())[0]!.costUsd, 2);
+ const cronParent = await store.getOrCreateByThread(`cron:${prefix}-changing:fire:aaa`, "dm", bob);
+ await store.setParentSession(changing.id, cronParent.id);
+ assert.equal((await readChanging())[0]!.origin, "cron");
+ await store.deleteSession(cronParent.id);
+ assert.equal((await readChanging())[0]!.origin, "conversation");
+ await store.deleteSession(changing.id);
+ assert.deepEqual(await readChanging(), []);
}