Skip to content

The open-source app everyone uses to manage agents at work

[codex] Enforce backend execution release gates (#9089)

Dottaef617bee5c15
diff --git a/server/src/__tests__/budgets-service.test.ts b/server/src/__tests__/budgets-service.test.ts
index b85050c54b..8a67682726 100644
--- a/server/src/__tests__/budgets-service.test.ts
+++ b/server/src/__tests__/budgets-service.test.ts
@@ -1,5 +1,20 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { randomUUID } from "node:crypto";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ agents,
+ approvals,
+ budgetIncidents,
+ budgetPolicies,
+ companies,
+ costEvents,
+ createDb,
+ projects,
+} from "@paperclipai/db";
import { budgetService } from "../services/budgets.ts";
+import {
+ getEmbeddedPostgresTestSupport,
+ startEmbeddedPostgresTestDatabase,
+} from "./helpers/embedded-postgres.js";
const mockLogActivity = vi.hoisted(() => vi.fn());
@@ -309,3 +324,317 @@ describe("budgetService", () => {
);
});
});
+
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+describeEmbeddedPostgres("budgetService release gate enforcement", () => {
+ let db!: ReturnType<typeof createDb>;
+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+
+ beforeAll(async () => {
+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-");
+ db = createDb(tempDb.connectionString);
+ }, 20_000);
+
+ afterEach(async () => {
+ await db.delete(budgetIncidents);
+ await db.delete(approvals);
+ await db.delete(budgetPolicies);
+ await db.delete(costEvents);
+ await db.delete(projects);
+ await db.delete(agents);
+ await db.delete(companies);
+ mockLogActivity.mockClear();
+ });
+
+ afterAll(async () => {
+ await tempDb?.cleanup();
+ });
+
+ async function createBudgetFixture() {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const projectId = randomUUID();
+
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+ requireBoardApprovalForNewAgents: false,
+ });
+ await db.insert(agents).values({
+ id: agentId,
+ companyId,
+ name: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",
+ role: "engineer",
+ status: "active",
+ adapterType: "codex_local",
+ adapterConfig: {},
+ runtimeConfig: {},
+ permissions: {},
+ });
+ await db.insert(projects).values({
+ id: projectId,
+ companyId,
+ name: "Budget Project",
+ status: "in_progress",
+ });
+
+ return { companyId, agentId, projectId };
+ }
+
+ async function insertCostEvent(input: {
+ companyId: string;
+ agentId: string;
+ projectId?: string | null;
+ costCents: number;
+ occurredAt?: Date;
+ }) {
+ const [event] = await db
+ .insert(costEvents)
+ .values({
+ companyId: input.companyId,
+ agentId: input.agentId,
+ projectId: input.projectId ?? null,
+ provider: "openai",
+ biller: "openai",
+ billingType: "metered_api",
+ model: "gpt-5-release-gate",
+ inputTokens: 100,
+ cachedInputTokens: 10,
+ outputTokens: 20,
+ costCents: input.costCents,
+ occurredAt: input.occurredAt ?? new Date(),
+ })
+ .returning();
+
+ return event!;
+ }
+
+ it("raises one soft incident per window before hard-stopping and safely logging agent telemetry", async () => {
+ const { companyId, agentId } = await createBudgetFixture();
+ const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
+ const service = budgetService(db, { cancelWorkForScope });
+ const [policy] = await db
+ .insert(budgetPolicies)
+ .values({
+ companyId,
+ scopeType: "agent",
+ scopeId: agentId,
+ metric: "billed_cents",
+ windowKind: "calendar_month_utc",
+ amount: 100,
+ warnPercent: 80,
+ hardStopEnabled: true,
+ notifyEnabled: true,
+ isActive: true,
+ })
+ .returning();
+
+ const softEvent = await insertCostEvent({ companyId, agentId, costCents: 80 });
+ await service.evaluateCostEvent(softEvent);
+ await service.evaluateCostEvent(softEvent);
+
+ let incidentRows = await db
+ .select()
+ .from(budgetIncidents);
+ expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);
+ expect(incidentRows[0]).toMatchObject({
+ companyId,
+ policyId: policy!.id,
+ scopeType: "agent",
+ scopeId: agentId,
+ thresholdType: "soft",
+ amountLimit: 100,
+ amountObserved: 80,
+ approvalId: null,
+ status: "open",
+ });
+
+ const [agentBeforeHardStop] = await db
+ .select({ status: agents.status, pauseReason: agents.pauseReason })
+ .from(agents);
+ expect(agentBeforeHardStop).toEqual({ status: "active", pauseReason: null });
+
+ const hardEvent = await insertCostEvent({ companyId, agentId, costCents: 25 });
+ await service.evaluateCostEvent(hardEvent);
+ await service.evaluateCostEvent(hardEvent);
+
+ incidentRows = await db
+ .select()
+ .from(budgetIncidents);
+ expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);
+ expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);
+ expect(incidentRows.find((incident) => incident.thresholdType === "soft")).toMatchObject({
+ status: "resolved",
+ });
+ expect(incidentRows.find((incident) => incident.thresholdType === "hard")).toMatchObject({
+ amountLimit: 100,
+ amountObserved: 105,
+ status: "open",
+ });
+
+ const [approval] = await db.select().from(approvals);
+ expect(approval).toMatchObject({
+ companyId,
+ type: "budget_override_required",
+ status: "pending",
+ });
+
+ const [agentAfterHardStop] = await db
+ .select({ status: agents.status, pauseReason: agents.pauseReason, pausedAt: agents.pausedAt })
+ .from(agents);
+ expect(agentAfterHardStop).toMatchObject({ status: "paused", pauseReason: "budget" });
+ expect(agentAfterHardStop?.pausedAt).toBeInstanceOf(Date);
+ expect(cancelWorkForScope).toHaveBeenCalledTimes(2);
+ expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId });
+
+ const block = await service.getInvocationBlock(companyId, agentId);
+ expect(block).toEqual({
+ scopeType: "agent",
+ scopeId: agentId,
+ scopeName: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",
+ reason: "Agent is paused because its budget hard-stop was reached.",
+ });
+
+ const telemetryCalls = mockLogActivity.mock.calls.map(([, input]) => input);
+ expect(telemetryCalls.filter((call) => call.action === "budget.soft_threshold_crossed")).toHaveLength(1);
+ expect(telemetryCalls.filter((call) => call.action === "budget.hard_threshold_crossed")).toHaveLength(1);
+ expect(telemetryCalls).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ action: "budget.soft_threshold_crossed",
+ entityType: "budget_incident",
+ details: expect.objectContaining({
+ scopeType: "agent",
+ scopeId: agentId,
+ amountObserved: 80,
+ amountLimit: 100,
+ }),
+ }),
+ expect.objectContaining({
+ action: "budget.hard_threshold_crossed",
+ entityType: "budget_incident",
+ details: expect.objectContaining({
+ scopeType: "agent",
+ scopeId: agentId,
+ amountObserved: 105,
+ amountLimit: 100,
+ approvalId: approval!.id,
+ }),
+ }),
+ ]),
+ );
+ for (const call of telemetryCalls) {
+ expect(JSON.stringify(call.details)).not.toContain("SECRET_TOKEN_SHOULD_NOT_LEAK");
+ expect(call.details).not.toHaveProperty("prompt");
+ expect(call.details).not.toHaveProperty("message");
+ }
+ });
+
+ it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => {
+ const { companyId, agentId, projectId } = await createBudgetFixture();
+ const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
+ const service = budgetService(db, { cancelWorkForScope });
+ await db.insert(budgetPolicies).values({
+ companyId,
+ scopeType: "project",
+ scopeId: projectId,
+ metric: "billed_cents",
+ windowKind: "lifetime",
+ amount: 100,
+ warnPercent: 75,
+ hardStopEnabled: true,
+ notifyEnabled: true,
+ isActive: true,
+ });
+
+ const event = await insertCostEvent({ companyId, agentId, projectId, costCents: 125 });
+ await service.evaluateCostEvent(event);
+ await service.evaluateCostEvent(event);
+
+ const incidentRows = await db
+ .select()
+ .from(budgetIncidents);
+ expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);
+ const hardIncident = incidentRows.find((incident) => incident.thresholdType === "hard")!;
+ expect(hardIncident).toMatchObject({
+ companyId,
+ scopeType: "project",
+ scopeId: projectId,
+ amountLimit: 100,
+ amountObserved: 125,
+ status: "open",
+ });
+
+ const [projectAfterHardStop] = await db
+ .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })
+ .from(projects);
+ expect(projectAfterHardStop?.pauseReason).toBe("budget");
+ expect(projectAfterHardStop?.pausedAt).toBeInstanceOf(Date);
+ expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId });
+
+ const overviewWhileBlocked = await service.overview(companyId);
+ expect(overviewWhileBlocked.pausedProjectCount).toBe(1);
+ expect(overviewWhileBlocked.pendingApprovalCount).toBe(1);
+ expect(overviewWhileBlocked.policies[0]).toMatchObject({
+ scopeType: "project",
+ scopeId: projectId,
+ amount: 100,
+ observedAmount: 125,
+ remainingAmount: 0,
+ utilizationPercent: 125,
+ status: "hard_stop",
+ paused: true,
+ pauseReason: "budget",
+ });
+ expect(overviewWhileBlocked.activeIncidents).toHaveLength(1);
+
+ await expect(
+ service.resolveIncident(
+ companyId,
+ hardIncident.id,
+ { action: "raise_budget_and_resume", amount: 125 },
+ "board-user",
+ ),
+ ).rejects.toThrow("New budget must exceed current observed spend");
+
+ expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toEqual({
+ scopeType: "project",
+ scopeId: projectId,
+ scopeName: "Budget Project",
+ reason: "Project cannot start work because its budget hard-stop is still exceeded.",
+ });
+
+ const resolved = await service.resolveIncident(
+ companyId,
+ hardIncident.id,
+ { action: "raise_budget_and_resume", amount: 175, decisionNote: "Approved release-gate budget raise." },
+ "board-user",
+ );
+ expect(resolved).toMatchObject({ status: "resolved", approvalStatus: "approved" });
+
+ const [projectAfterResume] = await db
+ .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })
+ .from(projects);
+ expect(projectAfterResume).toEqual({ pauseReason: null, pausedAt: null });
+ expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toBeNull();
+
+ const overviewAfterResume = await service.overview(companyId);
+ expect(overviewAfterResume.pausedProjectCount).toBe(0);
+ expect(overviewAfterResume.pendingApprovalCount).toBe(0);
+ expect(overviewAfterResume.policies[0]).toMatchObject({
+ scopeType: "project",
+ scopeId: projectId,
+ amount: 175,
+ observedAmount: 125,
+ remainingAmount: 50,
+ utilizationPercent: expect.closeTo(71.43, 2),
+ status: "ok",
+ paused: false,
+ pauseReason: null,
+ });
+ expect(overviewAfterResume.activeIncidents).toHaveLength(0);
+ });
+});
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index a340df5059..22154cb9b3 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -917,6 +917,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, issueId };
}
+ async function seedIdleTimerAgentFixture() {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix,
+ requireBoardApprovalForNewAgents: false,
+ });
+
+ await db.insert(agents).values({
+ id: agentId,
+ companyId,
+ name: "CodexCoder",
+ role: "engineer",
+ status: "idle",
+ adapterType: "codex_local",
+ adapterConfig: {},
+ runtimeConfig: {
+ heartbeat: {
+ enabled: true,
+ intervalSec: 60,
+ wakeOnDemand: true,
+ skipTimerWhenNoActionableWork: true,
+ },
+ },
+ permissions: {},
+ });
+
+ return { companyId, agentId };
+ }
+
async function expectSourceScopedStrandedRecoveryAction(input: {
companyId: string;
agentId: string;
@@ -1156,6 +1190,46 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(wakeup?.status).toBe("claimed");
});
+ it("skips generic timer wakes without invoking an adapter when no assigned work is actionable", async () => {
+ const { companyId, agentId } = await seedIdleTimerAgentFixture();
+ const heartbeat = heartbeatService(db);
+
+ const run = await heartbeat.wakeup(agentId, {
+ source: "timer",
+ triggerDetail: "system",
+ reason: "heartbeat_timer",
+ requestedByActorType: "system",
+ requestedByActorId: "heartbeat_scheduler",
+ contextSnapshot: {
+ source: "scheduler",
+ reason: "interval_elapsed",
+ now: "2026-03-19T00:00:00.000Z",
+ },
+ });
+
+ expect(run).toBeNull();
+ expect(mockAdapterExecute).not.toHaveBeenCalled();
+
+ const requests = await db
+ .select()
+ .from(agentWakeupRequests)
+ .where(eq(agentWakeupRequests.agentId, agentId));
+ expect(requests).toHaveLength(1);
+ expect(requests[0]).toMatchObject({
+ companyId,
+ source: "timer",
+ reason: "heartbeat.timer.no_actionable_work",
+ status: "skipped",
+ error: null,
+ });
+
+ const runs = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.agentId, agentId));
+ expect(runs).toHaveLength(0);
+ });
+
it("queues exactly one retry when the recorded local pid is dead", async () => {
const { agentId, runId, issueId } = await seedRunFixture({
agentStatus: "idle",
diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
index b560d82f22..32611ab4fa 100644
--- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
+++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
@@ -394,7 +394,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
- it("runs generic timer wakes by default for proactive agents without assigned issue work", async () => {
+ it("allows legacy generic timer wakes by default when no skip policy is set", async () => {
const { agentId } = await seedCompanyAndAgent({
heartbeatConfig: {
enabled: true,
@@ -408,7 +408,24 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(run).not.toBeNull();
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
+ expect(countExecuteCallsForRun(run!.id)).toBe(1);
+ });
+
+ it("allows explicit proactive generic timer wakes without assigned issue work", async () => {
+ const { agentId } = await seedCompanyAndAgent({
+ heartbeatConfig: {
+ enabled: true,
+ skipTimerWhenNoActionableWork: false,
+ },
+ });
+ const run = await heartbeat.wakeup(agentId, {
+ source: "timer",
+ triggerDetail: "schedule",
+ });
+
+ expect(run).not.toBeNull();
+ await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
diff --git a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts b/server/src/__tests__/issue-stale-execution-lock-routes.test.ts
index 1f91fa278b..944948ec03 100644
--- a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts
+++ b/server/src/__tests__/issue-stale-execution-lock-routes.test.ts
@@ -290,6 +290,67 @@ describeEmbeddedPostgres("stale issue execution lock routes", () => {
expect(res.body?.error).toBe("Issue run ownership conflict");
});
+ it("preserves live checkout ownership on checkout conflicts without retry side effects", async () => {
+ const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns();
+ const contenderRunId = randomUUID();
+ const issueId = randomUUID();
+ await db.insert(heartbeatRuns).values({
+ id: contenderRunId,
+ companyId,
+ agentId,
+ status: "running",
+ invocationSource: "assignment",
+ startedAt: new Date(),
+ });
+ await db.insert(issues).values({
+ id: issueId,
+ companyId,
+ title: "Live checkout race",
+ status: "in_progress",
+ priority: "high",
+ assigneeAgentId: agentId,
+ checkoutRunId: currentRunId,
+ executionRunId: currentRunId,
+ executionAgentNameKey: "codexcoder",
+ executionLockedAt: new Date(),
+ });
+
+ const res = await request(createApp(agentActor(companyId, agentId, contenderRunId)))
+ .post(`/api/issues/${issueId}/checkout`)
+ .send({
+ agentId,
+ expectedStatuses: ["todo", "backlog", "blocked", "in_review"],
+ });
+
+ expect(res.status, JSON.stringify(res.body)).toBe(409);
+ expect(res.body).toMatchObject({
+ error: "Issue checkout conflict",
+ });
+
+ const row = await db
+ .select({
+ status: issues.status,
+ assigneeAgentId: issues.assigneeAgentId,
+ checkoutRunId: issues.checkoutRunId,
+ executionRunId: issues.executionRunId,
+ })
+ .from(issues)
+ .where(eq(issues.id, issueId))
+ .then((rows) => rows[0]);
+ expect(row).toEqual({
+ status: "in_progress",
+ assigneeAgentId: agentId,
+ checkoutRunId: currentRunId,
+ executionRunId: currentRunId,
+ });
+
+ const checkoutActivity = await db
+ .select()
+ .from(activityLog)
+ .where(eq(activityLog.action, "issue.checked_out"));
+ expect(checkoutActivity).toHaveLength(0);
+ });
+
it("restricts admin force-release to board users with company access and writes an audit event", async () => {
const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns();
const issueId = randomUUID();
diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts
index 84d6663e12..e281dcb51f 100644
--- a/server/src/services/budgets.ts
+++ b/server/src/services/budgets.ts
@@ -365,7 +365,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
),
)
.then((rows) => rows[0] ?? null);
- if (existing) return existing;
+ if (existing) return { incident: existing, created: false };
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const payload = buildApprovalPayload({
@@ -392,7 +392,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
.then((rows) => rows[0] ?? null)
: null;
- return db
+ const incident = await db
.insert(budgetIncidents)
.values({
companyId: policy.companyId,
@@ -411,6 +411,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
})
.returning()
.then((rows) => rows[0] ?? null);
+ return incident ? { incident, created: true } : null;
}
async function resolveOpenSoftIncidents(policyId: string) {
@@ -671,14 +672,14 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
if (policy.notifyEnabled && observedAmount >= softThreshold) {
const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount);
- if (softIncident) {
+ if (softIncident?.created) {
await logActivity(db, {
companyId: policy.companyId,
actorType: "system",
actorId: "budget_service",
action: "budget.soft_threshold_crossed",
entityType: "budget_incident",
- entityId: softIncident.id,
+ entityId: softIncident.incident.id,
details: {
scopeType: policy.scopeType,
scopeId: policy.scopeId,
@@ -693,20 +694,20 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
await resolveOpenSoftIncidents(policy.id);
const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount);
await pauseAndCancelScopeForBudget(policy);
- if (hardIncident) {
+ if (hardIncident?.created) {
await logActivity(db, {
companyId: policy.companyId,
actorType: "system",
actorId: "budget_service",
action: "budget.hard_threshold_crossed",
entityType: "budget_incident",
- entityId: hardIncident.id,
+ entityId: hardIncident.incident.id,
details: {
scopeType: policy.scopeType,
scopeId: policy.scopeId,
amountObserved: observedAmount,
amountLimit: policy.amount,
- approvalId: hardIncident.approvalId ?? null,
+ approvalId: hardIncident.incident.approvalId ?? null,
},
});
}