The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
[codex] Enforce backend execution release gates (#9089)
Dottaef617bee5c15
diff --git a/server/src/__tests__/budgets-service.test.ts b/server/src/__tests__/budgets-service.test.tsindex b85050c54b..8a67682726 100644--- a/server/src/__tests__/budgets-service.test.ts+++ b/server/src/__tests__/budgets-service.test.ts@@ -1,5 +1,20 @@-import { beforeEach, describe, expect, it, vi } from "vitest";+import { randomUUID } from "node:crypto";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import {+ agents,+ approvals,+ budgetIncidents,+ budgetPolicies,+ companies,+ costEvents,+ createDb,+ projects,+} from "@paperclipai/db";import { budgetService } from "../services/budgets.ts";+import {+ getEmbeddedPostgresTestSupport,+ startEmbeddedPostgresTestDatabase,+} from "./helpers/embedded-postgres.js";const mockLogActivity = vi.hoisted(() => vi.fn());@@ -309,3 +324,317 @@ describe("budgetService", () => {);});});++const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;++describeEmbeddedPostgres("budgetService release gate enforcement", () => {+ let db!: ReturnType<typeof createDb>;+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;++ beforeAll(async () => {+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-");+ db = createDb(tempDb.connectionString);+ }, 20_000);++ afterEach(async () => {+ await db.delete(budgetIncidents);+ await db.delete(approvals);+ await db.delete(budgetPolicies);+ await db.delete(costEvents);+ await db.delete(projects);+ await db.delete(agents);+ await db.delete(companies);+ mockLogActivity.mockClear();+ });++ afterAll(async () => {+ await tempDb?.cleanup();+ });++ async function createBudgetFixture() {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const projectId = randomUUID();++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,+ requireBoardApprovalForNewAgents: false,+ });+ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",+ role: "engineer",+ status: "active",+ adapterType: "codex_local",+ adapterConfig: {},+ runtimeConfig: {},+ permissions: {},+ });+ await db.insert(projects).values({+ id: projectId,+ companyId,+ name: "Budget Project",+ status: "in_progress",+ });++ return { companyId, agentId, projectId };+ }++ async function insertCostEvent(input: {+ companyId: string;+ agentId: string;+ projectId?: string | null;+ costCents: number;+ occurredAt?: Date;+ }) {+ const [event] = await db+ .insert(costEvents)+ .values({+ companyId: input.companyId,+ agentId: input.agentId,+ projectId: input.projectId ?? null,+ provider: "openai",+ biller: "openai",+ billingType: "metered_api",+ model: "gpt-5-release-gate",+ inputTokens: 100,+ cachedInputTokens: 10,+ outputTokens: 20,+ costCents: input.costCents,+ occurredAt: input.occurredAt ?? new Date(),+ })+ .returning();++ return event!;+ }++ it("raises one soft incident per window before hard-stopping and safely logging agent telemetry", async () => {+ const { companyId, agentId } = await createBudgetFixture();+ const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);+ const service = budgetService(db, { cancelWorkForScope });+ const [policy] = await db+ .insert(budgetPolicies)+ .values({+ companyId,+ scopeType: "agent",+ scopeId: agentId,+ metric: "billed_cents",+ windowKind: "calendar_month_utc",+ amount: 100,+ warnPercent: 80,+ hardStopEnabled: true,+ notifyEnabled: true,+ isActive: true,+ })+ .returning();++ const softEvent = await insertCostEvent({ companyId, agentId, costCents: 80 });+ await service.evaluateCostEvent(softEvent);+ await service.evaluateCostEvent(softEvent);++ let incidentRows = await db+ .select()+ .from(budgetIncidents);+ expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);+ expect(incidentRows[0]).toMatchObject({+ companyId,+ policyId: policy!.id,+ scopeType: "agent",+ scopeId: agentId,+ thresholdType: "soft",+ amountLimit: 100,+ amountObserved: 80,+ approvalId: null,+ status: "open",+ });++ const [agentBeforeHardStop] = await db+ .select({ status: agents.status, pauseReason: agents.pauseReason })+ .from(agents);+ expect(agentBeforeHardStop).toEqual({ status: "active", pauseReason: null });++ const hardEvent = await insertCostEvent({ companyId, agentId, costCents: 25 });+ await service.evaluateCostEvent(hardEvent);+ await service.evaluateCostEvent(hardEvent);++ incidentRows = await db+ .select()+ .from(budgetIncidents);+ expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);+ expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);+ expect(incidentRows.find((incident) => incident.thresholdType === "soft")).toMatchObject({+ status: "resolved",+ });+ expect(incidentRows.find((incident) => incident.thresholdType === "hard")).toMatchObject({+ amountLimit: 100,+ amountObserved: 105,+ status: "open",+ });++ const [approval] = await db.select().from(approvals);+ expect(approval).toMatchObject({+ companyId,+ type: "budget_override_required",+ status: "pending",+ });++ const [agentAfterHardStop] = await db+ .select({ status: agents.status, pauseReason: agents.pauseReason, pausedAt: agents.pausedAt })+ .from(agents);+ expect(agentAfterHardStop).toMatchObject({ status: "paused", pauseReason: "budget" });+ expect(agentAfterHardStop?.pausedAt).toBeInstanceOf(Date);+ expect(cancelWorkForScope).toHaveBeenCalledTimes(2);+ expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId });++ const block = await service.getInvocationBlock(companyId, agentId);+ expect(block).toEqual({+ scopeType: "agent",+ scopeId: agentId,+ scopeName: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",+ reason: "Agent is paused because its budget hard-stop was reached.",+ });++ const telemetryCalls = mockLogActivity.mock.calls.map(([, input]) => input);+ expect(telemetryCalls.filter((call) => call.action === "budget.soft_threshold_crossed")).toHaveLength(1);+ expect(telemetryCalls.filter((call) => call.action === "budget.hard_threshold_crossed")).toHaveLength(1);+ expect(telemetryCalls).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ action: "budget.soft_threshold_crossed",+ entityType: "budget_incident",+ details: expect.objectContaining({+ scopeType: "agent",+ scopeId: agentId,+ amountObserved: 80,+ amountLimit: 100,+ }),+ }),+ expect.objectContaining({+ action: "budget.hard_threshold_crossed",+ entityType: "budget_incident",+ details: expect.objectContaining({+ scopeType: "agent",+ scopeId: agentId,+ amountObserved: 105,+ amountLimit: 100,+ approvalId: approval!.id,+ }),+ }),+ ]),+ );+ for (const call of telemetryCalls) {+ expect(JSON.stringify(call.details)).not.toContain("SECRET_TOKEN_SHOULD_NOT_LEAK");+ expect(call.details).not.toHaveProperty("prompt");+ expect(call.details).not.toHaveProperty("message");+ }+ });++ it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => {+ const { companyId, agentId, projectId } = await createBudgetFixture();+ const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);+ const service = budgetService(db, { cancelWorkForScope });+ await db.insert(budgetPolicies).values({+ companyId,+ scopeType: "project",+ scopeId: projectId,+ metric: "billed_cents",+ windowKind: "lifetime",+ amount: 100,+ warnPercent: 75,+ hardStopEnabled: true,+ notifyEnabled: true,+ isActive: true,+ });++ const event = await insertCostEvent({ companyId, agentId, projectId, costCents: 125 });+ await service.evaluateCostEvent(event);+ await service.evaluateCostEvent(event);++ const incidentRows = await db+ .select()+ .from(budgetIncidents);+ expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);+ const hardIncident = incidentRows.find((incident) => incident.thresholdType === "hard")!;+ expect(hardIncident).toMatchObject({+ companyId,+ scopeType: "project",+ scopeId: projectId,+ amountLimit: 100,+ amountObserved: 125,+ status: "open",+ });++ const [projectAfterHardStop] = await db+ .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })+ .from(projects);+ expect(projectAfterHardStop?.pauseReason).toBe("budget");+ expect(projectAfterHardStop?.pausedAt).toBeInstanceOf(Date);+ expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId });++ const overviewWhileBlocked = await service.overview(companyId);+ expect(overviewWhileBlocked.pausedProjectCount).toBe(1);+ expect(overviewWhileBlocked.pendingApprovalCount).toBe(1);+ expect(overviewWhileBlocked.policies[0]).toMatchObject({+ scopeType: "project",+ scopeId: projectId,+ amount: 100,+ observedAmount: 125,+ remainingAmount: 0,+ utilizationPercent: 125,+ status: "hard_stop",+ paused: true,+ pauseReason: "budget",+ });+ expect(overviewWhileBlocked.activeIncidents).toHaveLength(1);++ await expect(+ service.resolveIncident(+ companyId,+ hardIncident.id,+ { action: "raise_budget_and_resume", amount: 125 },+ "board-user",+ ),+ ).rejects.toThrow("New budget must exceed current observed spend");++ expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toEqual({+ scopeType: "project",+ scopeId: projectId,+ scopeName: "Budget Project",+ reason: "Project cannot start work because its budget hard-stop is still exceeded.",+ });++ const resolved = await service.resolveIncident(+ companyId,+ hardIncident.id,+ { action: "raise_budget_and_resume", amount: 175, decisionNote: "Approved release-gate budget raise." },+ "board-user",+ );+ expect(resolved).toMatchObject({ status: "resolved", approvalStatus: "approved" });++ const [projectAfterResume] = await db+ .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })+ .from(projects);+ expect(projectAfterResume).toEqual({ pauseReason: null, pausedAt: null });+ expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toBeNull();++ const overviewAfterResume = await service.overview(companyId);+ expect(overviewAfterResume.pausedProjectCount).toBe(0);+ expect(overviewAfterResume.pendingApprovalCount).toBe(0);+ expect(overviewAfterResume.policies[0]).toMatchObject({+ scopeType: "project",+ scopeId: projectId,+ amount: 175,+ observedAmount: 125,+ remainingAmount: 50,+ utilizationPercent: expect.closeTo(71.43, 2),+ status: "ok",+ paused: false,+ pauseReason: null,+ });+ expect(overviewAfterResume.activeIncidents).toHaveLength(0);+ });+});diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex a340df5059..22154cb9b3 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -917,6 +917,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {return { companyId, agentId, issueId };}+ async function seedIdleTimerAgentFixture() {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix,+ requireBoardApprovalForNewAgents: false,+ });++ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "CodexCoder",+ role: "engineer",+ status: "idle",+ adapterType: "codex_local",+ adapterConfig: {},+ runtimeConfig: {+ heartbeat: {+ enabled: true,+ intervalSec: 60,+ wakeOnDemand: true,+ skipTimerWhenNoActionableWork: true,+ },+ },+ permissions: {},+ });++ return { companyId, agentId };+ }+async function expectSourceScopedStrandedRecoveryAction(input: {companyId: string;agentId: string;@@ -1156,6 +1190,46 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(wakeup?.status).toBe("claimed");});+ it("skips generic timer wakes without invoking an adapter when no assigned work is actionable", async () => {+ const { companyId, agentId } = await seedIdleTimerAgentFixture();+ const heartbeat = heartbeatService(db);++ const run = await heartbeat.wakeup(agentId, {+ source: "timer",+ triggerDetail: "system",+ reason: "heartbeat_timer",+ requestedByActorType: "system",+ requestedByActorId: "heartbeat_scheduler",+ contextSnapshot: {+ source: "scheduler",+ reason: "interval_elapsed",+ now: "2026-03-19T00:00:00.000Z",+ },+ });++ expect(run).toBeNull();+ expect(mockAdapterExecute).not.toHaveBeenCalled();++ const requests = await db+ .select()+ .from(agentWakeupRequests)+ .where(eq(agentWakeupRequests.agentId, agentId));+ expect(requests).toHaveLength(1);+ expect(requests[0]).toMatchObject({+ companyId,+ source: "timer",+ reason: "heartbeat.timer.no_actionable_work",+ status: "skipped",+ error: null,+ });++ const runs = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.agentId, agentId));+ expect(runs).toHaveLength(0);+ });+it("queues exactly one retry when the recorded local pid is dead", async () => {const { agentId, runId, issueId } = await seedRunFixture({agentStatus: "idle",diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.tsindex b560d82f22..32611ab4fa 100644--- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts+++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts@@ -394,7 +394,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {expect(countExecuteCallsForRun(run!.id)).toBe(1);});- it("runs generic timer wakes by default for proactive agents without assigned issue work", async () => {+ it("allows legacy generic timer wakes by default when no skip policy is set", async () => {const { agentId } = await seedCompanyAndAgent({heartbeatConfig: {enabled: true,@@ -408,7 +408,24 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {expect(run).not.toBeNull();await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);+ expect(countExecuteCallsForRun(run!.id)).toBe(1);+ });++ it("allows explicit proactive generic timer wakes without assigned issue work", async () => {+ const { agentId } = await seedCompanyAndAgent({+ heartbeatConfig: {+ enabled: true,+ skipTimerWhenNoActionableWork: false,+ },+ });+ const run = await heartbeat.wakeup(agentId, {+ source: "timer",+ triggerDetail: "schedule",+ });++ expect(run).not.toBeNull();+ await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);expect(countExecuteCallsForRun(run!.id)).toBe(1);});diff --git a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts b/server/src/__tests__/issue-stale-execution-lock-routes.test.tsindex 1f91fa278b..944948ec03 100644--- a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts+++ b/server/src/__tests__/issue-stale-execution-lock-routes.test.ts@@ -290,6 +290,67 @@ describeEmbeddedPostgres("stale issue execution lock routes", () => {expect(res.body?.error).toBe("Issue run ownership conflict");});+ it("preserves live checkout ownership on checkout conflicts without retry side effects", async () => {+ const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns();+ const contenderRunId = randomUUID();+ const issueId = randomUUID();+ await db.insert(heartbeatRuns).values({+ id: contenderRunId,+ companyId,+ agentId,+ status: "running",+ invocationSource: "assignment",+ startedAt: new Date(),+ });+ await db.insert(issues).values({+ id: issueId,+ companyId,+ title: "Live checkout race",+ status: "in_progress",+ priority: "high",+ assigneeAgentId: agentId,+ checkoutRunId: currentRunId,+ executionRunId: currentRunId,+ executionAgentNameKey: "codexcoder",+ executionLockedAt: new Date(),+ });++ const res = await request(createApp(agentActor(companyId, agentId, contenderRunId)))+ .post(`/api/issues/${issueId}/checkout`)+ .send({+ agentId,+ expectedStatuses: ["todo", "backlog", "blocked", "in_review"],+ });++ expect(res.status, JSON.stringify(res.body)).toBe(409);+ expect(res.body).toMatchObject({+ error: "Issue checkout conflict",+ });++ const row = await db+ .select({+ status: issues.status,+ assigneeAgentId: issues.assigneeAgentId,+ checkoutRunId: issues.checkoutRunId,+ executionRunId: issues.executionRunId,+ })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0]);+ expect(row).toEqual({+ status: "in_progress",+ assigneeAgentId: agentId,+ checkoutRunId: currentRunId,+ executionRunId: currentRunId,+ });++ const checkoutActivity = await db+ .select()+ .from(activityLog)+ .where(eq(activityLog.action, "issue.checked_out"));+ expect(checkoutActivity).toHaveLength(0);+ });+it("restricts admin force-release to board users with company access and writes an audit event", async () => {const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns();const issueId = randomUUID();diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.tsindex 84d6663e12..e281dcb51f 100644--- a/server/src/services/budgets.ts+++ b/server/src/services/budgets.ts@@ -365,7 +365,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {),).then((rows) => rows[0] ?? null);- if (existing) return existing;+ if (existing) return { incident: existing, created: false };const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);const payload = buildApprovalPayload({@@ -392,7 +392,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {.then((rows) => rows[0] ?? null): null;- return db+ const incident = await db.insert(budgetIncidents).values({companyId: policy.companyId,@@ -411,6 +411,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {}).returning().then((rows) => rows[0] ?? null);+ return incident ? { incident, created: true } : null;}async function resolveOpenSoftIncidents(policyId: string) {@@ -671,14 +672,14 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {if (policy.notifyEnabled && observedAmount >= softThreshold) {const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount);- if (softIncident) {+ if (softIncident?.created) {await logActivity(db, {companyId: policy.companyId,actorType: "system",actorId: "budget_service",action: "budget.soft_threshold_crossed",entityType: "budget_incident",- entityId: softIncident.id,+ entityId: softIncident.incident.id,details: {scopeType: policy.scopeType,scopeId: policy.scopeId,@@ -693,20 +694,20 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {await resolveOpenSoftIncidents(policy.id);const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount);await pauseAndCancelScopeForBudget(policy);- if (hardIncident) {+ if (hardIncident?.created) {await logActivity(db, {companyId: policy.companyId,actorType: "system",actorId: "budget_service",action: "budget.hard_threshold_crossed",entityType: "budget_incident",- entityId: hardIncident.id,+ entityId: hardIncident.incident.id,details: {scopeType: policy.scopeType,scopeId: policy.scopeId,amountObserved: observedAmount,amountLimit: policy.amount,- approvalId: hardIncident.approvalId ?? null,+ approvalId: hardIncident.incident.approvalId ?? null,},});}