The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
[codex] Harden issue recovery reliability (#4875)
Dottaad5432feced3
diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.tsindex 0bfa209caa..2ac8658da5 100644--- a/server/src/__tests__/agent-live-run-routes.test.ts+++ b/server/src/__tests__/agent-live-run-routes.test.ts@@ -10,7 +10,6 @@ const mockHeartbeatService = vi.hoisted(() => ({buildRunOutputSilence: vi.fn(),getRunIssueSummary: vi.fn(),getActiveRunIssueSummaryForAgent: vi.fn(),- buildRunOutputSilence: vi.fn(),getRunLogAccess: vi.fn(),readLog: vi.fn(),}));@@ -71,7 +70,7 @@ function registerModuleMocks() {}));}-async function createApp() {+async function createApp(db: Record<string, unknown> = {}) {const [{ agentRoutes }, { errorHandler }] = await Promise.all([vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),@@ -88,11 +87,32 @@ async function createApp() {};next();});- app.use("/api", agentRoutes({} as any));+ app.use("/api", agentRoutes(db as any));app.use(errorHandler);return app;}+function createLiveRunsDbStub(rows: Array<Record<string, unknown>>) {+ const limit = vi.fn(async (value: number) => rows.slice(0, value));+ const orderedQuery = {+ limit,+ then: (resolve: (value: Array<Record<string, unknown>>) => unknown) => Promise.resolve(rows).then(resolve),+ };+ const query = {+ from: vi.fn().mockReturnThis(),+ innerJoin: vi.fn().mockReturnThis(),+ where: vi.fn().mockReturnThis(),+ orderBy: vi.fn().mockReturnValue(orderedQuery),+ };++ return {+ db: {+ select: vi.fn().mockReturnValue(query),+ },+ limit,+ };+}+async function requestApp(app: express.Express,buildRequest: (baseUrl: string) => request.Test,@@ -284,4 +304,81 @@ describe("agent live run routes", () => {nextOffset: 5,});});++ it("caps company live run polling by default", async () => {+ const rows = Array.from({ length: 75 }, (_, index) => ({+ id: `run-${index}`,+ companyId: "company-1",+ status: "running",+ invocationSource: "on_demand",+ triggerDetail: "manual",+ startedAt: new Date("2026-04-10T09:30:00.000Z"),+ finishedAt: null,+ createdAt: new Date(`2026-04-10T09:${String(index % 60).padStart(2, "0")}:00.000Z`),+ agentId: "agent-1",+ agentName: "Builder",+ adapterType: "codex_local",+ logBytes: 0,+ livenessState: "healthy",+ livenessReason: null,+ continuationAttempt: 0,+ lastUsefulActionAt: null,+ nextAction: null,+ lastOutputAt: null,+ lastOutputSeq: null,+ lastOutputStream: null,+ lastOutputBytes: 0,+ processStartedAt: null,+ issueId: "issue-1",+ }));+ const { db, limit } = createLiveRunsDbStub(rows);++ const res = await requestApp(+ await createApp(db),+ (baseUrl) => request(baseUrl).get("/api/companies/company-1/live-runs"),+ );++ expect(res.status, JSON.stringify(res.body)).toBe(200);+ expect(limit).toHaveBeenCalledWith(50);+ expect(res.body).toHaveLength(50);+ expect(mockHeartbeatService.buildRunOutputSilence).toHaveBeenCalledTimes(50);+ });++ it("treats explicit zero live run limits as the capped default", async () => {+ const rows = Array.from({ length: 75 }, (_, index) => ({+ id: `run-${index}`,+ companyId: "company-1",+ status: "running",+ invocationSource: "on_demand",+ triggerDetail: "manual",+ startedAt: new Date("2026-04-10T09:30:00.000Z"),+ finishedAt: null,+ createdAt: new Date(`2026-04-10T09:${String(index % 60).padStart(2, "0")}:00.000Z`),+ agentId: "agent-1",+ agentName: "Builder",+ adapterType: "codex_local",+ logBytes: 0,+ livenessState: "healthy",+ livenessReason: null,+ continuationAttempt: 0,+ lastUsefulActionAt: null,+ nextAction: null,+ lastOutputAt: null,+ lastOutputSeq: null,+ lastOutputStream: null,+ lastOutputBytes: 0,+ processStartedAt: null,+ issueId: "issue-1",+ }));+ const { db, limit } = createLiveRunsDbStub(rows);++ const res = await requestApp(+ await createApp(db),+ (baseUrl) => request(baseUrl).get("/api/companies/company-1/live-runs?limit=0&minCount=0"),+ );++ expect(res.status, JSON.stringify(res.body)).toBe(200);+ expect(limit).toHaveBeenCalledWith(50);+ expect(res.body).toHaveLength(50);+ });});diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.tsindex 369bb05c2b..10ad103cd4 100644--- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts+++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts@@ -289,10 +289,23 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {const heartbeat = heartbeatService(db);const first = await heartbeat.reconcileIssueGraphLiveness();- const second = await heartbeat.reconcileIssueGraphLiveness();expect(first.escalationsCreated).toBe(1);+ const [sourceAfterFirst] = await db+ .select({ updatedAt: issues.updatedAt })+ .from(issues)+ .where(eq(issues.id, blockedIssueId));+ const eventsAfterFirst = await db.select().from(activityLog).where(eq(activityLog.companyId, companyId));+ expect(eventsAfterFirst.filter((event) => event.action === "issue.blockers.updated")).toHaveLength(1);++ const second = await heartbeat.reconcileIssueGraphLiveness();+expect(second.escalationsCreated).toBe(0);+ const [sourceAfterSecond] = await db+ .select({ updatedAt: issues.updatedAt })+ .from(issues)+ .where(eq(issues.id, blockedIssueId));+ expect(sourceAfterSecond?.updatedAt.getTime()).toBe(sourceAfterFirst?.updatedAt.getTime());const escalations = await db.select()@@ -345,7 +358,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {projectWorkspaceSourceIssueId: blockerIssueId,},});- expect(events.some((event) => event.action === "issue.blockers.updated")).toBe(true);+ expect(events.filter((event) => event.action === "issue.blockers.updated")).toHaveLength(1);});it("skips budget-blocked direct owners and assigns recovery to the manager fallback", async () => {diff --git a/server/src/__tests__/issue-blocker-attention.test.ts b/server/src/__tests__/issue-blocker-attention.test.tsindex 611921911a..66df695937 100644--- a/server/src/__tests__/issue-blocker-attention.test.ts+++ b/server/src/__tests__/issue-blocker-attention.test.ts@@ -76,6 +76,9 @@ describeEmbeddedPostgres("issue blocker attention", () => {status: string;parentId?: string | null;assigneeAgentId?: string | null;+ originKind?: string | null;+ originId?: string | null;+ originFingerprint?: string | null;}) {const id = input.id ?? randomUUID();await db.insert(issues).values({@@ -87,6 +90,9 @@ describeEmbeddedPostgres("issue blocker attention", () => {priority: "medium",parentId: input.parentId ?? null,assigneeAgentId: input.assigneeAgentId ?? null,+ originKind: input.originKind ?? "manual",+ originId: input.originId ?? null,+ originFingerprint: input.originFingerprint ?? "default",});return id;}@@ -356,6 +362,52 @@ describeEmbeddedPostgres("issue blocker attention", () => {});});+ it("treats open liveness escalation blockers as covered waiting paths", async () => {+ const { companyId, agentId } = await createCompany("PBL");+ const parentId = await insertIssue({ companyId, identifier: "PBL-1", title: "Parent", status: "blocked" });+ const cancelledLeafId = await insertIssue({+ companyId,+ identifier: "PBL-2",+ title: "Cancelled blocker",+ status: "cancelled",+ assigneeAgentId: agentId,+ });+ const incidentKey = [+ "harness_liveness",+ companyId,+ parentId,+ "blocked_by_cancelled_issue",+ cancelledLeafId,+ ].join(":");+ const escalationId = await insertIssue({+ companyId,+ identifier: "PBL-3",+ title: "Liveness escalation",+ status: "todo",+ assigneeAgentId: agentId,+ originKind: "harness_liveness_escalation",+ originId: incidentKey,+ originFingerprint: [+ "harness_liveness_leaf",+ companyId,+ "blocked_by_cancelled_issue",+ cancelledLeafId,+ ].join(":"),+ });+ await block({ companyId, blockerIssueId: cancelledLeafId, blockedIssueId: parentId });+ await block({ companyId, blockerIssueId: escalationId, blockedIssueId: parentId });++ const parent = (await svc.list(companyId, { status: "blocked,todo" })).find((issue) => issue.id === parentId);++ expect(parent?.blockerAttention).toMatchObject({+ state: "covered",+ reason: "active_dependency",+ unresolvedBlockerCount: 2,+ coveredBlockerCount: 2,+ attentionBlockerCount: 0,+ });+ });+it("does not treat a scheduled retry as actively covered work", async () => {const { companyId, agentId } = await createCompany("PBY");const parentId = await insertIssue({ companyId, identifier: "PBY-1", title: "Parent", status: "blocked" });diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.tsindex 3fb95edafd..2dbe8ad825 100644--- a/server/src/routes/agents.ts+++ b/server/src/routes/agents.ts@@ -99,7 +99,8 @@ function readRunLogLimitBytes(value: unknown) {function readLiveRunsQueryInt(value: unknown, max: number, fallback = 0) {const parsed = Number(value);if (!Number.isFinite(parsed)) return fallback;- return Math.max(0, Math.min(max, Math.trunc(parsed)));+ if (parsed <= 0) return fallback;+ return Math.min(max, Math.trunc(parsed));}export function agentRoutes(@@ -2821,8 +2822,8 @@ export function agentRoutes(const companyId = req.params.companyId as string;assertCompanyAccess(req, companyId);- const minCount = readLiveRunsQueryInt(req.query.minCount, 50);- const limit = readLiveRunsQueryInt(req.query.limit, 50);+ const minCount = readLiveRunsQueryInt(req.query.minCount, 50, 50);+ const limit = readLiveRunsQueryInt(req.query.limit, 50, 50);const columns = {id: heartbeatRuns.id,@@ -2862,8 +2863,8 @@ export function agentRoutes().orderBy(desc(heartbeatRuns.createdAt));- const liveRuns = limit > 0 ? await liveRunsQuery.limit(limit) : await liveRunsQuery;- const targetRunCount = limit > 0 ? Math.min(minCount, limit) : minCount;+ const liveRuns = await liveRunsQuery.limit(limit);+ const targetRunCount = Math.min(minCount, limit);if (targetRunCount > 0 && liveRuns.length < targetRunCount) {const activeIds = liveRuns.map((r) => r.id);diff --git a/server/src/services/issues.ts b/server/src/services/issues.tsindex cce66d09c9..8990a73038 100644--- a/server/src/services/issues.ts+++ b/server/src/services/issues.ts@@ -52,6 +52,7 @@ import {issueTreeControlService,type ActiveIssueTreePauseHoldGate,} from "./issue-tree-control.js";+import { parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js";const ALL_ISSUE_STATUSES = ["backlog", "todo", "in_progress", "in_review", "blocked", "done", "cancelled"];const MAX_ISSUE_COMMENT_PAGE_LIMIT = 500;@@ -1174,12 +1175,12 @@ async function listIssueBlockerAttentionMap(}}- const reviewNodeIds = [...nodesById.values()]- .filter((node) => node.status === "in_review")+ const explicitWaitCandidateIds = [...nodesById.values()]+ .filter((node) => node.status !== "done").map((node) => node.id);const explicitWaitingIssueIds = new Set<string>();- if (reviewNodeIds.length > 0) {- for (const chunk of chunkList(reviewNodeIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {+ if (explicitWaitCandidateIds.length > 0) {+ for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {const interactionRows: Array<{ issueId: string }> = await dbOrTx.select({ issueId: issueThreadInteractions.issueId }).from(issueThreadInteractions)@@ -1204,22 +1205,28 @@ async function listIssueBlockerAttentionMap(),);for (const row of approvalRows) explicitWaitingIssueIds.add(row.issueId);+ }- const recoveryRows: Array<{ originId: string | null }> = await dbOrTx- .select({ originId: issues.originId })- .from(issues)- .where(- and(- eq(issues.companyId, companyId),- eq(issues.originKind, BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND),- isNull(issues.hiddenAt),- inArray(issues.originId, chunk),- notInArray(issues.status, BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES),- ),- );- for (const row of recoveryRows) {- if (row.originId) explicitWaitingIssueIds.add(row.originId);- }+ // Recovery rows are intentionally company-wide: a liveness escalation for+ // the same leaf blocker represents an active waiting path even when that+ // blocker is reached through another blocked graph.+ const recoveryRows: Array<{ id: string; originId: string | null }> = await dbOrTx+ .select({ id: issues.id, originId: issues.originId })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, companyId),+ eq(issues.originKind, BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND),+ isNull(issues.hiddenAt),+ notInArray(issues.status, BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES),+ ),+ );+ for (const row of recoveryRows) {+ const parsed = parseIssueGraphLivenessIncidentKey(row.originId);+ if (!parsed || parsed.companyId !== companyId) continue;+ explicitWaitingIssueIds.add(row.id);+ explicitWaitingIssueIds.add(parsed.issueId);+ explicitWaitingIssueIds.add(parsed.leafIssueId);}}@@ -1257,8 +1264,11 @@ async function listIssueBlockerAttentionMap(if (node.status === "done") {return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };}+ if (explicitWaitingIssueIds.has(node.id)) {+ return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };+ }if (node.status === "in_review") {- const hasWaitingPath = activeIssueIds.has(node.id) || Boolean(node.assigneeUserId) || explicitWaitingIssueIds.has(node.id);+ const hasWaitingPath = activeIssueIds.has(node.id) || Boolean(node.assigneeUserId);if (hasWaitingPath) {return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };}diff --git a/server/src/services/recovery/run-liveness-continuations.ts b/server/src/services/recovery/run-liveness-continuations.tsindex 1b4d2cf412..b23625c1a9 100644--- a/server/src/services/recovery/run-liveness-continuations.ts+++ b/server/src/services/recovery/run-liveness-continuations.ts@@ -127,7 +127,6 @@ export function decideRunLivenessContinuation(input: {if (budgetBlocked) {return { kind: "skip", reason: "budget hard stop blocks continuation" };}-const currentAttempt = readContinuationAttempt(run.continuationAttempt);if (currentAttempt >= maxAttempts) {return {diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex fc9fb7e1f6..ec7213515e 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -2250,10 +2250,16 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })}) {const blockerIds = await existingBlockerIssueIds(input.issue.companyId, input.issue.id);const nextBlockerIds = [...new Set([...blockerIds, input.escalationIssueId])];+ const isAlreadyBlockedByEscalation = blockerIds.includes(input.escalationIssueId);+ const isAlreadyBlocked = input.issue.status === "blocked";+ if (isAlreadyBlockedByEscalation && isAlreadyBlocked) {+ return input.issue;+ }+const update: Partial<typeof issues.$inferInsert> & { blockedByIssueIds: string[] } = {blockedByIssueIds: nextBlockerIds,};- if (input.issue.status !== "blocked") {+ if (!isAlreadyBlocked) {update.status = "blocked";}