The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(server): support issue monitor deadlines in idle sleep reports (#16325)
Devin Foley35b3345033c9
diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.mdindex ec87d70985..82f3652fbc 100644--- a/doc/idle-sleep-safety.md+++ b/doc/idle-sleep-safety.md@@ -74,6 +74,41 @@ Verify a full scheduled sleep/wake/execution cycle on staging before enablingthis host capability. The [tests](../server/src/__tests__/idle-sleep-safety.test.ts)cover ownership, future deadlines, changed generations and conservative fallback.+## Passive issues and scheduled monitors++Signed Cloud control callers can request `scheduledWake=2` to include issue+monitoring alongside routines. Earlier hosts requesting `scheduledWake=1`+retain the routine-only rules, including the conservative open-issue check.+Browser administrators and tenant credentials cannot request either extension.++With the second capability, backlog tasks, unassigned todo tasks, explicit+blocked tasks and human-owned tasks may be passive records. Their presence+alone does not require compute. An assigned onboarding first task is also+passive with an idle assignee before its first run, user-authored comment or user-resolved opening+card. This uses the same awaiting-user predicate as recovery; a seeded greeting+does not start work. User input still enters through the host wake path.+Execution locks, automated pending review,+conversation state, unknown state and dependency repair remain blockers. A+blocked task whose blockers have all finished stays awake for reconciliation.+Enabled task watchdogs and unsupported recovery keep their existing checks.+Incoming changes still require the hosting admission and wake path.++An active issue monitor needs a saved future `monitor_next_check_at` matching+its validated execution policy, a company-scoped agent, no outstanding dispatch+claim, and remaining attempts and timeout. Due, malformed, quota-recovery,+unowned and inconsistent monitors keep the workspace awake. Inspection does+not clear schedules, advance attempts or dispatch an agent.++The combined report uses `version: 2` and a scheduled proof with `version: 1`+and `kind: "routines_and_issue_monitors"`. Its deadline is the earliest required+routine or monitor time across all companies. Its generation binds the current+process, idle owner and inspected catalogs. Hosts must acknowledge this known+proof kind, persist the alarm and validate the unchanged owned report before+stopping compute. Existing hosts reject the new kind. Monitor dispatch and+startup catch-up continue through the existing scheduler and atomic claims;+self-hosted scheduling is unchanged. This capability does not cover all future+work in the instance: every other safety check continues to apply.+## Retry historyRetry promotion keeps `scheduled_retry_at` on a run for inspection. That datediff --git a/server/src/__tests__/idle-issue-schedule.test.ts b/server/src/__tests__/idle-issue-schedule.test.tsnew file mode 100644index 0000000000..6406a4b962--- /dev/null+++ b/server/src/__tests__/idle-issue-schedule.test.ts@@ -0,0 +1,187 @@+import { randomUUID } from "node:crypto";+import { eq, sql } from "drizzle-orm";+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";+import { ONBOARDING_FIRST_TASK_ORIGIN_KIND, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";+import { agents, agentApiKeys, companies, createDb, heartbeatRuns, issues, issueComments, issueRelations, issueThreadInteractions, routines, routineTriggers } from "@paperclipai/db";+import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";+import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";++const now = Date.UTC(2020, 0, 1);+const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895";+const owned = (): IdleSleepDrainStatus => ({ ownerId, draining: true, startedAt: new Date(now - 1000),+ expiresAt: new Date(now + 60_000), activeRuns: 0, pendingWakes: 0 });+const emptyLocal = async () => "none" as const;+const present = { version: 1, backgroundWork: "present" };+const none = { version: 1, backgroundWork: "none" };+const support = await getEmbeddedPostgresTestSupport();+if (!support.supported) console.warn(`Skipping issue schedule Postgres tests: ${support.reason}`);+(support.supported ? describe : describe.skip)("idle issue schedule safety", () => {+ let db: ReturnType<typeof createDb>;+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("idle-issue-schedule-");+ db = createDb(database.connectionString);+ }, 60_000);+ afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });+ afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });+ const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);+ const readIssueSchedules = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true, true);+ async function seed() {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const runId = randomUUID();+ await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: `T${companyId.slice(0, 6).toUpperCase()}` });+ await db.insert(agents).values({ id: agentId, companyId, name: "On-demand agent", role: "engineer", status: "idle", adapterType: "process" });+ await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });+ return { companyId, agentId, runId };+ }++ async function scheduledRoutine(at = now + 600_000) {+ const fixture = await seed();+ const [routine] = await db.insert(routines).values({ companyId: fixture.companyId,+ title: "Scheduled fixture", status: "active", assigneeAgentId: fixture.agentId }).returning();+ const [trigger] = await db.insert(routineTriggers).values({ companyId: fixture.companyId,+ routineId: routine!.id, kind: "schedule", cronExpression: "*/15 * * * *", timezone: "UTC",+ nextRunAt: new Date(at), enabled: true, archived: false, setupPending: false }).returning();+ return { ...fixture, routine: routine!, trigger: trigger! };+ }++ async function monitoredIssue(at = now + 600_000) {+ const fixture = await seed();+ const monitor = { nextCheckAt: new Date(at).toISOString(), notes: "Inspect fixture result" };+ const [issue] = await db.insert(issues).values({ companyId: fixture.companyId, title: "Monitor fixture",+ status: "in_progress", assigneeAgentId: fixture.agentId, monitorNextCheckAt: new Date(at),+ executionPolicy: { monitor } }).returning();+ return { ...fixture, issue: issue!, monitor };+ }++ it("allows passive issue records only with the issue-scheduling host capability", async () => {+ const f = await seed();+ await db.insert(issues).values([+ { companyId: f.companyId, title: "Planned task", status: "backlog", assigneeAgentId: f.agentId },+ { companyId: f.companyId, title: "Unassigned task", status: "todo" },+ { companyId: f.companyId, title: "Explicitly blocked task", status: "blocked", assigneeAgentId: f.agentId },+ { companyId: f.companyId, title: "Human review", status: "in_review", assigneeUserId: "fixture-user" },+ ]);+ expect(await readScheduled()).toEqual(present);+ expect(await readIssueSchedules()).toEqual(none);+ expect(await db.select().from(issues)).toHaveLength(4);+ });++ it.each(["todo", "in_progress", "in_review"])("retains an assigned %s task without a proved waiting path", async status => {+ const f = await seed();+ await db.insert(issues).values({ companyId: f.companyId, title: "Unfinished automated task", status, assigneeAgentId: f.agentId });+ expect(await readIssueSchedules()).toEqual(present);+ });++ it.each(["typed", "legacy-typed", "interaction", "run", "origin", "agent-state"])(+ "allows an untouched onboarding task, but retains it after %s activity", async activity => {+ const f = await seed();+ const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "First task",+ status: "todo", assigneeAgentId: f.agentId, originKind: ONBOARDING_FIRST_TASK_ORIGIN_KIND }).returning();+ await db.insert(issueComments).values({ companyId: f.companyId, issueId: issue!.id,+ authorType: "agent", authorAgentId: f.agentId, body: "Seeded greeting" });+ const [card] = await db.insert(issueThreadInteractions).values({ companyId: f.companyId,+ issueId: issue!.id, kind: "request_confirmation", payload: { version: 1, prompt: "Start?" } }).returning();+ expect(await readScheduled()).toEqual(present);+ expect(await readIssueSchedules()).toEqual(none);+ if (activity === "typed" || activity === "legacy-typed") {+ await db.insert(issueComments).values({ companyId: f.companyId, issueId: issue!.id,+ authorType: activity === "typed" ? "user" : null, authorUserId: "fixture-user", body: "Start work" });+ }+ if (activity === "interaction") await db.update(issueThreadInteractions).set({ status: "resolved",+ resolvedByUserId: "fixture-user", resolvedAt: new Date(now) }).where(eq(issueThreadInteractions.id, card!.id));+ if (activity === "run") await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: issue!.id } }).where(eq(heartbeatRuns.id, f.runId));+ if (activity === "origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, issue!.id));+ if (activity === "agent-state") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId));+ expect(await readIssueSchedules()).toEqual(present);+ },+ );++ it("keeps resolved dependency repair awake while allowing an unresolved blocked task", async () => {+ const f = await seed();+ const [blocker] = await db.insert(issues).values({ companyId: f.companyId, title: "Unassigned blocker", status: "todo" }).returning();+ const [dependent] = await db.insert(issues).values({ companyId: f.companyId, title: "Waiting task", status: "blocked", assigneeAgentId: f.agentId }).returning();+ await db.insert(issueRelations).values({ companyId: f.companyId, issueId: blocker!.id, relatedIssueId: dependent!.id, type: "blocks" });+ expect(await readIssueSchedules()).toEqual(none);+ await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, blocker!.id));+ expect(await readIssueSchedules()).toEqual(none);+ await db.update(issues).set({ status: "done" }).where(eq(issues.id, blocker!.id));+ expect(await readIssueSchedules()).toEqual(present);+ });++ it("combines routine and issue monitor deadlines across companies without changing dispatch state", async () => {+ await scheduledRoutine(now + 900_000);+ const f = await monitoredIssue();+ expect(await readScheduled()).toEqual(present);+ const first = await readIssueSchedules();+ expect(first).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {+ version: 1, kind: "routines_and_issue_monitors", nextWakeAt: f.monitor.nextCheckAt } });+ expect((await readIssueSchedules()).scheduledWake).toEqual(first.scheduledWake);+ const earlier = new Date(now + 400_000);+ await db.update(issues).set({ monitorNextCheckAt: earlier, executionPolicy: { monitor: { ...f.monitor, nextCheckAt: earlier.toISOString() } } }).where(eq(issues.id, f.issue.id));+ const edited = await readIssueSchedules();+ expect(edited.scheduledWake?.nextWakeAt).toBe(earlier.toISOString());+ expect(edited.scheduledWake?.generation).not.toBe(first.scheduledWake?.generation);+ expect(await db.select().from(issues).where(eq(issues.id, f.issue.id))).toMatchObject([{ monitorNextCheckAt: earlier, monitorWakeRequestedAt: null, monitorAttemptCount: 0 }]);+ await db.update(issues).set({ status: "blocked", monitorNextCheckAt: null, executionPolicy: null }).where(eq(issues.id, f.issue.id));+ expect(await readIssueSchedules()).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {+ kind: "routines", nextWakeAt: new Date(now + 900_000).toISOString() } });+ });++ it.each(["due", "mismatch", "claimed", "timeout", "exhausted", "negative-attempts", "pending-review", "quota", "unowned"])(+ "retains %s monitor work", async kind => {+ const f = await monitoredIssue();+ const patch: Partial<typeof issues.$inferInsert> = {};+ if (kind === "due") patch.monitorNextCheckAt = new Date(now);+ if (kind === "mismatch") patch.monitorNextCheckAt = new Date(now + 700_000);+ if (kind === "claimed") patch.monitorWakeRequestedAt = new Date(now);+ if (kind === "pending-review") patch.executionState = { status: "pending" };+ if (kind === "unowned") patch.assigneeAgentId = null;+ if (kind === "timeout") patch.executionPolicy = { monitor: { ...f.monitor, timeoutAt: new Date(now + 500_000).toISOString() } };+ if (kind === "exhausted") { patch.monitorAttemptCount = 2; patch.executionPolicy = { monitor: { ...f.monitor, maxAttempts: 2 } }; }+ if (kind === "negative-attempts") patch.monitorAttemptCount = -1;+ if (kind === "quota") patch.executionPolicy = { monitor: { ...f.monitor, serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } };+ await db.update(issues).set(patch).where(eq(issues.id, f.issue.id));+ expect(await readIssueSchedules()).toEqual(present);+ },+ );++ it("retains a cross-company assignee and unknown issue states", async () => {+ const f = await monitoredIssue();+ const other = await seed();+ await db.update(issues).set({ assigneeAgentId: other.agentId }).where(eq(issues.id, f.issue.id));+ expect(await readIssueSchedules()).toEqual(present);+ await db.update(issues).set({ assigneeAgentId: f.agentId, status: "unknown" }).where(eq(issues.id, f.issue.id));+ expect(await readIssueSchedules()).toEqual(present);+ });++ it("retains an unknown assignee state even for a blocked task", async () => {+ const f = await seed();+ await db.insert(issues).values({ companyId: f.companyId, title: "Blocked fixture", status: "blocked", assigneeAgentId: f.agentId });+ await db.execute(sql`UPDATE agents SET status = 'unknown' WHERE id = ${f.agentId}`);+ expect(await readIssueSchedules()).toEqual(present);+ });++ it("refuses monitor sleep if a new run or external credential needs the instance", async () => {+ const f = await monitoredIssue();+ await db.update(heartbeatRuns).set({ status: "queued" }).where(eq(heartbeatRuns.id, f.runId));+ expect(await readIssueSchedules()).toEqual(present);+ await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));+ await db.insert(agentApiKeys).values({ companyId: f.companyId, agentId: f.agentId, name: "Fixture external client", keyHash: "fixture-key-hash" });+ expect(await readIssueSchedules()).toEqual(present);+ });++ it("retains a monitor which becomes due during the owned backup checkpoint", async () => {+ await monitoredIssue(now + 1000);+ let clock = now;+ const backup = async () => {+ const done = beginIdleTrackedWork();+ try { clock += 1000; return true; } finally { done(); }+ };+ expect(await readIdleSleepSafety(db, owned, () => clock, ownerId, emptyLocal, undefined, backup, true, true)).toEqual(present);+ expect(idleWorkSnapshot().active).toBe(0);+ });++});diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.tsindex 23d8c32dd1..14abf47810 100644--- a/server/src/__tests__/instance-settings-routes.test.ts+++ b/server/src/__tests__/instance-settings-routes.test.ts@@ -907,7 +907,7 @@ describe("instance settings routes", () => {await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);- expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false);+ expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false);});it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {@@ -934,21 +934,21 @@ describe("instance settings routes", () => {const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");expect(res.status).toBe(200);expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });- expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false);+ expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false);expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);});- it("reserves routine deadline reports for verified Cloud control callers", async () => {+ it.each(["1", "2"])("reserves scheduling capability %s for verified Cloud control callers", async version => {mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);for (const source of ["session", "cloud_tenant", "local_implicit"]) {await request(createApp({ ...adminActor, source }))- .get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(403);+ .get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(403);}expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();await request(createApp({ ...adminActor, source: "cloud_control" }))- .get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(200);+ .get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(200);expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,- "fixture-owner", expect.any(Function), undefined, undefined, true);+ "fixture-owner", expect.any(Function), undefined, undefined, true, version === "2");});it("uses the live worker manager for inspection and releases only the owned hold", async () => {@@ -983,7 +983,7 @@ describe("instance settings routes", () => {expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,- "fixture-owner", expect.any(Function), undefined, backup, false);+ "fixture-owner", expect.any(Function), undefined, backup, false, false);mockReadIdleSleepSafety.mockClear();await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();@@ -1005,7 +1005,7 @@ describe("instance settings routes", () => {const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup);await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,- "fixture-owner", expect.any(Function), undefined, backup, false);+ "fixture-owner", expect.any(Function), undefined, backup, false, false);} finally {delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;}diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.tsindex 0df320ace0..800d226182 100644--- a/server/src/routes/instance-settings.ts+++ b/server/src/routes/instance-settings.ts@@ -305,7 +305,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag// The report covers every company in this process. Ordinary company// members may read process counters, but not instance-wide work state.assertCanManageInstanceSettings(req);- const scheduledRoutines = req.query.scheduledWake === "1";+ const scheduledIssueMonitors = req.query.scheduledWake === "2";+ const scheduledRoutines = req.query.scheduledWake === "1" || scheduledIssueMonitors;if (scheduledRoutines && req.actor.source !== "cloud_control") {throw forbidden("Scheduled wake reports require Cloud control authority");}@@ -320,7 +321,7 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManagconst idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now,typeof req.query.ownerId === "string" ? req.query.ownerId : undefined,() => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }),- pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines);+ pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors);res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety });return;}diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.tsindex 9895e95065..8dd5210e07 100644--- a/server/src/routes/openapi.ts+++ b/server/src/routes/openapi.ts@@ -6857,7 +6857,7 @@ registry.registerPath({tags: ["instance"],summary:"Get the task-drain status for this process only; quiescent counts in-process work, and a process restart clears it even when the database still holds running rows",- request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep."), scheduledWake: z.literal("1").optional().describe("Cloud control callers may request an owned routine deadline report. Cloud must acknowledge a durable alarm and revalidate this report before stopping the instance.") }) },+ request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep."), scheduledWake: z.enum(["1", "2"]).optional().describe("Cloud control callers may request routine deadlines (1), or routines and issue monitors (2). Cloud must acknowledge a durable alarm and revalidate this report before stopping the instance.") }) },responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden },});diff --git a/server/src/services/heartbeat/scheduling.test.ts b/server/src/services/heartbeat/scheduling.test.tsindex 8a5659de78..8c1bb544da 100644--- a/server/src/services/heartbeat/scheduling.test.ts+++ b/server/src/services/heartbeat/scheduling.test.ts@@ -11,6 +11,7 @@ import { conflict } from "../../errors.js";import { createHeartbeatRunPreparation } from "./run-preparation.js";import { createHeartbeatRunState } from "./run-state.js";import { createHeartbeatScheduling, type HeartbeatSchedulingDependencies } from "./scheduling.js";+import { readIdleSleepSafety } from "../idle-sleep-safety.js";vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));@@ -134,6 +135,28 @@ describe.skipIf(!postgres.supported)("heartbeat scheduling database boundary", (expect(await readIssue(f.issue.id)).toMatchObject({ monitorNextCheckAt: null, monitorAttemptCount: 1 });});+ it("restores the reported future deadline and claims monitor delivery exactly once", async () => {+ const f = await fixture();+ const at = new Date(now.getTime() + 600_000);+ await db.update(issues).set({ monitorNextCheckAt: at,+ executionPolicy: { monitor: { nextCheckAt: at.toISOString() } } }).where(eq(issues.id, f.issue.id));+ const ownerId = randomUUID();+ const hold = () => ({ ownerId, draining: true, activeRuns: 0, pendingWakes: 0,+ startedAt: now, expiresAt: new Date(now.getTime() + 120_000) });+ const report = await readIdleSleepSafety(db, hold, () => now.getTime(), ownerId,+ async () => "none", undefined, undefined, true, true);+ expect(report).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {+ kind: "routines_and_issue_monitors", nextWakeAt: at.toISOString() } });+ expect(await f.scheduler.tickTimers(now)).toEqual({ checked: 0, enqueued: 0, skipped: 0 });+ expect(f.deps.enqueueWakeup).not.toHaveBeenCalled();+ const restored = createHeartbeatScheduling(db, f.deps);+ const deadline = new Date(report.scheduledWake!.nextWakeAt);+ await Promise.all([restored.tickTimers(deadline), f.scheduler.tickTimers(deadline)]);+ expect(f.deps.enqueueWakeup).toHaveBeenCalledOnce();+ expect(f.deps.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ reason: "issue_monitor_due" }));+ expect(await readIssue(f.issue.id)).toMatchObject({ monitorNextCheckAt: null, monitorAttemptCount: 1 });+ });+it("preserves a replacement schedule written during wake admission", async () => {const f = await fixture();const replacement = new Date(now.getTime() + 60_000);diff --git a/server/src/services/idle-issue-schedule.ts b/server/src/services/idle-issue-schedule.tsnew file mode 100644index 0000000000..431ea72f2f--- /dev/null+++ b/server/src/services/idle-issue-schedule.ts@@ -0,0 +1,94 @@+import { createHash, randomUUID } from "node:crypto";+import { sql } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { AGENT_STATUSES, issueExecutionMonitorPolicySchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";+import type { ScheduledWake } from "./idle-scheduled-work.js";+import { onboardingFirstTaskAwaitingUserSql } from "./recovery/onboarding-first-task-wait.js";++const processGeneration = randomUUID();+const limit = 10_000;+const statuses = new Set(["backlog", "todo", "in_progress", "in_review", "blocked"]);+const agentStatuses = new Set<string>(AGENT_STATUSES);+const record = (value: unknown): Record<string, unknown> | null =>+ value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null;++export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: string, now: number): Promise<{+ blocked: boolean;+ scheduledWake?: ScheduledWake;+}> {+ const rows = await tx.execute<{+ id: string; company_id: string; status: string; updated_at: Date | string;+ assignee_agent_id: string | null; assignee_user_id: string | null;+ checkout_run_id: string | null; execution_run_id: string | null;+ execution_policy: unknown; execution_state: unknown; unblock_descriptor: unknown;+ conversation_agent_id: string | null; monitor_next_check_at: Date | string | null;+ monitor_wake_requested_at: Date | string | null; monitor_attempt_count: number;+ company_status: string; company_updated: Date | string; agent_status: string | null;+ agent_updated: Date | string | null; scoped: boolean; unsafe_dependencies: boolean; dependency_ready: boolean;+ untouched_onboarding: boolean;+ }>(sql`SELECT i.id, i.company_id, i.status, i.updated_at, i.assignee_agent_id, i.assignee_user_id,+ i.checkout_run_id, i.execution_run_id, i.execution_policy, i.execution_state, i.unblock_descriptor,+ i.conversation_agent_id, i.monitor_next_check_at, i.monitor_wake_requested_at, i.monitor_attempt_count,+ c.status AS company_status, c.updated_at AS company_updated,+ a.status AS agent_status, a.updated_at AS agent_updated,+ (i.assignee_agent_id IS NULL OR a.id IS NOT NULL) AS scoped,+ (${onboardingFirstTaskAwaitingUserSql(sql`i.company_id`, sql`i.id`, sql`i.origin_kind`)}+ AND NOT EXISTS (SELECT 1 FROM heartbeat_runs h WHERE h.company_id = i.company_id+ AND (h.context_snapshot ->> 'issueId' = i.id::text OR h.context_snapshot ->> 'taskId' = i.id::text))) AS untouched_onboarding,+ EXISTS (SELECT 1 FROM issue_relations r LEFT JOIN issues b ON b.id = r.issue_id+ WHERE r.related_issue_id = i.id AND (r.company_id <> i.company_id OR b.company_id <> i.company_id+ OR b.id IS NULL OR r.type <> 'blocks' OR b.status NOT IN ('backlog', 'todo', 'in_progress', 'in_review', 'blocked', 'done', 'cancelled'))) AS unsafe_dependencies,+ (EXISTS (SELECT 1 FROM issue_relations r WHERE r.related_issue_id = i.id AND r.type = 'blocks')+ AND NOT EXISTS (SELECT 1 FROM issue_relations r JOIN issues b ON b.id = r.issue_id+ WHERE r.related_issue_id = i.id AND r.type = 'blocks' AND b.status <> 'done')) AS dependency_ready+ FROM issues i JOIN companies c ON c.id = i.company_id+ LEFT JOIN agents a ON a.id = i.assignee_agent_id AND a.company_id = i.company_id+ WHERE i.status NOT IN ('done', 'cancelled') ORDER BY i.id LIMIT ${limit + 1}`);+ if (rows.length > limit) return { blocked: true };+ let earliest = Infinity;+ for (const row of rows) {+ const policy = record(row.execution_policy);+ const state = record(row.execution_state);+ if (!statuses.has(row.status) || row.company_status !== "active" || row.scoped !== true+ || (row.assignee_agent_id && !agentStatuses.has(row.agent_status ?? ""))+ || row.unsafe_dependencies !== false || row.checkout_run_id || row.execution_run_id+ || row.conversation_agent_id || row.monitor_wake_requested_at+ || (row.execution_state !== null && state?.status !== "idle")+ || (row.execution_policy !== null && !policy)) return { blocked: true };+ if (row.status === "blocked" && row.dependency_ready) return { blocked: true };+ if (row.monitor_next_check_at !== null) {+ const monitor = issueExecutionMonitorPolicySchema.safeParse(policy?.monitor);+ const at = new Date(row.monitor_next_check_at).getTime();+ if (!monitor.success || !row.assignee_agent_id || row.assignee_user_id+ || !["in_progress", "in_review"].includes(row.status)+ || !["idle", "active", "error"].includes(row.agent_status ?? "")+ || !Number.isFinite(at) || at <= now || Date.parse(monitor.data.nextCheckAt) !== at+ || monitor.data.serviceName === PROVIDER_QUOTA_MONITOR_SERVICE_NAME+ || !Number.isSafeInteger(row.monitor_attempt_count) || row.monitor_attempt_count < 0+ || (monitor.data.timeoutAt && Date.parse(monitor.data.timeoutAt) <= at)+ || (monitor.data.maxAttempts !== null && row.monitor_attempt_count >= monitor.data.maxAttempts)) return { blocked: true };+ earliest = Math.min(earliest, at);+ continue;+ }+ if (policy?.monitor != null) return { blocked: true };+ const unblock = record(row.unblock_descriptor);+ const passive = row.status === "backlog"+ || (row.status === "todo" && !row.assignee_agent_id)+ || (row.status === "todo" && row.assignee_agent_id && row.agent_status === "idle" && row.untouched_onboarding === true)+ || (row.status === "blocked" && (row.unblock_descriptor === null || unblock?.owner === "board"))+ || Boolean(row.assignee_user_id && !row.assignee_agent_id);+ if (!passive) return { blocked: true };+ }+ if (!Number.isFinite(earliest)) return { blocked: false };+ return { blocked: false, scheduledWake: { version: 1, kind: "routines_and_issue_monitors",+ generation: createHash("sha256").update(JSON.stringify([processGeneration, ownerId, rows])).digest("hex"),+ nextWakeAt: new Date(earliest).toISOString() } };+}++export function combineIdleSchedules(routine?: ScheduledWake, issue?: ScheduledWake): ScheduledWake | undefined {+ if (!issue) return routine;+ if (!routine) return issue;+ return { version: 1, kind: "routines_and_issue_monitors",+ generation: createHash("sha256").update(JSON.stringify([routine, issue])).digest("hex"),+ nextWakeAt: new Date(Math.min(Date.parse(routine.nextWakeAt), Date.parse(issue.nextWakeAt))).toISOString() };+}diff --git a/server/src/services/idle-scheduled-work.ts b/server/src/services/idle-scheduled-work.tsindex b909f798fd..6e9ed7f670 100644--- a/server/src/services/idle-scheduled-work.ts+++ b/server/src/services/idle-scheduled-work.ts@@ -8,7 +8,7 @@ const limit = 10_000;export interface ScheduledWake {version: 1;- kind: "routines";+ kind: "routines" | "routines_and_issue_monitors";generation: string;nextWakeAt: string;}diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.tsindex fe9be54ffd..c5dfb63049 100644--- a/server/src/services/idle-sleep-safety.ts+++ b/server/src/services/idle-sleep-safety.ts@@ -1,6 +1,7 @@import { idleWorkSnapshot } from "./task-admission.js";import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js";import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js";+import { combineIdleSchedules, readIdleIssueSchedule } from "./idle-issue-schedule.js";import type { Db } from "@paperclipai/db";import { sql } from "drizzle-orm";@@ -222,6 +223,7 @@ export async function readIdleSleepSafety(inspectPlugins?: InspectIdlePlugins,prepareBackup?: () => Promise<boolean>,scheduledRoutines = false,+ scheduledIssueMonitors = false,): Promise<IdleSleepSafety> {const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" };const before = getDrainStatus();@@ -242,8 +244,9 @@ export async function readIdleSleepSafety(let scheduledWake: ScheduledWake | undefined;const blocked = await db.transaction(async (tx) => {await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);- const checks = WORK_CHECKS.filter(query => !scheduledRoutines ||- query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`)+ const checks = WORK_CHECKS.filter(query =>+ (!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`)+ && (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`)).map((query) => sql`EXISTS (${sql.raw(query)})`);// Only the current worker's owned runtime drain can exempt a plugin.// Missing workers, old SDKs, and newly installed plugins still block.@@ -259,6 +262,12 @@ export async function readIdleSleepSafety(if (schedule.blocked) return true;scheduledWake = schedule.scheduledWake;}+ if (scheduledIssueMonitors) {+ if (!scheduledRoutines || !ownerId || ownerId !== before.ownerId) return undefined;+ const schedule = await readIdleIssueSchedule(tx, ownerId, now());+ if (schedule.blocked) return true;+ scheduledWake = combineIdleSchedules(scheduledWake, schedule.scheduledWake);+ }return false;}, { isolationLevel: "repeatable read", accessMode: "read only" });if (blocked === undefined || !sameQuietHold(before, getDrainStatus(), now())) return unknown;diff --git a/server/src/services/recovery/onboarding-first-task-wait.ts b/server/src/services/recovery/onboarding-first-task-wait.tsnew file mode 100644index 0000000000..9fa3e7a280--- /dev/null+++ b/server/src/services/recovery/onboarding-first-task-wait.ts@@ -0,0 +1,11 @@+import { sql, type SQLWrapper } from "drizzle-orm";+import { ONBOARDING_FIRST_TASK_ORIGIN_KIND } from "@paperclipai/shared";++export function onboardingFirstTaskAwaitingUserSql(companyId: SQLWrapper, issueId: SQLWrapper, originKind: SQLWrapper) {+ return sql`${originKind} = ${ONBOARDING_FIRST_TASK_ORIGIN_KIND}+ AND NOT EXISTS (SELECT 1 FROM issue_comments c+ WHERE c.company_id = ${companyId} AND c.issue_id = ${issueId}+ AND (c.author_type = 'user' OR (c.author_type IS NULL AND c.author_user_id IS NOT NULL)))+ AND NOT EXISTS (SELECT 1 FROM issue_thread_interactions t+ WHERE t.company_id = ${companyId} AND t.issue_id = ${issueId} AND t.resolved_by_user_id IS NOT NULL)`;+}diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex 5d43e26ae0..23a092420c 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -1,4 +1,5 @@import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";+import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";import { hasCommittedNativePlanWait } from "../native-runtime/native-plan-wait.js";import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared";@@ -2077,40 +2078,9 @@ export function recoveryService(issue: typeof issues.$inferSelect,) {if (issue.originKind !== ONBOARDING_FIRST_TASK_ORIGIN_KIND) return false;- const userComment = await db- .select({ id: issueComments.id })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, issue.companyId),- eq(issueComments.issueId, issue.id),- or(- eq(issueComments.authorType, "user"),- and(- isNull(issueComments.authorType),- sql`${issueComments.authorUserId} is not null`,- ),- ),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- if (userComment !== null) return false;- // Answering the seeded opening card ("interview me" / "I have a task in- // mind") is the user's first input too, even though it is not a comment.- const userResolvedInteraction = await db- .select({ id: issueThreadInteractions.id })- .from(issueThreadInteractions)- .where(- and(- eq(issueThreadInteractions.companyId, issue.companyId),- eq(issueThreadInteractions.issueId, issue.id),- sql`${issueThreadInteractions.resolvedByUserId} is not null`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- return userResolvedInteraction === null;+ const [result] = await db.execute<{ awaiting: boolean }>(sql`SELECT+ ${onboardingFirstTaskAwaitingUserSql(sql`${issue.companyId}`, sql`${issue.id}`, sql`${issue.originKind}`)} AS awaiting`);+ return result?.awaiting === true;}async function isInvocationBudgetBlocked(