The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(sentry): keep provider-rejected Grok wakes out of run failure reports (#16297)
Devin Foleybcfc03bc9493
diff --git a/packages/db/src/migrations/0346_cloudy_whiplash.sql b/packages/db/src/migrations/0346_cloudy_whiplash.sqlnew file mode 100644index 0000000000..c9018ab00a--- /dev/null+++ b/packages/db/src/migrations/0346_cloudy_whiplash.sql@@ -0,0 +1,2 @@+-- Find an assignment's work wakes without scanning the company's dispatch history.+CREATE INDEX "grok_dispatch_assignment_kind_idx" ON "grok_bot_dispatches" USING btree ("assignment_id","kind");diff --git a/packages/db/src/migrations/meta/0341_snapshot.json b/packages/db/src/migrations/meta/0346_snapshot.jsonsimilarity index 98%rename from packages/db/src/migrations/meta/0341_snapshot.jsonrename to packages/db/src/migrations/meta/0346_snapshot.jsonindex 1c38823f95..f57eebd88d 100644--- a/packages/db/src/migrations/meta/0341_snapshot.json+++ b/packages/db/src/migrations/meta/0346_snapshot.json@@ -1,6 +1,6 @@{- "id": "927d7532-059b-4a4f-846a-5e3445a81891",- "prevId": "b32fa655-79c7-4793-a021-22258b6c369c",+ "id": "40048887-b6ac-4334-b77b-6b8fc4b52701",+ "prevId": "3bd7b1df-1434-4fac-87d6-53b90a9eb9e1","version": "7","dialect": "postgresql","tables": {@@ -4098,6 +4098,12 @@"primaryKey": false,"notNull": false},+ "created_by_user_id": {+ "name": "created_by_user_id",+ "type": "text",+ "primaryKey": false,+ "notNull": false+ },"created_at": {"name": "created_at","type": "timestamp with time zone",@@ -16852,6 +16858,53 @@"checkConstraints": {},"isRLSEnabled": false},+ "public.company_skill_import_budgets": {+ "name": "company_skill_import_budgets",+ "schema": "",+ "columns": {+ "company_id": {+ "name": "company_id",+ "type": "uuid",+ "primaryKey": true,+ "notNull": true+ },+ "available_bytes": {+ "name": "available_bytes",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "default": 536870912+ },+ "updated_at": {+ "name": "updated_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true,+ "default": "now()"+ }+ },+ "indexes": {},+ "foreignKeys": {+ "company_skill_import_budgets_company_id_companies_id_fk": {+ "name": "company_skill_import_budgets_company_id_companies_id_fk",+ "tableFrom": "company_skill_import_budgets",+ "tableTo": "companies",+ "columnsFrom": [+ "company_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "policies": {},+ "checkConstraints": {},+ "isRLSEnabled": false+ },"public.company_skill_source_entries": {"name": "company_skill_source_entries","schema": "",@@ -24927,6 +24980,13 @@"primaryKey": false,"notNull": false},+ "provider_kind": {+ "name": "provider_kind",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "default": "'openai_dot'"+ },"generation": {"name": "generation","type": "integer",@@ -25554,6 +25614,18 @@"primaryKey": false,"notNull": false},+ "pending_returned_at": {+ "name": "pending_returned_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "observed_at": {+ "name": "observed_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },"source_event_id": {"name": "source_event_id","type": "text",@@ -30638,6 +30710,495 @@"checkConstraints": {},"isRLSEnabled": false},+ "public.grok_bot_dispatches": {+ "name": "grok_bot_dispatches",+ "schema": "",+ "columns": {+ "id": {+ "name": "id",+ "type": "uuid",+ "primaryKey": true,+ "notNull": true,+ "default": "gen_random_uuid()"+ },+ "company_id": {+ "name": "company_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "routine_id": {+ "name": "routine_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "binding_id": {+ "name": "binding_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "binding_generation": {+ "name": "binding_generation",+ "type": "integer",+ "primaryKey": false,+ "notNull": true+ },+ "grant_id": {+ "name": "grant_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "routine_revision": {+ "name": "routine_revision",+ "type": "integer",+ "primaryKey": false,+ "notNull": true+ },+ "assignment_id": {+ "name": "assignment_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": false+ },+ "source_event_id": {+ "name": "source_event_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true+ },+ "kind": {+ "name": "kind",+ "type": "text",+ "primaryKey": false,+ "notNull": true+ },+ "payload": {+ "name": "payload",+ "type": "jsonb",+ "primaryKey": false,+ "notNull": true+ },+ "challenge_hash": {+ "name": "challenge_hash",+ "type": "text",+ "primaryKey": false,+ "notNull": false+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "default": "'prepared'"+ },+ "http_status": {+ "name": "http_status",+ "type": "integer",+ "primaryKey": false,+ "notNull": false+ },+ "expires_at": {+ "name": "expires_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true+ },+ "sent_at": {+ "name": "sent_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "acknowledged_at": {+ "name": "acknowledged_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "created_at": {+ "name": "created_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true,+ "default": "now()"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true,+ "default": "now()"+ }+ },+ "indexes": {+ "grok_dispatch_event_uq": {+ "name": "grok_dispatch_event_uq",+ "columns": [+ {+ "expression": "binding_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "binding_generation",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "routine_revision",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "source_event_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ }+ ],+ "isUnique": true,+ "concurrently": false,+ "method": "btree",+ "with": {}+ },+ "grok_dispatch_company_status_idx": {+ "name": "grok_dispatch_company_status_idx",+ "columns": [+ {+ "expression": "company_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "status",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ }+ ],+ "isUnique": false,+ "concurrently": false,+ "method": "btree",+ "with": {}+ },+ "grok_dispatch_assignment_kind_idx": {+ "name": "grok_dispatch_assignment_kind_idx",+ "columns": [+ {+ "expression": "assignment_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "kind",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ }+ ],+ "isUnique": false,+ "concurrently": false,+ "method": "btree",+ "with": {}+ }+ },+ "foreignKeys": {+ "grok_bot_dispatches_company_id_companies_id_fk": {+ "name": "grok_bot_dispatches_company_id_companies_id_fk",+ "tableFrom": "grok_bot_dispatches",+ "tableTo": "companies",+ "columnsFrom": [+ "company_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk": {+ "name": "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk",+ "tableFrom": "grok_bot_dispatches",+ "tableTo": "grok_bot_routines",+ "columnsFrom": [+ "routine_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk": {+ "name": "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk",+ "tableFrom": "grok_bot_dispatches",+ "tableTo": "dot_agent_bindings",+ "columnsFrom": [+ "binding_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk": {+ "name": "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk",+ "tableFrom": "grok_bot_dispatches",+ "tableTo": "mcp_oauth_grants",+ "columnsFrom": [+ "grant_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk": {+ "name": "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk",+ "tableFrom": "grok_bot_dispatches",+ "tableTo": "dot_runner_assignments",+ "columnsFrom": [+ "assignment_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "policies": {},+ "checkConstraints": {},+ "isRLSEnabled": false+ },+ "public.grok_bot_routines": {+ "name": "grok_bot_routines",+ "schema": "",+ "columns": {+ "id": {+ "name": "id",+ "type": "uuid",+ "primaryKey": true,+ "notNull": true,+ "default": "gen_random_uuid()"+ },+ "company_id": {+ "name": "company_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "agent_id": {+ "name": "agent_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "binding_id": {+ "name": "binding_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "binding_generation": {+ "name": "binding_generation",+ "type": "integer",+ "primaryKey": false,+ "notNull": true+ },+ "bot_name": {+ "name": "bot_name",+ "type": "text",+ "primaryKey": false,+ "notNull": true+ },+ "secret_id": {+ "name": "secret_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": true+ },+ "secret_version": {+ "name": "secret_version",+ "type": "integer",+ "primaryKey": false,+ "notNull": true+ },+ "revision": {+ "name": "revision",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "default": 1+ },+ "verified_grant_id": {+ "name": "verified_grant_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": false+ },+ "verified_at": {+ "name": "verified_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "last_contact_at": {+ "name": "last_contact_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "recovery_required_at": {+ "name": "recovery_required_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": false+ },+ "recovery_reason": {+ "name": "recovery_reason",+ "type": "text",+ "primaryKey": false,+ "notNull": false+ },+ "created_at": {+ "name": "created_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true,+ "default": "now()"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "timestamp with time zone",+ "primaryKey": false,+ "notNull": true,+ "default": "now()"+ }+ },+ "indexes": {+ "grok_routines_binding_uq": {+ "name": "grok_routines_binding_uq",+ "columns": [+ {+ "expression": "binding_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ }+ ],+ "isUnique": true,+ "concurrently": false,+ "method": "btree",+ "with": {}+ },+ "grok_routines_company_agent_idx": {+ "name": "grok_routines_company_agent_idx",+ "columns": [+ {+ "expression": "company_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "agent_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ }+ ],+ "isUnique": false,+ "concurrently": false,+ "method": "btree",+ "with": {}+ }+ },+ "foreignKeys": {+ "grok_bot_routines_company_id_companies_id_fk": {+ "name": "grok_bot_routines_company_id_companies_id_fk",+ "tableFrom": "grok_bot_routines",+ "tableTo": "companies",+ "columnsFrom": [+ "company_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_routines_agent_id_agents_id_fk": {+ "name": "grok_bot_routines_agent_id_agents_id_fk",+ "tableFrom": "grok_bot_routines",+ "tableTo": "agents",+ "columnsFrom": [+ "agent_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_routines_binding_id_dot_agent_bindings_id_fk": {+ "name": "grok_bot_routines_binding_id_dot_agent_bindings_id_fk",+ "tableFrom": "grok_bot_routines",+ "tableTo": "dot_agent_bindings",+ "columnsFrom": [+ "binding_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },+ "grok_bot_routines_secret_id_company_secrets_id_fk": {+ "name": "grok_bot_routines_secret_id_company_secrets_id_fk",+ "tableFrom": "grok_bot_routines",+ "tableTo": "company_secrets",+ "columnsFrom": [+ "secret_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "no action",+ "onUpdate": "no action"+ },+ "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk": {+ "name": "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk",+ "tableFrom": "grok_bot_routines",+ "tableTo": "mcp_oauth_grants",+ "columnsFrom": [+ "verified_grant_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "set null",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "policies": {},+ "checkConstraints": {},+ "isRLSEnabled": false+ },"public.heartbeat_run_events": {"name": "heartbeat_run_events","schema": "",@@ -31967,6 +32528,54 @@"concurrently": false,"method": "btree","with": {}+ },+ "heartbeat_runs_company_ctx_paperclip_issue_created_idx": {+ "name": "heartbeat_runs_company_ctx_paperclip_issue_created_idx",+ "columns": [+ {+ "expression": "company_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "(\"context_snapshot\" -> 'paperclipIssue' ->> 'id')",+ "asc": true,+ "isExpression": true,+ "nulls": "last"+ },+ {+ "expression": "created_at",+ "isExpression": false,+ "asc": false,+ "nulls": "last"+ }+ ],+ "isUnique": false,+ "concurrently": true,+ "method": "btree",+ "with": {}+ },+ "heartbeat_runs_company_origin_issue_idx": {+ "name": "heartbeat_runs_company_origin_issue_idx",+ "columns": [+ {+ "expression": "company_id",+ "isExpression": false,+ "asc": true,+ "nulls": "last"+ },+ {+ "expression": "coalesce(\n \"native_issue_id\"::text,\n nullif(\"context_snapshot\"->>'issueId', ''),\n nullif(\"context_snapshot\"->>'taskId', ''),\n nullif(\"context_snapshot\"->>'taskKey', '')\n )",+ "asc": true,+ "isExpression": true,+ "nulls": "last"+ }+ ],+ "isUnique": false,+ "concurrently": true,+ "method": "btree",+ "with": {}}},"foreignKeys": {@@ -58385,6 +58994,18 @@"name": "mcp_attachment_uploads","schema": "","columns": {+ "grant_id": {+ "name": "grant_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": false+ },+ "work_session_id": {+ "name": "work_session_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": false+ },"id": {"name": "id","type": "uuid",@@ -58529,6 +59150,19 @@}},"foreignKeys": {+ "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk": {+ "name": "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk",+ "tableFrom": "mcp_attachment_uploads",+ "tableTo": "mcp_oauth_grants",+ "columnsFrom": [+ "grant_id"+ ],+ "columnsTo": [+ "id"+ ],+ "onDelete": "cascade",+ "onUpdate": "no action"+ },"mcp_attachment_uploads_company_id_companies_id_fk": {"name": "mcp_attachment_uploads_company_id_companies_id_fk","tableFrom": "mcp_attachment_uploads",@@ -59005,6 +59639,12 @@"name": "mcp_file_tickets","schema": "","columns": {+ "work_session_id": {+ "name": "work_session_id",+ "type": "uuid",+ "primaryKey": false,+ "notNull": false+ },"token_hash": {"name": "token_hash","type": "text",diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.jsonindex 1fd5c5fb5a..00a90de13c 100644--- a/packages/db/src/migrations/meta/_journal.json+++ b/packages/db/src/migrations/meta/_journal.json@@ -2409,6 +2409,13 @@"when": 1791978393174,"tag": "0345_luxuriant_carmella_unuscione","breakpoints": true+ },+ {+ "idx": 346,+ "version": "7",+ "when": 1791989860950,+ "tag": "0346_cloudy_whiplash",+ "breakpoints": true}]}\ No newline at end of filediff --git a/packages/db/src/schema/grok_bot.ts b/packages/db/src/schema/grok_bot.tsindex 1ec3f1d98c..da978d3a62 100644--- a/packages/db/src/schema/grok_bot.ts+++ b/packages/db/src/schema/grok_bot.ts@@ -50,4 +50,5 @@ export const grokBotDispatches = pgTable("grok_bot_dispatches", {}, t => [uniqueIndex("grok_dispatch_event_uq").on(t.bindingId, t.bindingGeneration, t.routineRevision, t.sourceEventId),index("grok_dispatch_company_status_idx").on(t.companyId, t.status),+ index("grok_dispatch_assignment_kind_idx").on(t.assignmentId, t.kind),]);diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.tsindex c4f5271e20..9c6f143e9e 100644--- a/server/src/services/__tests__/run-failure-report.test.ts+++ b/server/src/services/__tests__/run-failure-report.test.ts@@ -1,7 +1,9 @@import { randomUUID } from "node:crypto";import os from "node:os";import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";-import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db";+import { agents, applyPendingMigrations, authUsers, closeRegisteredClients, companies, companySecrets, createDb, dotAgentBindings,+ dotRunnerAssignments, grokBotDispatches, grokBotRoutines, heartbeatRuns, issues, mcpOauthClients, mcpOauthGrants, nativeRunFinalizations,+ type Db } from "@paperclipai/db";import {getEmbeddedPostgresTestSupport,startEmbeddedPostgresTestDatabase,@@ -67,9 +69,15 @@ describeEmbeddedPostgres("reportRunFailure", () => {vi.clearAllMocks();vi.unstubAllEnvs();process.env = inheritedEnv;+ await db.delete(grokBotRoutines);+ await db.delete(companySecrets);+ await db.delete(nativeRunFinalizations);await db.delete(heartbeatRuns);+ await db.delete(issues);await db.delete(agents);await db.delete(companies);+ await db.delete(mcpOauthClients);+ await db.delete(authUsers);});afterAll(async () => {@@ -335,6 +343,80 @@ describeEmbeddedPostgres("reportRunFailure", () => {expect(mockCaptureRunFailure).toHaveBeenCalledOnce();});+ async function expiredGrokOffer(input: {+ wakes: Array<{ status: "rejected" | "accepted" | "acknowledged" | "unknown"; httpStatus: number | null }>;+ acceptedAt?: Date | null;+ originalFailureCode?: string;+ finalization?: boolean;+ offers?: number;+ }) {+ await seedCompanyAndAgent({ adapterType: "paperclip_runner", adapterConfig: { provider: "grok_bot" } });+ const userId = randomUUID();+ await db.insert(authUsers).values({ id: userId, name: "Operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });+ await db.insert(mcpOauthClients).values({ id: userId, name: "Grok", redirectUris: ["https://client.example/callback"] });+ const [grant] = await db.insert(mcpOauthGrants).values({ companyId, userId, clientId: userId, resource: "https://paperclip.example/mcp/runner",+ purpose: "agent", agentId, scopes: ["paperclip:agent"] }).returning();+ const [binding] = await db.insert(dotAgentBindings).values({ companyId, agentId, operatorId: userId, grantId: grant!.id,+ providerKind: "grok_bot", status: "ready" }).returning();+ const [secret] = await db.insert(companySecrets).values({ companyId, key: `grok-${binding!.id}`, name: "Grok routine" }).returning();+ const [routine] = await db.insert(grokBotRoutines).values({ companyId, agentId, bindingId: binding!.id, bindingGeneration: 1,+ botName: "Grok", secretId: secret!.id, secretVersion: 1 }).returning();+ const [issue] = await db.insert(issues).values({ companyId, title: "Grok task", assigneeAgentId: agentId, status: "in_progress" }).returning();+ const run = buildRun({ errorCode: "native_provider_terminal_failed", error: "Dot assignment acceptance or execution authority expired.",+ nativeIssueId: issue!.id, runtimeMode: "native", executionStage: "preparing", nativePhase: "terminal_failure", exitCode: null, signal: null });+ await db.insert(heartbeatRuns).values(run);+ if (input.finalization !== false) {+ await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId, issueId: run.nativeIssueId!, phase: "terminal_failure",+ failureCode: "native_provider_terminal_failed", failureDetail: { originalFailureCode: input.originalFailureCode ?? "dot_assignment_expired" } });+ }+ const offers = await db.insert(dotRunnerAssignments).values(Array.from({ length: input.offers ?? 1 }, () => ({ companyId, agentId,+ bindingId: binding!.id, bindingGeneration: 1, runId: run.id, normalizedSessionId: randomUUID(), turnId: randomUUID(), controllerGeneration: 1,+ catalogDigest: "fixture", projection: {}, status: "fenced" as const, acceptBy: new Date(Date.now() - 1000),+ expiresAt: new Date(Date.now() + 3_600_000), acceptedAt: input.acceptedAt ?? null }))).returning();+ const assignment = offers[0];+ for (const wake of input.wakes) {+ await db.insert(grokBotDispatches).values({ companyId, routineId: routine!.id, bindingId: binding!.id, bindingGeneration: 1,+ grantId: grant!.id, routineRevision: 1, assignmentId: assignment!.id, sourceEventId: randomUUID(), kind: "mailbox_updated",+ payload: {}, status: wake.status, httpStatus: wake.httpStatus, expiresAt: new Date(Date.now() + 600_000) });+ }+ return run;+ }++ it("keeps a Grok offer local when the endpoint rejected every work wake with HTTP 4xx", async () => {+ const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: 400 }] });+ await reportRunFailure(db, run, { phase: "execute" });+ expect(mockCaptureRunFailure).not.toHaveBeenCalled();+ });++ it.each([+ ["no wake was sent", { wakes: [] }],+ ["a wake was rejected locally without HTTP evidence", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: null }] }],+ ["the endpoint accepted a wake", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "acknowledged", httpStatus: 200 }] }],+ ["delivery is uncertain", { wakes: [{ status: "unknown", httpStatus: 503 }] }],+ ["the bot accepted the work", { wakes: [{ status: "rejected", httpStatus: 400 }], acceptedAt: new Date() }],+ ["the run failed for another provider reason", { wakes: [{ status: "rejected", httpStatus: 400 }], originalFailureCode: "provider_failed" }],+ ["the run has no finalization", { wakes: [{ status: "rejected", httpStatus: 400 }], finalization: false }],+ ["the run has more than one offer", { wakes: [{ status: "rejected", httpStatus: 400 }], offers: 2 }],+ ] as const)("keeps an expired Grok offer reportable when %s", async (_label, input) => {+ const run = await expiredGrokOffer({ ...input, wakes: [...input.wakes] });+ await reportRunFailure(db, run, { phase: "execute" });+ expect(mockCaptureRunFailure).toHaveBeenCalledOnce();+ });++ it("keeps an expired Grok offer reportable when its receipts cannot be read", async () => {+ const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }] });+ const select = db.select.bind(db);+ // The agent lookup succeeds; the receipt reads fail.+ const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select)+ .mockImplementation(() => { throw new Error("fixture receipt read failed"); });+ try {+ await reportRunFailure(db, run, { phase: "execute" });+ } finally {+ spy.mockRestore();+ }+ expect(mockCaptureRunFailure).toHaveBeenCalledOnce();+ });+function aiSelectionRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {return buildRun({errorCode: "configuration_incomplete",diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.tsindex 60e6309e54..e738dfe67a 100644--- a/server/src/services/run-failure-report.ts+++ b/server/src/services/run-failure-report.ts@@ -1,5 +1,5 @@import { and, eq } from "drizzle-orm";-import { agents, heartbeatRuns, type Db } from "@paperclipai/db";+import { agents, dotRunnerAssignments, grokBotDispatches, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";import { captureRunFailure, type RunFailureStatus } from "../sentry.js";import {collectRunFailureDiagnostics,@@ -174,6 +174,27 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOptioptions.phase !== "setup" && run.signal == null;}+/**+ * The Grok endpoint refused every work wake with an HTTP 4xx, so the offer+ * expired unaccepted. Durable delivery receipts prove the external cause. A+ * local rejection (no HTTP status), any delivered wake, or accepted work stays+ * reportable.+ */+async function isRejectedExternalBotWake(db: Db, run: HeartbeatRun): Promise<boolean> {+ if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== "native_provider_terminal_failed") return false;+ const [finalization] = await db.select({ failureDetail: nativeRunFinalizations.failureDetail }).from(nativeRunFinalizations)+ .where(and(eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId)));+ if (finalization?.failureDetail?.originalFailureCode !== "dot_assignment_expired") return false;+ const offers = await db.select({ id: dotRunnerAssignments.id, acceptedAt: dotRunnerAssignments.acceptedAt }).from(dotRunnerAssignments)+ .where(and(eq(dotRunnerAssignments.runId, run.id), eq(dotRunnerAssignments.companyId, run.companyId)));+ if (offers.length !== 1 || offers[0]!.acceptedAt) return false;+ const wakes = await db.select({ status: grokBotDispatches.status, httpStatus: grokBotDispatches.httpStatus }).from(grokBotDispatches)+ .where(and(eq(grokBotDispatches.assignmentId, offers[0]!.id), eq(grokBotDispatches.companyId, run.companyId),+ eq(grokBotDispatches.kind, "mailbox_updated")));+ return wakes.length > 0 && wakes.every((wake) =>+ wake.status === "rejected" && wake.httpStatus != null && wake.httpStatus >= 400 && wake.httpStatus < 500);+}+/*** Report a terminal run failure to Sentry. Returns at once for any status* other than failures and unexpected started cancellations. Never throws — a Sentry failure or a@@ -222,6 +243,12 @@ async function captureTerminalRunFailure(.then((rows) => rows[0] ?? null);if (isHermesConnectionFailure(run, options, agent?.adapterType)) return;+ // An unreadable receipt proves nothing; keep the failure reportable.+ const rejectedExternally = await isRejectedExternalBotWake(db, run).catch((err) => {+ logger.warn({ err, runId: run.id }, "failed to read bot wake receipts for run failure report");+ return false;+ });+ if (rejectedExternally) return;const taskId = readTaskId(run);if (!taskId) {