Skip to content

The open-source app everyone uses to manage agents at work

fix(sentry): keep provider-rejected Grok wakes out of run failure reports (#16297)

Devin Foleybcfc03bc9493
diff --git a/packages/db/src/migrations/0346_cloudy_whiplash.sql b/packages/db/src/migrations/0346_cloudy_whiplash.sql
new file mode 100644
index 0000000000..c9018ab00a
--- /dev/null
+++ b/packages/db/src/migrations/0346_cloudy_whiplash.sql
@@ -0,0 +1,2 @@
+-- Find an assignment's work wakes without scanning the company's dispatch history.
+CREATE INDEX "grok_dispatch_assignment_kind_idx" ON "grok_bot_dispatches" USING btree ("assignment_id","kind");
diff --git a/packages/db/src/migrations/meta/0341_snapshot.json b/packages/db/src/migrations/meta/0346_snapshot.json
similarity index 98%
rename from packages/db/src/migrations/meta/0341_snapshot.json
rename to packages/db/src/migrations/meta/0346_snapshot.json
index 1c38823f95..f57eebd88d 100644
--- a/packages/db/src/migrations/meta/0341_snapshot.json
+++ b/packages/db/src/migrations/meta/0346_snapshot.json
@@ -1,6 +1,6 @@
{
- "id": "927d7532-059b-4a4f-846a-5e3445a81891",
- "prevId": "b32fa655-79c7-4793-a021-22258b6c369c",
+ "id": "40048887-b6ac-4334-b77b-6b8fc4b52701",
+ "prevId": "3bd7b1df-1434-4fac-87d6-53b90a9eb9e1",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -4098,6 +4098,12 @@
"primaryKey": false,
"notNull": false
},
+ "created_by_user_id": {
+ "name": "created_by_user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
@@ -16852,6 +16858,53 @@
"checkConstraints": {},
"isRLSEnabled": false
},
+ "public.company_skill_import_budgets": {
+ "name": "company_skill_import_budgets",
+ "schema": "",
+ "columns": {
+ "company_id": {
+ "name": "company_id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "available_bytes": {
+ "name": "available_bytes",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 536870912
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "company_skill_import_budgets_company_id_companies_id_fk": {
+ "name": "company_skill_import_budgets_company_id_companies_id_fk",
+ "tableFrom": "company_skill_import_budgets",
+ "tableTo": "companies",
+ "columnsFrom": [
+ "company_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
"public.company_skill_source_entries": {
"name": "company_skill_source_entries",
"schema": "",
@@ -24927,6 +24980,13 @@
"primaryKey": false,
"notNull": false
},
+ "provider_kind": {
+ "name": "provider_kind",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'openai_dot'"
+ },
"generation": {
"name": "generation",
"type": "integer",
@@ -25554,6 +25614,18 @@
"primaryKey": false,
"notNull": false
},
+ "pending_returned_at": {
+ "name": "pending_returned_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "observed_at": {
+ "name": "observed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
"source_event_id": {
"name": "source_event_id",
"type": "text",
@@ -30638,6 +30710,495 @@
"checkConstraints": {},
"isRLSEnabled": false
},
+ "public.grok_bot_dispatches": {
+ "name": "grok_bot_dispatches",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "company_id": {
+ "name": "company_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "routine_id": {
+ "name": "routine_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "binding_id": {
+ "name": "binding_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "binding_generation": {
+ "name": "binding_generation",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "grant_id": {
+ "name": "grant_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "routine_revision": {
+ "name": "routine_revision",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "assignment_id": {
+ "name": "assignment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_event_id": {
+ "name": "source_event_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "kind": {
+ "name": "kind",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "payload": {
+ "name": "payload",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "challenge_hash": {
+ "name": "challenge_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'prepared'"
+ },
+ "http_status": {
+ "name": "http_status",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "acknowledged_at": {
+ "name": "acknowledged_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "grok_dispatch_event_uq": {
+ "name": "grok_dispatch_event_uq",
+ "columns": [
+ {
+ "expression": "binding_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "binding_generation",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "routine_revision",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "source_event_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "grok_dispatch_company_status_idx": {
+ "name": "grok_dispatch_company_status_idx",
+ "columns": [
+ {
+ "expression": "company_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "status",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "grok_dispatch_assignment_kind_idx": {
+ "name": "grok_dispatch_assignment_kind_idx",
+ "columns": [
+ {
+ "expression": "assignment_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "kind",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "grok_bot_dispatches_company_id_companies_id_fk": {
+ "name": "grok_bot_dispatches_company_id_companies_id_fk",
+ "tableFrom": "grok_bot_dispatches",
+ "tableTo": "companies",
+ "columnsFrom": [
+ "company_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk": {
+ "name": "grok_bot_dispatches_routine_id_grok_bot_routines_id_fk",
+ "tableFrom": "grok_bot_dispatches",
+ "tableTo": "grok_bot_routines",
+ "columnsFrom": [
+ "routine_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk": {
+ "name": "grok_bot_dispatches_binding_id_dot_agent_bindings_id_fk",
+ "tableFrom": "grok_bot_dispatches",
+ "tableTo": "dot_agent_bindings",
+ "columnsFrom": [
+ "binding_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk": {
+ "name": "grok_bot_dispatches_grant_id_mcp_oauth_grants_id_fk",
+ "tableFrom": "grok_bot_dispatches",
+ "tableTo": "mcp_oauth_grants",
+ "columnsFrom": [
+ "grant_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk": {
+ "name": "grok_bot_dispatches_assignment_id_dot_runner_assignments_id_fk",
+ "tableFrom": "grok_bot_dispatches",
+ "tableTo": "dot_runner_assignments",
+ "columnsFrom": [
+ "assignment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.grok_bot_routines": {
+ "name": "grok_bot_routines",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "company_id": {
+ "name": "company_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "agent_id": {
+ "name": "agent_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "binding_id": {
+ "name": "binding_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "binding_generation": {
+ "name": "binding_generation",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "bot_name": {
+ "name": "bot_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "secret_id": {
+ "name": "secret_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "secret_version": {
+ "name": "secret_version",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "revision": {
+ "name": "revision",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 1
+ },
+ "verified_grant_id": {
+ "name": "verified_grant_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "verified_at": {
+ "name": "verified_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_contact_at": {
+ "name": "last_contact_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "recovery_required_at": {
+ "name": "recovery_required_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "recovery_reason": {
+ "name": "recovery_reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "grok_routines_binding_uq": {
+ "name": "grok_routines_binding_uq",
+ "columns": [
+ {
+ "expression": "binding_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "grok_routines_company_agent_idx": {
+ "name": "grok_routines_company_agent_idx",
+ "columns": [
+ {
+ "expression": "company_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "agent_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "grok_bot_routines_company_id_companies_id_fk": {
+ "name": "grok_bot_routines_company_id_companies_id_fk",
+ "tableFrom": "grok_bot_routines",
+ "tableTo": "companies",
+ "columnsFrom": [
+ "company_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_routines_agent_id_agents_id_fk": {
+ "name": "grok_bot_routines_agent_id_agents_id_fk",
+ "tableFrom": "grok_bot_routines",
+ "tableTo": "agents",
+ "columnsFrom": [
+ "agent_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_routines_binding_id_dot_agent_bindings_id_fk": {
+ "name": "grok_bot_routines_binding_id_dot_agent_bindings_id_fk",
+ "tableFrom": "grok_bot_routines",
+ "tableTo": "dot_agent_bindings",
+ "columnsFrom": [
+ "binding_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "grok_bot_routines_secret_id_company_secrets_id_fk": {
+ "name": "grok_bot_routines_secret_id_company_secrets_id_fk",
+ "tableFrom": "grok_bot_routines",
+ "tableTo": "company_secrets",
+ "columnsFrom": [
+ "secret_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk": {
+ "name": "grok_bot_routines_verified_grant_id_mcp_oauth_grants_id_fk",
+ "tableFrom": "grok_bot_routines",
+ "tableTo": "mcp_oauth_grants",
+ "columnsFrom": [
+ "verified_grant_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
"public.heartbeat_run_events": {
"name": "heartbeat_run_events",
"schema": "",
@@ -31967,6 +32528,54 @@
"concurrently": false,
"method": "btree",
"with": {}
+ },
+ "heartbeat_runs_company_ctx_paperclip_issue_created_idx": {
+ "name": "heartbeat_runs_company_ctx_paperclip_issue_created_idx",
+ "columns": [
+ {
+ "expression": "company_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "(\"context_snapshot\" -> 'paperclipIssue' ->> 'id')",
+ "asc": true,
+ "isExpression": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": false,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": true,
+ "method": "btree",
+ "with": {}
+ },
+ "heartbeat_runs_company_origin_issue_idx": {
+ "name": "heartbeat_runs_company_origin_issue_idx",
+ "columns": [
+ {
+ "expression": "company_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "coalesce(\n \"native_issue_id\"::text,\n nullif(\"context_snapshot\"->>'issueId', ''),\n nullif(\"context_snapshot\"->>'taskId', ''),\n nullif(\"context_snapshot\"->>'taskKey', '')\n )",
+ "asc": true,
+ "isExpression": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": true,
+ "method": "btree",
+ "with": {}
}
},
"foreignKeys": {
@@ -58385,6 +58994,18 @@
"name": "mcp_attachment_uploads",
"schema": "",
"columns": {
+ "grant_id": {
+ "name": "grant_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "work_session_id": {
+ "name": "work_session_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
"id": {
"name": "id",
"type": "uuid",
@@ -58529,6 +59150,19 @@
}
},
"foreignKeys": {
+ "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk": {
+ "name": "mcp_attachment_uploads_grant_id_mcp_oauth_grants_id_fk",
+ "tableFrom": "mcp_attachment_uploads",
+ "tableTo": "mcp_oauth_grants",
+ "columnsFrom": [
+ "grant_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
"mcp_attachment_uploads_company_id_companies_id_fk": {
"name": "mcp_attachment_uploads_company_id_companies_id_fk",
"tableFrom": "mcp_attachment_uploads",
@@ -59005,6 +59639,12 @@
"name": "mcp_file_tickets",
"schema": "",
"columns": {
+ "work_session_id": {
+ "name": "work_session_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
"token_hash": {
"name": "token_hash",
"type": "text",
diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json
index 1fd5c5fb5a..00a90de13c 100644
--- a/packages/db/src/migrations/meta/_journal.json
+++ b/packages/db/src/migrations/meta/_journal.json
@@ -2409,6 +2409,13 @@
"when": 1791978393174,
"tag": "0345_luxuriant_carmella_unuscione",
"breakpoints": true
+ },
+ {
+ "idx": 346,
+ "version": "7",
+ "when": 1791989860950,
+ "tag": "0346_cloudy_whiplash",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/db/src/schema/grok_bot.ts b/packages/db/src/schema/grok_bot.ts
index 1ec3f1d98c..da978d3a62 100644
--- a/packages/db/src/schema/grok_bot.ts
+++ b/packages/db/src/schema/grok_bot.ts
@@ -50,4 +50,5 @@ export const grokBotDispatches = pgTable("grok_bot_dispatches", {
}, t => [
uniqueIndex("grok_dispatch_event_uq").on(t.bindingId, t.bindingGeneration, t.routineRevision, t.sourceEventId),
index("grok_dispatch_company_status_idx").on(t.companyId, t.status),
+ index("grok_dispatch_assignment_kind_idx").on(t.assignmentId, t.kind),
]);
diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts
index c4f5271e20..9c6f143e9e 100644
--- a/server/src/services/__tests__/run-failure-report.test.ts
+++ b/server/src/services/__tests__/run-failure-report.test.ts
@@ -1,7 +1,9 @@
import { randomUUID } from "node:crypto";
import os from "node:os";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db";
+import { agents, applyPendingMigrations, authUsers, closeRegisteredClients, companies, companySecrets, createDb, dotAgentBindings,
+ dotRunnerAssignments, grokBotDispatches, grokBotRoutines, heartbeatRuns, issues, mcpOauthClients, mcpOauthGrants, nativeRunFinalizations,
+ type Db } from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -67,9 +69,15 @@ describeEmbeddedPostgres("reportRunFailure", () => {
vi.clearAllMocks();
vi.unstubAllEnvs();
process.env = inheritedEnv;
+ await db.delete(grokBotRoutines);
+ await db.delete(companySecrets);
+ await db.delete(nativeRunFinalizations);
await db.delete(heartbeatRuns);
+ await db.delete(issues);
await db.delete(agents);
await db.delete(companies);
+ await db.delete(mcpOauthClients);
+ await db.delete(authUsers);
});
afterAll(async () => {
@@ -335,6 +343,80 @@ describeEmbeddedPostgres("reportRunFailure", () => {
expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
});
+ async function expiredGrokOffer(input: {
+ wakes: Array<{ status: "rejected" | "accepted" | "acknowledged" | "unknown"; httpStatus: number | null }>;
+ acceptedAt?: Date | null;
+ originalFailureCode?: string;
+ finalization?: boolean;
+ offers?: number;
+ }) {
+ await seedCompanyAndAgent({ adapterType: "paperclip_runner", adapterConfig: { provider: "grok_bot" } });
+ const userId = randomUUID();
+ await db.insert(authUsers).values({ id: userId, name: "Operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
+ await db.insert(mcpOauthClients).values({ id: userId, name: "Grok", redirectUris: ["https://client.example/callback"] });
+ const [grant] = await db.insert(mcpOauthGrants).values({ companyId, userId, clientId: userId, resource: "https://paperclip.example/mcp/runner",
+ purpose: "agent", agentId, scopes: ["paperclip:agent"] }).returning();
+ const [binding] = await db.insert(dotAgentBindings).values({ companyId, agentId, operatorId: userId, grantId: grant!.id,
+ providerKind: "grok_bot", status: "ready" }).returning();
+ const [secret] = await db.insert(companySecrets).values({ companyId, key: `grok-${binding!.id}`, name: "Grok routine" }).returning();
+ const [routine] = await db.insert(grokBotRoutines).values({ companyId, agentId, bindingId: binding!.id, bindingGeneration: 1,
+ botName: "Grok", secretId: secret!.id, secretVersion: 1 }).returning();
+ const [issue] = await db.insert(issues).values({ companyId, title: "Grok task", assigneeAgentId: agentId, status: "in_progress" }).returning();
+ const run = buildRun({ errorCode: "native_provider_terminal_failed", error: "Dot assignment acceptance or execution authority expired.",
+ nativeIssueId: issue!.id, runtimeMode: "native", executionStage: "preparing", nativePhase: "terminal_failure", exitCode: null, signal: null });
+ await db.insert(heartbeatRuns).values(run);
+ if (input.finalization !== false) {
+ await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId, issueId: run.nativeIssueId!, phase: "terminal_failure",
+ failureCode: "native_provider_terminal_failed", failureDetail: { originalFailureCode: input.originalFailureCode ?? "dot_assignment_expired" } });
+ }
+ const offers = await db.insert(dotRunnerAssignments).values(Array.from({ length: input.offers ?? 1 }, () => ({ companyId, agentId,
+ bindingId: binding!.id, bindingGeneration: 1, runId: run.id, normalizedSessionId: randomUUID(), turnId: randomUUID(), controllerGeneration: 1,
+ catalogDigest: "fixture", projection: {}, status: "fenced" as const, acceptBy: new Date(Date.now() - 1000),
+ expiresAt: new Date(Date.now() + 3_600_000), acceptedAt: input.acceptedAt ?? null }))).returning();
+ const assignment = offers[0];
+ for (const wake of input.wakes) {
+ await db.insert(grokBotDispatches).values({ companyId, routineId: routine!.id, bindingId: binding!.id, bindingGeneration: 1,
+ grantId: grant!.id, routineRevision: 1, assignmentId: assignment!.id, sourceEventId: randomUUID(), kind: "mailbox_updated",
+ payload: {}, status: wake.status, httpStatus: wake.httpStatus, expiresAt: new Date(Date.now() + 600_000) });
+ }
+ return run;
+ }
+
+ it("keeps a Grok offer local when the endpoint rejected every work wake with HTTP 4xx", async () => {
+ const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: 400 }] });
+ await reportRunFailure(db, run, { phase: "execute" });
+ expect(mockCaptureRunFailure).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["no wake was sent", { wakes: [] }],
+ ["a wake was rejected locally without HTTP evidence", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: null }] }],
+ ["the endpoint accepted a wake", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "acknowledged", httpStatus: 200 }] }],
+ ["delivery is uncertain", { wakes: [{ status: "unknown", httpStatus: 503 }] }],
+ ["the bot accepted the work", { wakes: [{ status: "rejected", httpStatus: 400 }], acceptedAt: new Date() }],
+ ["the run failed for another provider reason", { wakes: [{ status: "rejected", httpStatus: 400 }], originalFailureCode: "provider_failed" }],
+ ["the run has no finalization", { wakes: [{ status: "rejected", httpStatus: 400 }], finalization: false }],
+ ["the run has more than one offer", { wakes: [{ status: "rejected", httpStatus: 400 }], offers: 2 }],
+ ] as const)("keeps an expired Grok offer reportable when %s", async (_label, input) => {
+ const run = await expiredGrokOffer({ ...input, wakes: [...input.wakes] });
+ await reportRunFailure(db, run, { phase: "execute" });
+ expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
+ });
+
+ it("keeps an expired Grok offer reportable when its receipts cannot be read", async () => {
+ const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }] });
+ const select = db.select.bind(db);
+ // The agent lookup succeeds; the receipt reads fail.
+ const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select)
+ .mockImplementation(() => { throw new Error("fixture receipt read failed"); });
+ try {
+ await reportRunFailure(db, run, { phase: "execute" });
+ } finally {
+ spy.mockRestore();
+ }
+ expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
+ });
+
function aiSelectionRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
return buildRun({
errorCode: "configuration_incomplete",
diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts
index 60e6309e54..e738dfe67a 100644
--- a/server/src/services/run-failure-report.ts
+++ b/server/src/services/run-failure-report.ts
@@ -1,5 +1,5 @@
import { and, eq } from "drizzle-orm";
-import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
+import { agents, dotRunnerAssignments, grokBotDispatches, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { captureRunFailure, type RunFailureStatus } from "../sentry.js";
import {
collectRunFailureDiagnostics,
@@ -174,6 +174,27 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti
options.phase !== "setup" && run.signal == null;
}
+/**
+ * The Grok endpoint refused every work wake with an HTTP 4xx, so the offer
+ * expired unaccepted. Durable delivery receipts prove the external cause. A
+ * local rejection (no HTTP status), any delivered wake, or accepted work stays
+ * reportable.
+ */
+async function isRejectedExternalBotWake(db: Db, run: HeartbeatRun): Promise<boolean> {
+ if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== "native_provider_terminal_failed") return false;
+ const [finalization] = await db.select({ failureDetail: nativeRunFinalizations.failureDetail }).from(nativeRunFinalizations)
+ .where(and(eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId)));
+ if (finalization?.failureDetail?.originalFailureCode !== "dot_assignment_expired") return false;
+ const offers = await db.select({ id: dotRunnerAssignments.id, acceptedAt: dotRunnerAssignments.acceptedAt }).from(dotRunnerAssignments)
+ .where(and(eq(dotRunnerAssignments.runId, run.id), eq(dotRunnerAssignments.companyId, run.companyId)));
+ if (offers.length !== 1 || offers[0]!.acceptedAt) return false;
+ const wakes = await db.select({ status: grokBotDispatches.status, httpStatus: grokBotDispatches.httpStatus }).from(grokBotDispatches)
+ .where(and(eq(grokBotDispatches.assignmentId, offers[0]!.id), eq(grokBotDispatches.companyId, run.companyId),
+ eq(grokBotDispatches.kind, "mailbox_updated")));
+ return wakes.length > 0 && wakes.every((wake) =>
+ wake.status === "rejected" && wake.httpStatus != null && wake.httpStatus >= 400 && wake.httpStatus < 500);
+}
+
/**
* Report a terminal run failure to Sentry. Returns at once for any status
* other than failures and unexpected started cancellations. Never throws — a Sentry failure or a
@@ -222,6 +243,12 @@ async function captureTerminalRunFailure(
.then((rows) => rows[0] ?? null);
if (isHermesConnectionFailure(run, options, agent?.adapterType)) return;
+ // An unreadable receipt proves nothing; keep the failure reportable.
+ const rejectedExternally = await isRejectedExternalBotWake(db, run).catch((err) => {
+ logger.warn({ err, runId: run.id }, "failed to read bot wake receipts for run failure report");
+ return false;
+ });
+ if (rejectedExternally) return;
const taskId = readTaskId(run);
if (!taskId) {