Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: do not take the recipe advisory lock inside the SDK sweeper
Alfredo Lopezde05aec4c0ec
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex ce0f7fbf8..49592ae27 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.532",+ "version": "0.1.533","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.532",+ "version": "0.1.533","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex a45bc5200..4e36c1aca 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.532",+ "version": "0.1.533","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex c20488a4e..dc51b0dde 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -2149,9 +2149,9 @@ export async function failStaleInvocationsInTransaction(recipe_namespace: stringrecipe_name: string}>) {- await db.query(`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, [- `plugin_workload_sdk:${row.recipe_namespace}/${row.recipe_name}`,- ])+ // Invocation FOR UPDATE is the fence. Do not take the recipe advisory lock+ // here: finalize takes advisory first, then the invocation row. Row-then-+ // advisory deadlocks the sweeper/finalizer interleaving.const physicalAttempts = await db.query(`SELECT id, recipe_namespace, recipe_name, attempt_generation,attempt_index, target_ref, provider, model, credential_slotdiff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex 31f75bf02..11f70ea8a 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -410,7 +410,6 @@ describe('invocation retry lifecycle timestamps', () => {],rowCount: 1,} as never)- .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never).mockResolvedValueOnce({rows: [{@@ -435,21 +434,25 @@ describe('invocation retry lifecycle timestamps', () => {expect(sql).toContain('lease_expires_at IS NULL')expect(sql).toContain('updated_at < now()')expect(sql).toContain('FOR UPDATE')- expect(pool.query).toHaveBeenCalledTimes(7)- expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain('pg_advisory_xact_lock')- expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain(+ expect(+ vi+ .mocked(pool.query)+ .mock.calls.some(([statement]: [string]) => statement.includes('pg_advisory_xact_lock'))+ ).toBe(false)+ expect(pool.query).toHaveBeenCalledTimes(6)+ expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain('FROM plugin_workload_sdk_provider_attempts')- expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain(+ expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain("SET status = 'provider_unavailable'")- expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain(+ expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain('INSERT INTO plugin_workload_sdk_spend_outcomes')- expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain(+ expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain("SET status = 'provider_unavailable'")- expect(vi.mocked(pool.query).mock.calls[6]?.[0] as string).toContain(+ expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain("SET status = 'provider_unavailable'")})@@ -467,7 +470,6 @@ describe('invocation retry lifecycle timestamps', () => {],rowCount: 1,} as never)- .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never).mockResolvedValueOnce({rows: [{