Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: do not take the recipe advisory lock inside the SDK sweeper

Alfredo Lopezde05aec4c0ec
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index ce0f7fbf8..49592ae27 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.532",
+ "version": "0.1.533",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.532",
+ "version": "0.1.533",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index a45bc5200..4e36c1aca 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.532",
+ "version": "0.1.533",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.ts
index c20488a4e..dc51b0dde 100644
--- a/control-api/src/services/pluginWorkloadSdkDb.ts
+++ b/control-api/src/services/pluginWorkloadSdkDb.ts
@@ -2149,9 +2149,9 @@ export async function failStaleInvocationsInTransaction(
recipe_namespace: string
recipe_name: string
}>) {
- await db.query(`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, [
- `plugin_workload_sdk:${row.recipe_namespace}/${row.recipe_name}`,
- ])
+ // Invocation FOR UPDATE is the fence. Do not take the recipe advisory lock
+ // here: finalize takes advisory first, then the invocation row. Row-then-
+ // advisory deadlocks the sweeper/finalizer interleaving.
const physicalAttempts = await db.query(
`SELECT id, recipe_namespace, recipe_name, attempt_generation,
attempt_index, target_ref, provider, model, credential_slot
diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.ts
index 31f75bf02..11f70ea8a 100644
--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts
@@ -410,7 +410,6 @@ describe('invocation retry lifecycle timestamps', () => {
],
rowCount: 1,
} as never)
- .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never)
.mockResolvedValueOnce({
rows: [
{
@@ -435,21 +434,25 @@ describe('invocation retry lifecycle timestamps', () => {
expect(sql).toContain('lease_expires_at IS NULL')
expect(sql).toContain('updated_at < now()')
expect(sql).toContain('FOR UPDATE')
- expect(pool.query).toHaveBeenCalledTimes(7)
- expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain('pg_advisory_xact_lock')
- expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain(
+ expect(
+ vi
+ .mocked(pool.query)
+ .mock.calls.some(([statement]: [string]) => statement.includes('pg_advisory_xact_lock'))
+ ).toBe(false)
+ expect(pool.query).toHaveBeenCalledTimes(6)
+ expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain(
'FROM plugin_workload_sdk_provider_attempts'
)
- expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain(
+ expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain(
"SET status = 'provider_unavailable'"
)
- expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain(
+ expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain(
'INSERT INTO plugin_workload_sdk_spend_outcomes'
)
- expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain(
+ expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain(
"SET status = 'provider_unavailable'"
)
- expect(vi.mocked(pool.query).mock.calls[6]?.[0] as string).toContain(
+ expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain(
"SET status = 'provider_unavailable'"
)
})
@@ -467,7 +470,6 @@ describe('invocation retry lifecycle timestamps', () => {
],
rowCount: 1,
} as never)
- .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never)
.mockResolvedValueOnce({
rows: [
{