Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq): use FOR UPDATE SKIP LOCKED instead of advisory locks

Gergő Móriczb71f8cdfb4ff
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 8150f3a96..893751c2d 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -916,9 +916,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY owner_id
),
acquired_owner_locks AS (
- SELECT owner_id
- FROM distinct_owners_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true
+ SELECT id as owner_id
+ FROM ${this.queueName}_owner_concurrency
+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)
+ FOR UPDATE SKIP LOCKED
),
lockable_jobs AS (
SELECT id
@@ -1000,9 +1001,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY owner_id
),
acquired_owner_locks AS (
- SELECT owner_id
- FROM distinct_owners_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true
+ SELECT id as owner_id
+ FROM ${this.queueName}_owner_concurrency
+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)
+ FOR UPDATE SKIP LOCKED
),
distinct_groups_to_lock AS (
SELECT DISTINCT group_id
@@ -1011,9 +1013,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY group_id
),
acquired_group_locks AS (
- SELECT group_id
- FROM distinct_groups_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true
+ SELECT id as group_id
+ FROM ${this.queueName}_group_concurrency
+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)
+ FOR UPDATE SKIP LOCKED
),
lockable_jobs AS (
SELECT id
@@ -1164,9 +1167,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY owner_id
),
acquired_owner_locks AS (
- SELECT owner_id
- FROM distinct_owners_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true
+ SELECT id as owner_id
+ FROM ${this.queueName}_owner_concurrency
+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)
+ FOR UPDATE SKIP LOCKED
),
lockable_jobs AS (
SELECT id
@@ -1243,9 +1247,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY owner_id
),
acquired_owner_locks AS (
- SELECT owner_id
- FROM distinct_owners_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true
+ SELECT id as owner_id
+ FROM ${this.queueName}_owner_concurrency
+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)
+ FOR UPDATE SKIP LOCKED
),
distinct_groups_to_lock AS (
SELECT DISTINCT group_id
@@ -1254,9 +1259,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
ORDER BY group_id
),
acquired_group_locks AS (
- SELECT group_id
- FROM distinct_groups_to_lock
- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true
+ SELECT id as group_id
+ FROM ${this.queueName}_group_concurrency
+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)
+ FOR UPDATE SKIP LOCKED
),
lockable_jobs AS (
SELECT id
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index f0843d5eb..8f7198b22 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -111,10 +111,10 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
ORDER BY owner_id
),
acquired_owner_locks AS (
- SELECT
- owner_id
- FROM distinct_owners
- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true
+ SELECT id as owner_id
+ FROM nuq.queue_scrape_owner_concurrency
+ WHERE id IN (SELECT owner_id FROM distinct_owners)
+ FOR UPDATE SKIP LOCKED
),
distinct_groups AS (
SELECT DISTINCT sj.group_id
@@ -124,10 +124,10 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
ORDER BY sj.group_id
),
acquired_group_locks AS (
- SELECT
- group_id
- FROM distinct_groups
- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true
+ SELECT id as group_id
+ FROM nuq.queue_scrape_group_concurrency
+ WHERE id IN (SELECT group_id FROM distinct_groups)
+ FOR UPDATE SKIP LOCKED
),
requeued AS (
UPDATE nuq.queue_scrape