Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): use FOR UPDATE SKIP LOCKED instead of advisory locks
Gergő Móriczb71f8cdfb4ff
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 8150f3a96..893751c2d 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -916,9 +916,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY owner_id),acquired_owner_locks AS (- SELECT owner_id- FROM distinct_owners_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED),lockable_jobs AS (SELECT id@@ -1000,9 +1001,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY owner_id),acquired_owner_locks AS (- SELECT owner_id- FROM distinct_owners_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED),distinct_groups_to_lock AS (SELECT DISTINCT group_id@@ -1011,9 +1013,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY group_id),acquired_group_locks AS (- SELECT group_id- FROM distinct_groups_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true+ SELECT id as group_id+ FROM ${this.queueName}_group_concurrency+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)+ FOR UPDATE SKIP LOCKED),lockable_jobs AS (SELECT id@@ -1164,9 +1167,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY owner_id),acquired_owner_locks AS (- SELECT owner_id- FROM distinct_owners_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED),lockable_jobs AS (SELECT id@@ -1243,9 +1247,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY owner_id),acquired_owner_locks AS (- SELECT owner_id- FROM distinct_owners_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED),distinct_groups_to_lock AS (SELECT DISTINCT group_id@@ -1254,9 +1259,10 @@ class NuQ<JobData = any, JobReturnValue = any> {ORDER BY group_id),acquired_group_locks AS (- SELECT group_id- FROM distinct_groups_to_lock- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true+ SELECT id as group_id+ FROM ${this.queueName}_group_concurrency+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)+ FOR UPDATE SKIP LOCKED),lockable_jobs AS (SELECT iddiff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex f0843d5eb..8f7198b22 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -111,10 +111,10 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$ORDER BY owner_id),acquired_owner_locks AS (- SELECT- owner_id- FROM distinct_owners- WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true+ SELECT id as owner_id+ FROM nuq.queue_scrape_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners)+ FOR UPDATE SKIP LOCKED),distinct_groups AS (SELECT DISTINCT sj.group_id@@ -124,10 +124,10 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$ORDER BY sj.group_id),acquired_group_locks AS (- SELECT- group_id- FROM distinct_groups- WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true+ SELECT id as group_id+ FROM nuq.queue_scrape_group_concurrency+ WHERE id IN (SELECT group_id FROM distinct_groups)+ FOR UPDATE SKIP LOCKED),requeued AS (UPDATE nuq.queue_scrape