Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): reduce job contention
Gergő Móricz6abd8d7d0d52
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex 9b3d18154..ead9341e1 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -40,6 +40,6 @@ import { logger } from "../../lib/logger";while (true) {await scrapeQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 100));+ await new Promise(resolve => setTimeout(resolve, 750));}})();diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 187e4c60f..2d9f508bb 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -930,9 +930,16 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM owner_limited_jobsWHERE owner_rank <= owner_limit),+ locked_jobs AS (+ SELECT q.id, q.owner_id+ FROM ${this.queueName} q+ WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)+ AND q.status = 'queued'::nuq.job_status+ FOR UPDATE SKIP LOCKED+ ),missing_owners AS (SELECT DISTINCT owner_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE owner_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc@@ -948,7 +955,7 @@ class NuQ<JobData = any, JobReturnValue = any> {updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.id IN (SELECT id FROM locked_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")})SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;@@ -1016,9 +1023,16 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM owner_limited_jobsWHERE owner_rank <= owner_limit),+ locked_jobs AS (+ SELECT q.id, q.owner_id, q.group_id+ FROM ${this.queueName} q+ WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)+ AND q.status = 'queued'::nuq.job_status+ FOR UPDATE SKIP LOCKED+ ),missing_owners AS (SELECT DISTINCT owner_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE owner_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc@@ -1033,7 +1047,7 @@ class NuQ<JobData = any, JobReturnValue = any> {),missing_groups AS (SELECT DISTINCT group_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE group_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_group_concurrency gc@@ -1049,7 +1063,7 @@ class NuQ<JobData = any, JobReturnValue = any> {updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.id IN (SELECT id FROM locked_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")})SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;@@ -1160,9 +1174,16 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_rank <= owner_limitLIMIT 1),+ locked_jobs AS (+ SELECT q.id, q.owner_id+ FROM ${this.queueName} q+ WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)+ AND q.status = 'queued'::nuq.job_status+ FOR UPDATE SKIP LOCKED+ ),missing_owners AS (SELECT DISTINCT owner_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE owner_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc@@ -1178,7 +1199,7 @@ class NuQ<JobData = any, JobReturnValue = any> {updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.id IN (SELECT id FROM locked_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")})SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;@@ -1247,9 +1268,16 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_rank <= owner_limitLIMIT 1),+ locked_jobs AS (+ SELECT q.id, q.owner_id, q.group_id+ FROM ${this.queueName} q+ WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)+ AND q.status = 'queued'::nuq.job_status+ FOR UPDATE SKIP LOCKED+ ),missing_owners AS (SELECT DISTINCT owner_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE owner_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc@@ -1264,7 +1292,7 @@ class NuQ<JobData = any, JobReturnValue = any> {),missing_groups AS (SELECT DISTINCT group_id- FROM selected_jobs_with_metadata+ FROM locked_jobsWHERE group_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ${this.queueName}_group_concurrency gc@@ -1280,7 +1308,7 @@ class NuQ<JobData = any, JobReturnValue = any> {updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.id IN (SELECT id FROM locked_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")})SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex f7058bf92..1697a1f8c 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -64,6 +64,7 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_complete_idx ON nuq.queue_scr-- This allows fast COUNT(*) queries for active jobs per owner/group without maintaining countersCREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_owner_idx ON nuq.queue_scrape (owner_id) WHERE (status = 'active'::nuq.job_status AND owner_id IS NOT NULL);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_group_idx ON nuq.queue_scrape (group_id) WHERE (status = 'active'::nuq.job_status AND group_id IS NOT NULL);+CREATE INDEX IF NOT EXISTS idx_queue_scrape_prefetch_composite ON queue_scrape (status, owner_id, group_id, priority, created_at, id) WHERE status = 'queued';CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (id uuid NOT NULL,