Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): adjust prefetch query
Gergő Móricz952565cc8580
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 8128babca..ce818b132 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -857,13 +857,19 @@ class NuQ<JobData = any, JobReturnValue = any> {if (this.options.concurrencyLimit === "per-owner") {updateQuery = `WITH queued_owners AS (- SELECT DISTINCT owner_id+ SELECT DISTINCT ON (owner_id)+ owner_id,+ priority,+ created_atFROM ${this.queueName}WHERE status = 'queued'::nuq.job_status+ ORDER BY owner_id, priority ASC, created_at ASC),available_capacity AS (SELECTqo.owner_id,+ qo.priority,+ qo.created_at,CASEWHEN qo.owner_id IS NULL THEN 999999WHEN oc.max_concurrency IS NOT NULL THEN GREATEST(0, oc.max_concurrency - oc.current_concurrency)@@ -872,35 +878,37 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM queued_owners qoLEFT JOIN ${this.queueName}_owner_concurrency oc ON qo.owner_id = oc.id),+ limited_capacity AS (+ SELECT owner_id, slots+ FROM available_capacity+ WHERE slots > 0+ ORDER BY priority ASC, created_at ASC, owner_id ASC+ LIMIT 75+ ),distinct_owners AS (SELECT owner_id- FROM available_capacity- WHERE owner_id IS NOT NULL AND slots > 0+ FROM limited_capacity+ WHERE owner_id IS NOT NULLORDER BY owner_id),acquired_owner_locks AS (SELECT- owner_id,- pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ owner_idFROM distinct_owners- ),- available_capacity_locked AS (- SELECT *- FROM available_capacity ac- WHERE ac.slots > 0- AND (ac.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE acquired_owner_locks.owner_id = ac.owner_id))+ WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true),selected_jobs AS (SELECT j.id- FROM available_capacity_locked ac+ FROM limited_capacity lc+ WHERE (lc.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = lc.owner_id))CROSS JOIN LATERAL (SELECT j.idFROM ${this.queueName} jWHERE j.status = 'queued'::nuq.job_status- AND j.owner_id IS NOT DISTINCT FROM ac.owner_id+ AND j.owner_id IS NOT DISTINCT FROM lc.owner_idORDER BY j.priority ASC, j.created_at ASCFOR UPDATE SKIP LOCKED- LIMIT ac.slots+ LIMIT lc.slots) j),updated AS (@@ -937,14 +945,21 @@ class NuQ<JobData = any, JobReturnValue = any> {} else if (this.options.concurrencyLimit === "per-owner-per-group") {updateQuery = `WITH queued_combinations AS (- SELECT DISTINCT owner_id, group_id+ SELECT DISTINCT ON (owner_id, group_id)+ owner_id,+ group_id,+ priority,+ created_atFROM ${this.queueName}WHERE status = 'queued'::nuq.job_status+ ORDER BY owner_id, group_id, priority ASC, created_at ASC),available_capacity AS (SELECTqc.owner_id,qc.group_id,+ qc.priority,+ qc.created_at,CASEWHEN qc.owner_id IS NULL AND qc.group_id IS NULL THEN 999999WHEN qc.owner_id IS NULL AND gc.max_concurrency IS NOT NULL THEN GREATEST(0, gc.max_concurrency - gc.current_concurrency)@@ -960,49 +975,52 @@ class NuQ<JobData = any, JobReturnValue = any> {LEFT JOIN ${this.queueName}_owner_concurrency oc ON qc.owner_id = oc.idLEFT JOIN ${this.queueName}_group_concurrency gc ON qc.group_id = gc.id),+ limited_capacity AS (+ SELECT owner_id, group_id, slots+ FROM available_capacity+ WHERE slots > 0+ ORDER BY priority ASC, created_at ASC, group_id ASC+ LIMIT 75+ ),distinct_owners AS (SELECT DISTINCT owner_id- FROM available_capacity- WHERE owner_id IS NOT NULL AND slots > 0+ FROM limited_capacity+ WHERE owner_id IS NOT NULLORDER BY owner_id),acquired_owner_locks AS (SELECT- owner_id,- pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ owner_idFROM distinct_owners+ WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true),distinct_groups AS (- SELECT DISTINCT group_id- FROM available_capacity- WHERE group_id IS NOT NULL AND slots > 0- ORDER BY group_id+ SELECT DISTINCT lc.group_id+ FROM limited_capacity lc+ INNER JOIN acquired_owner_locks aol ON lc.owner_id = aol.owner_id+ WHERE lc.group_id IS NOT NULL+ ORDER BY lc.group_id),acquired_group_locks AS (SELECT- group_id,- pg_advisory_xact_lock(hashtext(group_id::text)) as dummy+ group_idFROM distinct_groups- ),- available_capacity_locked AS (- SELECT *- FROM available_capacity ac- WHERE ac.slots > 0- AND (ac.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE acquired_owner_locks.owner_id = ac.owner_id))- AND (ac.group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE acquired_group_locks.group_id = ac.group_id))+ WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true),selected_jobs_raw AS (SELECT j.id, j.owner_id, j.group_id, j.priority, j.created_at- FROM available_capacity_locked ac+ FROM limited_capacity lc+ WHERE (lc.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = lc.owner_id))+ AND (lc.group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = lc.group_id))CROSS JOIN LATERAL (SELECT j.id, j.owner_id, j.group_id, j.priority, j.created_atFROM ${this.queueName} jWHERE j.status = 'queued'::nuq.job_status- AND j.owner_id IS NOT DISTINCT FROM ac.owner_id- AND j.group_id IS NOT DISTINCT FROM ac.group_id+ AND j.owner_id IS NOT DISTINCT FROM lc.owner_id+ AND j.group_id IS NOT DISTINCT FROM lc.group_idORDER BY j.priority ASC, j.created_at ASCFOR UPDATE SKIP LOCKED- LIMIT LEAST(ac.slots, 20)+ LIMIT LEAST(lc.slots, 20)) j),selected_jobs_with_owner_rank AS (@@ -1167,19 +1185,15 @@ class NuQ<JobData = any, JobReturnValue = any> {),acquired_owner_locks AS (SELECT- owner_id,- pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ owner_idFROM distinct_owners- ),- available_capacity_locked AS (- SELECT *- FROM available_capacity ac- WHERE ac.slots > 0- AND (ac.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE acquired_owner_locks.owner_id = ac.owner_id))+ WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true),selected_jobs AS (SELECT j.id- FROM available_capacity_locked ac+ FROM available_capacity ac+ WHERE ac.slots > 0+ AND (ac.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = ac.owner_id))CROSS JOIN LATERAL (SELECT j.idFROM ${this.queueName} j@@ -1250,32 +1264,29 @@ class NuQ<JobData = any, JobReturnValue = any> {),acquired_owner_locks AS (SELECT- owner_id,- pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ owner_idFROM distinct_owners+ WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true),distinct_groups AS (- SELECT DISTINCT group_id- FROM available_capacity- WHERE group_id IS NOT NULL AND slots > 0- ORDER BY group_id+ SELECT DISTINCT ac.group_id+ FROM available_capacity ac+ INNER JOIN acquired_owner_locks aol ON ac.owner_id = aol.owner_id+ WHERE ac.group_id IS NOT NULL AND ac.slots > 0+ ORDER BY ac.group_id),acquired_group_locks AS (SELECT- group_id,- pg_advisory_xact_lock(hashtext(group_id::text)) as dummy+ group_idFROM distinct_groups+ WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true),- available_capacity_locked AS (- SELECT *+ selected_jobs AS (+ SELECT j.idFROM available_capacity acWHERE ac.slots > 0AND (ac.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = ac.owner_id))AND (ac.group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = ac.group_id))- ),- selected_jobs AS (- SELECT j.id- FROM available_capacity_locked acCROSS JOIN LATERAL (SELECT j.idFROM ${this.queueName} jdiff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 5e60c29cd..d553d5b1d 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -110,21 +110,22 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$),acquired_owner_locks AS (SELECT- owner_id,- pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ owner_idFROM distinct_owners+ WHERE pg_try_advisory_xact_lock(hashtext(owner_id::text)) = true),distinct_groups AS (- SELECT DISTINCT group_id- FROM stalled_jobs- WHERE group_id IS NOT NULL- ORDER BY group_id+ SELECT DISTINCT sj.group_id+ FROM stalled_jobs sj+ INNER JOIN acquired_owner_locks aol ON sj.owner_id = aol.owner_id+ WHERE sj.group_id IS NOT NULL+ ORDER BY sj.group_id),acquired_group_locks AS (SELECT- group_id,- pg_advisory_xact_lock(hashtext(group_id::text)) as dummy+ group_idFROM distinct_groups+ WHERE pg_try_advisory_xact_lock(hashtext(group_id::text)) = true),requeued AS (UPDATE nuq.queue_scrape