Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "fix(nuq): experiment: allow concurrency limit overshoot for perf"
Gergő Móricz0cd0723e4f44
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 5baedef62..5a47ad5c2 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -909,17 +909,18 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM owner_limited_jobsWHERE owner_rank <= owner_limit),- distinct_owners AS (+ distinct_owners_to_lock AS (SELECT DISTINCT owner_idFROM selected_jobs_with_metadataWHERE owner_id IS NOT NULL+ ORDER BY owner_id),missing_owners AS (- SELECT do.owner_id- FROM distinct_owners do+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = do.owner_id+ WHERE oc.id = dtl.owner_id)),ensure_owner_rows AS (@@ -927,10 +928,21 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)FROM missing_owners),+ acquired_owner_locks AS (+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ lockable_jobs AS (+ SELECT id+ FROM selected_jobs_with_metadata+ WHERE owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = selected_jobs_with_metadata.owner_id)+ ),updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM lockable_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}),owner_counts AS (@@ -984,17 +996,18 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM owner_limited_jobsWHERE owner_rank <= owner_limit),- distinct_owners AS (+ distinct_owners_to_lock AS (SELECT DISTINCT owner_idFROM selected_jobs_with_metadataWHERE owner_id IS NOT NULL+ ORDER BY owner_id),missing_owners AS (- SELECT do.owner_id- FROM distinct_owners do+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = do.owner_id+ WHERE oc.id = dtl.owner_id)),ensure_owner_rows AS (@@ -1002,17 +1015,24 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)FROM missing_owners),- distinct_groups AS (+ acquired_owner_locks AS (+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ distinct_groups_to_lock AS (SELECT DISTINCT group_idFROM selected_jobs_with_metadataWHERE group_id IS NOT NULL+ ORDER BY group_id),missing_groups AS (- SELECT dg.group_id- FROM distinct_groups dg+ SELECT dtl.group_id+ FROM distinct_groups_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_group_concurrency gc- WHERE gc.id = dg.group_id+ WHERE gc.id = dtl.group_id)),ensure_group_rows AS (@@ -1020,10 +1040,22 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT group_id, 0, NULLFROM missing_groups),+ acquired_group_locks AS (+ SELECT id as group_id+ FROM ${this.queueName}_group_concurrency+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ lockable_jobs AS (+ SELECT id+ FROM selected_jobs_with_metadata+ WHERE (owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = selected_jobs_with_metadata.owner_id))+ AND (group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = selected_jobs_with_metadata.group_id))+ ),updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM lockable_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}),owner_counts AS (@@ -1145,17 +1177,18 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_rank <= owner_limitLIMIT 1),- distinct_owners AS (+ distinct_owners_to_lock AS (SELECT DISTINCT owner_idFROM selected_jobs_with_metadataWHERE owner_id IS NOT NULL+ ORDER BY owner_id),missing_owners AS (- SELECT do.owner_id- FROM distinct_owners do+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = do.owner_id+ WHERE oc.id = dtl.owner_id)),ensure_owner_rows AS (@@ -1163,10 +1196,21 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)FROM missing_owners),+ acquired_owner_locks AS (+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ lockable_jobs AS (+ SELECT id+ FROM selected_jobs_with_metadata+ WHERE owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = selected_jobs_with_metadata.owner_id)+ ),updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM lockable_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}),owner_increment AS (@@ -1216,17 +1260,18 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_rank <= owner_limitLIMIT 1),- distinct_owners AS (+ distinct_owners_to_lock AS (SELECT DISTINCT owner_idFROM selected_jobs_with_metadataWHERE owner_id IS NOT NULL+ ORDER BY owner_id),missing_owners AS (- SELECT do.owner_id- FROM distinct_owners do+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = do.owner_id+ WHERE oc.id = dtl.owner_id)),ensure_owner_rows AS (@@ -1234,17 +1279,24 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)FROM missing_owners),- distinct_groups AS (+ acquired_owner_locks AS (+ SELECT id as owner_id+ FROM ${this.queueName}_owner_concurrency+ WHERE id IN (SELECT owner_id FROM distinct_owners_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ distinct_groups_to_lock AS (SELECT DISTINCT group_idFROM selected_jobs_with_metadataWHERE group_id IS NOT NULL+ ORDER BY group_id),missing_groups AS (- SELECT dg.group_id- FROM distinct_groups dg+ SELECT dtl.group_id+ FROM distinct_groups_to_lock dtlWHERE NOT EXISTS (SELECT 1 FROM ${this.queueName}_group_concurrency gc- WHERE gc.id = dg.group_id+ WHERE gc.id = dtl.group_id)),ensure_group_rows AS (@@ -1252,10 +1304,22 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT group_id, 0, NULLFROM missing_groups),+ acquired_group_locks AS (+ SELECT id as group_id+ FROM ${this.queueName}_group_concurrency+ WHERE id IN (SELECT group_id FROM distinct_groups_to_lock)+ FOR UPDATE SKIP LOCKED+ ),+ lockable_jobs AS (+ SELECT id+ FROM selected_jobs_with_metadata+ WHERE (owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = selected_jobs_with_metadata.owner_id))+ AND (group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = selected_jobs_with_metadata.group_id))+ ),updated AS (UPDATE ${this.queueName} qSET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM selected_jobs_with_metadata)+ WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM lockable_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}),owner_counts AS (