Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): correct owner concurrency upsert logic
Gergő Móricz9ce618ae10a5
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 893751c2d..5a47ad5c2 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -915,6 +915,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLORDER BY owner_id),+ missing_owners AS (+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_owner_concurrency oc+ WHERE oc.id = dtl.owner_id+ )+ ),+ ensure_owner_rows AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)+ FROM missing_owners+ ),acquired_owner_locks AS (SELECT id as owner_idFROM ${this.queueName}_owner_concurrency@@ -938,22 +951,11 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLGROUP BY owner_id),- owner_counts_with_max AS (- SELECT- oc.owner_id,- oc.job_count,- COALESCE(ocon.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)) as max_concurrency- FROM owner_counts oc- LEFT JOIN ${this.queueName}_owner_concurrency ocon ON oc.owner_id = ocon.id- ),- owner_upsert AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, job_count, max_concurrency- FROM owner_counts_with_max- ON CONFLICT (id)- DO UPDATE SET- current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency,- max_concurrency = EXCLUDED.max_concurrency+ owner_update AS (+ UPDATE ${this.queueName}_owner_concurrency oc+ SET current_concurrency = oc.current_concurrency + owner_counts.job_count+ FROM owner_counts+ WHERE oc.id = owner_counts.owner_id)SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;`;@@ -1000,6 +1002,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLORDER BY owner_id),+ missing_owners AS (+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_owner_concurrency oc+ WHERE oc.id = dtl.owner_id+ )+ ),+ ensure_owner_rows AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)+ FROM missing_owners+ ),acquired_owner_locks AS (SELECT id as owner_idFROM ${this.queueName}_owner_concurrency@@ -1012,6 +1027,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE group_id IS NOT NULLORDER BY group_id),+ missing_groups AS (+ SELECT dtl.group_id+ FROM distinct_groups_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_group_concurrency gc+ WHERE gc.id = dtl.group_id+ )+ ),+ ensure_group_rows AS (+ INSERT INTO ${this.queueName}_group_concurrency (id, current_concurrency, max_concurrency)+ SELECT group_id, 0, NULL+ FROM missing_groups+ ),acquired_group_locks AS (SELECT id as group_idFROM ${this.queueName}_group_concurrency@@ -1036,22 +1064,11 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLGROUP BY owner_id),- owner_counts_with_max AS (- SELECT- oc.owner_id,- oc.job_count,- COALESCE(ocon.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)) as max_concurrency- FROM owner_counts oc- LEFT JOIN ${this.queueName}_owner_concurrency ocon ON oc.owner_id = ocon.id- ),- owner_upsert AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, job_count, max_concurrency- FROM owner_counts_with_max- ON CONFLICT (id)- DO UPDATE SET- current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency,- max_concurrency = EXCLUDED.max_concurrency+ owner_update AS (+ UPDATE ${this.queueName}_owner_concurrency oc+ SET current_concurrency = oc.current_concurrency + owner_counts.job_count+ FROM owner_counts+ WHERE oc.id = owner_counts.owner_id),group_counts AS (SELECT group_id, COUNT(*)::int8 as job_count@@ -1166,6 +1183,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLORDER BY owner_id),+ missing_owners AS (+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_owner_concurrency oc+ WHERE oc.id = dtl.owner_id+ )+ ),+ ensure_owner_rows AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)+ FROM missing_owners+ ),acquired_owner_locks AS (SELECT id as owner_idFROM ${this.queueName}_owner_concurrency@@ -1183,22 +1213,12 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE q.status = 'queued'::nuq.job_status AND q.id IN (SELECT id FROM lockable_jobs)RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}),- updated_with_max AS (- SELECT- u.*,- COALESCE(ocon.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(u.owner_id)) as max_concurrency- FROM updated u- LEFT JOIN ${this.queueName}_owner_concurrency ocon ON u.owner_id = ocon.id- WHERE u.owner_id IS NOT NULL- ),owner_increment AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, 1, max_concurrency- FROM updated_with_max- ON CONFLICT (id)- DO UPDATE SET- current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1,- max_concurrency = EXCLUDED.max_concurrency+ UPDATE ${this.queueName}_owner_concurrency oc+ SET current_concurrency = oc.current_concurrency + 1+ FROM updated u+ WHERE oc.id = u.owner_id+ AND u.owner_id IS NOT NULL)SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;`;@@ -1246,6 +1266,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLORDER BY owner_id),+ missing_owners AS (+ SELECT dtl.owner_id+ FROM distinct_owners_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_owner_concurrency oc+ WHERE oc.id = dtl.owner_id+ )+ ),+ ensure_owner_rows AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, 0, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)+ FROM missing_owners+ ),acquired_owner_locks AS (SELECT id as owner_idFROM ${this.queueName}_owner_concurrency@@ -1258,6 +1291,19 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE group_id IS NOT NULLORDER BY group_id),+ missing_groups AS (+ SELECT dtl.group_id+ FROM distinct_groups_to_lock dtl+ WHERE NOT EXISTS (+ SELECT 1 FROM ${this.queueName}_group_concurrency gc+ WHERE gc.id = dtl.group_id+ )+ ),+ ensure_group_rows AS (+ INSERT INTO ${this.queueName}_group_concurrency (id, current_concurrency, max_concurrency)+ SELECT group_id, 0, NULL+ FROM missing_groups+ ),acquired_group_locks AS (SELECT id as group_idFROM ${this.queueName}_group_concurrency@@ -1282,22 +1328,11 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLGROUP BY owner_id),- owner_counts_with_max AS (- SELECT- oc.owner_id,- oc.job_count,- COALESCE(ocon.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)) as max_concurrency- FROM owner_counts oc- LEFT JOIN ${this.queueName}_owner_concurrency ocon ON oc.owner_id = ocon.id- ),- owner_upsert AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, job_count, max_concurrency- FROM owner_counts_with_max- ON CONFLICT (id)- DO UPDATE SET- current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency,- max_concurrency = EXCLUDED.max_concurrency+ owner_update AS (+ UPDATE ${this.queueName}_owner_concurrency oc+ SET current_concurrency = oc.current_concurrency + owner_counts.job_count+ FROM owner_counts+ WHERE oc.id = owner_counts.owner_id),group_counts AS (SELECT group_id, COUNT(*)::int8 as job_count