Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): fix group types and prefetch query
Gergő Móricz40404731dc45
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 7abc54872..669c0908f 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -136,12 +136,15 @@ export async function crawlController(});}- await crawlGroup.addGroup(id, [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency,- },- ]);+ await crawlGroup.addGroup(id, {+ concurrency: [+ {+ queue: scrapeQueue,+ maxConcurrency: sc.maxConcurrency,+ },+ ],+ ownerId: sc.team_id,+ });await saveCrawl(id, sc);diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 201ae2426..8150f3a96 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -891,38 +891,23 @@ class NuQ<JobData = any, JobReturnValue = any> {let updateQuery: string;if (this.options.concurrencyLimit === "per-owner") {updateQuery = `- WITH aggregate_jobs AS (+ WITH owner_limited_jobs AS (SELECT+ q.id,q.owner_id,- array_agg(- json_build_object(- 'id', q.id,- 'priority', q.priority,- 'created_at', q.created_at,- 'owner_id', q.owner_id- )+ ROW_NUMBER() OVER (+ PARTITION BY q.owner_idORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as jobs+ ) as owner_rank,+ GREATEST(COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id)) - COALESCE(oc.current_concurrency, 0), 0) as owner_limitFROM ${this.queueName} q+ LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_idWHERE q.status = 'queued'::nuq.job_status- GROUP BY q.owner_id- ),- aggregate_jobs_with_limit AS (- SELECT- aj.owner_id,- CASE- WHEN oc.max_concurrency IS NULL THEN aj.jobs- ELSE aj.jobs[:GREATEST(oc.max_concurrency - oc.current_concurrency, 0)]- END as jobs- FROM aggregate_jobs aj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = aj.owner_id),selected_jobs_with_metadata AS (- SELECT- (job->>'id')::uuid as id,- (job->>'owner_id')::uuid as owner_id- FROM aggregate_jobs_with_limit- CROSS JOIN LATERAL unnest(aggregate_jobs_with_limit.jobs) as job+ SELECT id, owner_id+ FROM owner_limited_jobs+ WHERE owner_rank <= owner_limit),distinct_owners_to_lock AS (SELECT DISTINCT owner_id@@ -973,63 +958,40 @@ class NuQ<JobData = any, JobReturnValue = any> {`;} else if (this.options.concurrencyLimit === "per-owner-per-group") {updateQuery = `- WITH aggregate_jobs AS (+ WITH group_limited_jobs AS (SELECT+ q.id,q.owner_id,q.group_id,- array_agg(- json_build_object(- 'id', q.id,- 'priority', q.priority,- 'created_at', q.created_at,- 'owner_id', q.owner_id,- 'group_id', q.group_id- )+ q.priority,+ q.created_at,+ ROW_NUMBER() OVER (+ PARTITION BY q.owner_id, q.group_idORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as jobs+ ) as group_rank,+ GREATEST(COALESCE(gc.max_concurrency, 999999) - COALESCE(gc.current_concurrency, 0), 0) as group_limitFROM ${this.queueName} q+ LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_idWHERE q.status = 'queued'::nuq.job_status- GROUP BY q.owner_id, q.group_id- ),- aggregate_jobs_with_group_limit AS (- SELECT- aj.owner_id,- aj.group_id,- CASE- WHEN gc.max_concurrency IS NULL THEN aj.jobs- ELSE aj.jobs[:GREATEST(gc.max_concurrency - gc.current_concurrency, 0)]- END as jobs- FROM aggregate_jobs aj- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = aj.group_id- ),- owner_aggregate_jobs AS (- SELECT- ajwgl.owner_id,- array_agg(- job- ORDER BY (job->>'priority')::int ASC, (job->>'created_at')::timestamptz ASC, (job->>'id')::uuid ASC- ) as jobs- FROM aggregate_jobs_with_group_limit ajwgl- CROSS JOIN LATERAL unnest(ajwgl.jobs) as job- GROUP BY ajwgl.owner_id),- owner_aggregate_jobs_with_limit AS (+ owner_limited_jobs AS (SELECT- oaj.owner_id,- CASE- WHEN oc.max_concurrency IS NULL THEN oaj.jobs- ELSE oaj.jobs[:GREATEST(oc.max_concurrency - oc.current_concurrency, 0)]- END as jobs- FROM owner_aggregate_jobs oaj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = oaj.owner_id+ glj.id,+ glj.owner_id,+ glj.group_id,+ ROW_NUMBER() OVER (+ PARTITION BY glj.owner_id+ ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC+ ) as owner_rank,+ GREATEST(COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id)) - COALESCE(oc.current_concurrency, 0), 0) as owner_limit+ FROM group_limited_jobs glj+ LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id+ WHERE glj.group_rank <= glj.group_limit),selected_jobs_with_metadata AS (- SELECT- (job->>'id')::uuid as id,- (job->>'owner_id')::uuid as owner_id,- (job->>'group_id')::uuid as group_id- FROM owner_aggregate_jobs_with_limit- CROSS JOIN LATERAL unnest(owner_aggregate_jobs_with_limit.jobs) as job+ SELECT id, owner_id, group_id+ FROM owner_limited_jobs+ WHERE owner_rank <= owner_limit),distinct_owners_to_lock AS (SELECT DISTINCT owner_id@@ -1176,38 +1138,23 @@ class NuQ<JobData = any, JobReturnValue = any> {let updateQuery: string;if (this.options.concurrencyLimit === "per-owner") {updateQuery = `- WITH aggregate_jobs AS (+ WITH owner_limited_jobs AS (SELECT+ q.id,q.owner_id,- array_agg(- json_build_object(- 'id', q.id,- 'priority', q.priority,- 'created_at', q.created_at,- 'owner_id', q.owner_id- )+ ROW_NUMBER() OVER (+ PARTITION BY q.owner_idORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as jobs+ ) as owner_rank,+ GREATEST(COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id)) - COALESCE(oc.current_concurrency, 0), 0) as owner_limitFROM ${this.queueName} q+ LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_idWHERE q.status = 'queued'::nuq.job_status- GROUP BY q.owner_id- ),- aggregate_jobs_with_limit AS (- SELECT- aj.owner_id,- CASE- WHEN oc.max_concurrency IS NULL THEN aj.jobs- ELSE aj.jobs[:GREATEST(oc.max_concurrency - oc.current_concurrency, 0)]- END as jobs- FROM aggregate_jobs aj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = aj.owner_id),selected_jobs_with_metadata AS (- SELECT- (job->>'id')::uuid as id,- (job->>'owner_id')::uuid as owner_id- FROM aggregate_jobs_with_limit- CROSS JOIN LATERAL unnest(aggregate_jobs_with_limit.jobs) as job+ SELECT id, owner_id+ FROM owner_limited_jobs+ WHERE owner_rank <= owner_limitLIMIT 1),distinct_owners_to_lock AS (@@ -1253,63 +1200,40 @@ class NuQ<JobData = any, JobReturnValue = any> {`;} else if (this.options.concurrencyLimit === "per-owner-per-group") {updateQuery = `- WITH aggregate_jobs AS (+ WITH group_limited_jobs AS (SELECT+ q.id,q.owner_id,q.group_id,- array_agg(- json_build_object(- 'id', q.id,- 'priority', q.priority,- 'created_at', q.created_at,- 'owner_id', q.owner_id,- 'group_id', q.group_id- )+ q.priority,+ q.created_at,+ ROW_NUMBER() OVER (+ PARTITION BY q.owner_id, q.group_idORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as jobs+ ) as group_rank,+ GREATEST(COALESCE(gc.max_concurrency, 999999) - COALESCE(gc.current_concurrency, 0), 0) as group_limitFROM ${this.queueName} q+ LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_idWHERE q.status = 'queued'::nuq.job_status- GROUP BY q.owner_id, q.group_id- ),- aggregate_jobs_with_group_limit AS (- SELECT- aj.owner_id,- aj.group_id,- CASE- WHEN gc.max_concurrency IS NULL THEN aj.jobs- ELSE aj.jobs[:GREATEST(gc.max_concurrency - gc.current_concurrency, 0)]- END as jobs- FROM aggregate_jobs aj- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = aj.group_id- ),- owner_aggregate_jobs AS (- SELECT- ajwgl.owner_id,- array_agg(- job- ORDER BY (job->>'priority')::int ASC, (job->>'created_at')::timestamptz ASC, (job->>'id')::uuid ASC- ) as jobs- FROM aggregate_jobs_with_group_limit ajwgl- CROSS JOIN LATERAL unnest(ajwgl.jobs) as job- GROUP BY ajwgl.owner_id),- owner_aggregate_jobs_with_limit AS (+ owner_limited_jobs AS (SELECT- oaj.owner_id,- CASE- WHEN oc.max_concurrency IS NULL THEN oaj.jobs- ELSE oaj.jobs[:GREATEST(oc.max_concurrency - oc.current_concurrency, 0)]- END as jobs- FROM owner_aggregate_jobs oaj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = oaj.owner_id+ glj.id,+ glj.owner_id,+ glj.group_id,+ ROW_NUMBER() OVER (+ PARTITION BY glj.owner_id+ ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC+ ) as owner_rank,+ GREATEST(COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id)) - COALESCE(oc.current_concurrency, 0), 0) as owner_limit+ FROM group_limited_jobs glj+ LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id+ WHERE glj.group_rank <= glj.group_limit),selected_jobs_with_metadata AS (- SELECT- (job->>'id')::uuid as id,- (job->>'owner_id')::uuid as owner_id,- (job->>'group_id')::uuid as group_id- FROM owner_aggregate_jobs_with_limit- CROSS JOIN LATERAL unnest(owner_aggregate_jobs_with_limit.jobs) as job+ SELECT id, owner_id, group_id+ FROM owner_limited_jobs+ WHERE owner_rank <= owner_limitLIMIT 1),distinct_owners_to_lock AS (