Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): concurrency tracking (#2291)
Gergő Móriczd8da45cffc40
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex a1a5ac968..e7eece097 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -677,14 +677,59 @@ class NuQ<JobData = any, JobReturnValue = any> {public async prefetchJobs(_logger: Logger = logger): Promise<number> {const start = Date.now();try {- const jobs = (- await nuqPool.query(- `- WITH next AS (SELECT id FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 500)- UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};- `,- )- ).rows.map(row => this.rowToJob(row)!);+ const queryGetNext = `+ SELECT ${this.jobReturning.map(x => `${this.queueName}.${x}`).join(", ")} FROM ${this.queueName}+ WHERE ${this.queueName}.status = 'queued'::nuq.job_status+ ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC+ FOR UPDATE SKIP LOCKED LIMIT 500+ `;++ const queryUpdateStatus = `+ UPDATE ${this.queueName} q+ SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()+ FROM next+ WHERE q.id = next.id+ RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}+ `++ let updateQuery: string;+ if (this.options.concurrencyLimit === "per-owner") {+ // Single query: select, update jobs, and update owner concurrency+ updateQuery = `+ WITH next AS (+ ${queryGetNext}+ ),+ updated AS (+ ${queryUpdateStatus}+ ),+ owner_counts AS (+ SELECT owner_id, COUNT(*)::int8 as job_count+ FROM next+ WHERE owner_id IS NOT NULL+ GROUP BY owner_id+ ),+ owner_increment AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, job_count, ${/* TODO max concurrency */ 100}+ FROM owner_counts+ ON CONFLICT (id)+ DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency+ )+ SELECT * FROM updated;+ `;+ } else {+ // Single query: select and update jobs+ updateQuery = `+ WITH next AS (+ ${queryGetNext}+ )+ ${queryUpdateStatus}+ `;+ }++ const result = await nuqPool.query(updateQuery);++ const jobs = result.rows.map(row => this.rowToJob(row)!);for (const job of jobs) {await this.sendJobPrefetch(@@ -736,16 +781,50 @@ class NuQ<JobData = any, JobReturnValue = any> {}}- return this.rowToJob(- (- await nuqPool.query(- `- WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1)- UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};- `,+ const queryGetNext = `+ SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName}+ WHERE ${this.queueName}.status = 'queued'::nuq.job_status+ ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC+ FOR UPDATE SKIP LOCKED LIMIT 1+ `;++ const queryUpdateStatus = `+ UPDATE ${this.queueName} q+ SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()+ FROM next+ WHERE q.id = next.id+ RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}+ `;++ let updateQuery: string;+ if (this.options.concurrencyLimit === "per-owner") {+ updateQuery = `+ WITH next AS (+ ${queryGetNext}+ ),+ updated AS (+ ${queryUpdateStatus}+ ),+ owner_increment AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, 1, ${/* TODO max concurrency */ 100}+ FROM updated+ WHERE owner_id IS NOT NULL+ ON CONFLICT (id)+ DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1+ )+ SELECT * FROM updated;+ `;+ } else {+ updateQuery = `+ WITH next AS (+ ${queryGetNext})- ).rows[0],- )!;+ ${queryUpdateStatus}+ `;+ }++ return this.rowToJob((await nuqPool.query(updateQuery)).rows[0])!;} finally {logger.info("nuqGetJobToProcess metrics", {module: "nuq/metrics",@@ -794,26 +873,48 @@ class NuQ<JobData = any, JobReturnValue = any> {const start = Date.now();try {- const result = await nuqPool.query(- `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,- [id, lock, returnvalue],- );+ let updateQuery: string;+ if (this.options.concurrencyLimit === "per-owner") {+ // Update job and decrement owner concurrency in one query+ updateQuery = `+ WITH updated AS (+ UPDATE ${this.queueName}+ SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3+ WHERE id = $1 AND lock = $2+ RETURNING id, listen_channel_id, owner_id+ ),+ owner_decrement AS (+ UPDATE ${this.queueName}_owner_concurrency+ SET current_concurrency = GREATEST(0, current_concurrency - 1)+ FROM updated+ WHERE updated.owner_id IS NOT NULL AND ${this.queueName}_owner_concurrency.id = updated.owner_id+ )+ SELECT * FROM updated;+ `;+ } else {+ updateQuery = `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;+ }++ const result = await nuqPool.query(updateQuery, [id, lock, returnvalue]);const success = result.rowCount !== 0;if (success) {const job = result.rows[0];- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [- job.id + "|completed",- ]);- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {- await this.sendJobEnd(- job.id,- "completed",- job.listen_channel_id,- _logger,- );++ if (job) {+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [+ job.id + "|completed",+ ]);+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ await this.sendJobEnd(+ job.id,+ "completed",+ job.listen_channel_id,+ _logger,+ );+ }}}@@ -852,26 +953,48 @@ class NuQ<JobData = any, JobReturnValue = any> {const start = Date.now();try {- const result = await nuqPool.query(- `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,- [id, lock, failedReason],- );+ let updateQuery: string;+ if (this.options.concurrencyLimit === "per-owner") {+ // Update job and decrement owner concurrency in one query+ updateQuery = `+ WITH updated AS (+ UPDATE ${this.queueName}+ SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3+ WHERE id = $1 AND lock = $2+ RETURNING id, listen_channel_id, owner_id+ ),+ owner_decrement AS (+ UPDATE ${this.queueName}_owner_concurrency+ SET current_concurrency = GREATEST(0, current_concurrency - 1)+ FROM updated+ WHERE updated.owner_id IS NOT NULL AND ${this.queueName}_owner_concurrency.id = updated.owner_id+ )+ SELECT * FROM updated;+ `;+ } else {+ updateQuery = `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;+ }++ const result = await nuqPool.query(updateQuery, [id, lock, failedReason]);const success = result.rowCount !== 0;if (success) {const job = result.rows[0];- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [- job.id + "|failed",- ]);- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {- await this.sendJobEnd(- job.id,- "failed",- job.listen_channel_id,- _logger,- );++ if (job) {+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [+ job.id + "|failed",+ ]);+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ await this.sendJobEnd(+ job.id,+ "failed",+ job.listen_channel_id,+ _logger,+ );+ }}}@@ -956,7 +1079,9 @@ export async function nuqHealthCheck(): Promise<boolean> {// === Instances-export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {+ concurrencyLimit: "per-owner",+});// === Cleanupdiff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex b52c78ec5..79b7626b4 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -37,6 +37,13 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scCREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (+ id uuid NOT NULL,+ current_concurrency int8 NOT NULL,+ max_concurrency int8 NOT NULL,+ CONSTRAINT queue_scrape_owner_concurrency_pkey PRIMARY KEY (id)+);+SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';$$);