Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq/concurrency-tracking): fix deadlock (#2295)
Gergő Móricz0e33ebe90ae9
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex 286265a6b..9b3d18154 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -1,35 +1,45 @@import "dotenv/config";-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck, nuqShutdown } from "./nuq";+import {+ scrapeQueue,+ nuqGetLocalMetrics,+ nuqHealthCheck,+ nuqShutdown,+} from "./nuq";import Express from "express";import { logger } from "../../lib/logger";(async () => {- const app = Express();+ const app = Express();- app.get("/metrics", (_, res) => res.contentType("text/plain").send(nuqGetLocalMetrics()));- app.get("/health", async (_, res) => {- if (await nuqHealthCheck()) {- res.status(200).send("OK");- } else {- res.status(500).send("Not OK");- }- });+ app.get("/metrics", (_, res) =>+ res.contentType("text/plain").send(nuqGetLocalMetrics()),+ );+ app.get("/health", async (_, res) => {+ if (await nuqHealthCheck()) {+ res.status(200).send("OK");+ } else {+ res.status(500).send("Not OK");+ }+ });- const server = app.listen(Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011), () => {- logger.info("NuQ prefetch worker metrics server started");- });+ const server = app.listen(+ Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),+ () => {+ logger.info("NuQ prefetch worker metrics server started");+ },+ );- async function shutdown() {- server.close();- await nuqShutdown();- process.exit(0);- }+ async function shutdown() {+ server.close();+ await nuqShutdown();+ process.exit(0);+ }- process.on("SIGINT", shutdown);- process.on("SIGTERM", shutdown);+ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);- while (true) {- await scrapeQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 250));- }+ while (true) {+ await scrapeQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 100));+ }})();diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex eeccf65b3..e187f035e 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -686,11 +686,13 @@ class NuQ<JobData = any, JobReturnValue = any> {public async prefetchJobs(_logger: Logger = logger): Promise<number> {const start = Date.now();try {+ // With per-owner concurrency limiting: smaller batch size reduces lock contention.+ // Use blocking advisory locks to ensure jobs aren't skipped.const queryGetNext = `SELECT ${this.jobReturning.map(x => `${this.queueName}.${x}`).join(", ")} FROM ${this.queueName}WHERE ${this.queueName}.status = 'queued'::nuq.job_statusORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC- FOR UPDATE SKIP LOCKED LIMIT 500+ FOR UPDATE SKIP LOCKED LIMIT 100`;const queryUpdateStatus = `@@ -702,50 +704,67 @@ class NuQ<JobData = any, JobReturnValue = any> {`;let updateQuery: string;- // if (this.options.concurrencyLimit === "per-owner") {- // // Single query: select, update jobs, and update owner concurrency- // updateQuery = `- // WITH next AS (- // ${queryGetNext}- // ),- // updated AS (- // ${queryUpdateStatus}- // ),- // owner_counts AS (- // SELECT owner_id, COUNT(*)::int8 as job_count- // FROM next- // WHERE owner_id IS NOT NULL- // GROUP BY owner_id- // ),- // owner_counts_with_max AS (- // SELECT- // oc.owner_id,- // oc.job_count,- // CASE- // WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = oc.owner_id)- // THEN 0- // ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)- // END as max_concurrency- // FROM owner_counts oc- // ),- // owner_increment AS (- // INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- // SELECT owner_id, job_count, max_concurrency- // FROM owner_counts_with_max- // ON CONFLICT (id)- // DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency- // )- // SELECT * FROM updated;- // `;- // } else {- // Single query: select and update jobs- updateQuery = `+ if (this.options.concurrencyLimit === "per-owner") {+ updateQuery = `+ WITH next_initial AS (+ ${queryGetNext}+ ),+ distinct_owners AS (+ SELECT DISTINCT owner_id+ FROM next_initial+ WHERE owner_id IS NOT NULL+ ORDER BY owner_id -- Deterministic lock order prevents deadlocks+ ),+ acquired_locks AS (+ SELECT+ owner_id,+ pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ FROM distinct_owners+ ),+ next AS (+ SELECT n.*+ FROM next_initial n+ WHERE n.owner_id IS NULL+ OR EXISTS (SELECT 1 FROM acquired_locks WHERE owner_id = n.owner_id)+ ),+ updated AS (+ ${queryUpdateStatus}+ ),+ owner_counts AS (+ SELECT owner_id, COUNT(*)::int8 as job_count+ FROM updated+ WHERE owner_id IS NOT NULL+ GROUP BY owner_id+ ),+ owner_counts_with_max AS (+ SELECT+ oc.owner_id,+ oc.job_count,+ COALESCE(+ (SELECT max_concurrency FROM ${this.queueName}_owner_concurrency WHERE id = oc.owner_id),+ ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)+ ) as max_concurrency+ FROM owner_counts oc+ ),+ owner_upsert AS (+ INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)+ SELECT owner_id, job_count, max_concurrency+ FROM owner_counts_with_max+ ON CONFLICT (id)+ DO UPDATE SET+ current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency,+ max_concurrency = EXCLUDED.max_concurrency+ )+ SELECT * FROM updated;+ `;+ } else {+ updateQuery = `WITH next AS (${queryGetNext})${queryUpdateStatus}`;- // }+ }const result = await nuqPool.query(updateQuery);@@ -819,20 +838,37 @@ class NuQ<JobData = any, JobReturnValue = any> {let updateQuery: string;if (this.options.concurrencyLimit === "per-owner") {updateQuery = `- WITH next AS (+ WITH next_initial AS (${queryGetNext}),+ distinct_owners AS (+ SELECT DISTINCT owner_id+ FROM next_initial+ WHERE owner_id IS NOT NULL+ ORDER BY owner_id -- Deterministic lock order prevents deadlocks+ ),+ acquired_locks AS (+ SELECT+ owner_id,+ pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy+ FROM distinct_owners+ ),+ next AS (+ SELECT n.*+ FROM next_initial n+ WHERE n.owner_id IS NULL+ OR EXISTS (SELECT 1 FROM acquired_locks WHERE owner_id = n.owner_id)+ ),updated AS (${queryUpdateStatus}),updated_with_max AS (SELECTu.*,- CASE- WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = u.owner_id)- THEN 0- ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(u.owner_id)- END as max_concurrency+ COALESCE(+ (SELECT max_concurrency FROM ${this.queueName}_owner_concurrency WHERE id = u.owner_id),+ ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(u.owner_id)+ ) as max_concurrencyFROM updated uWHERE u.owner_id IS NOT NULL),@@ -841,7 +877,9 @@ class NuQ<JobData = any, JobReturnValue = any> {SELECT owner_id, 1, max_concurrencyFROM updated_with_maxON CONFLICT (id)- DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1+ DO UPDATE SET+ current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1,+ max_concurrency = EXCLUDED.max_concurrency)SELECT * FROM updated;`;