Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): group_id, job backlogs, and group add operations (#2309)
Gergő Móricz646efbe3fc66
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 173187409..0fd875f51 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -71,7 +71,6 @@"@ai-sdk/fireworks": "^0.2.4","@ai-sdk/google": "^1.2.3","@ai-sdk/google-vertex": "^2.2.15",- "@pinecone-database/pinecone": "^6.1.2","@ai-sdk/groq": "^1.2.1","@ai-sdk/openai": "^1.3.12","@apidevtools/json-schema-ref-parser": "^11.7.3",@@ -88,6 +87,7 @@"@opentelemetry/core": "^2.1.0","@opentelemetry/exporter-trace-otlp-http": "^0.205.0","@opentelemetry/sdk-node": "^0.205.0",+ "@pinecone-database/pinecone": "^6.1.2","@sentry/cli": "^2.33.1","@sentry/node": "^9.40.0","@sentry/profiling-node": "^9.40.0",@@ -137,7 +137,7 @@"tough-cookie": "^4.1.4","turndown": "^7.1.3","undici": "^7.10.0",- "uuid": "^10.0.0",+ "uuid": "^13.0.0","winston": "^3.14.2","ws": "^8.18.0","x402-express": "^0.6.5",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 43bf374bf..0bbbc7abb 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -227,8 +227,8 @@ importers:specifier: ^7.10.0version: 7.10.0uuid:- specifier: ^10.0.0- version: 10.0.0+ specifier: ^13.0.0+ version: 13.0.0winston:specifier: ^3.14.2version: 3.14.2@@ -6751,8 +6751,8 @@ packages:resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}engines: {node: '>= 0.4.0'}- resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}+ resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==}hasBin: true@@ -7537,7 +7537,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7611,7 +7611,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7814,11 +7814,11 @@ snapshots:ethereum-cryptography: 2.2.1micro-ftch: 0.3.1- '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:'@metamask/rpc-errors': 7.0.2eventemitter3: 5.0.1transitivePeerDependencies:- supports-color@@ -9860,7 +9860,7 @@ snapshots:dependencies:big.js: 6.2.2dayjs: 1.11.13transitivePeerDependencies:- bufferutil- typescript@@ -9873,7 +9873,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10019,7 +10019,7 @@ snapshots:'@walletconnect/logger': 2.1.2'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10072,7 +10072,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])bs58: 6.0.0valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10113,7 +10113,7 @@ snapshots:'@safe-global/[email protected]([email protected])([email protected])([email protected])([email protected])':dependencies:'@safe-global/safe-gateway-typescript-sdk': 3.23.1transitivePeerDependencies:- bufferutil- typescript@@ -11072,19 +11072,19 @@ snapshots:dependencies:'@types/yargs-parser': 21.0.3- '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':+ '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':dependencies:'@base-org/account': 1.1.1([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])'@coinbase/wallet-sdk': 4.3.6([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])- '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])'@safe-global/safe-apps-sdk': 9.1.0([email protected])([email protected])([email protected])([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))'@walletconnect/ethereum-provider': 2.21.1([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])cbw-sdk: '@coinbase/[email protected]'- porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))+ porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -11118,11 +11118,11 @@ snapshots:- wagmi- zod- '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:eventemitter3: 5.0.1mipd: 0.0.7([email protected])optionalDependencies:'@tanstack/query-core': 5.83.1@@ -11670,11 +11670,6 @@ snapshots:typescript: 5.8.3zod: 3.22.4- optionalDependencies:- typescript: 5.8.3- zod: 3.24.2-optionalDependencies:typescript: 5.8.3@@ -11685,11 +11680,6 @@ snapshots:typescript: 5.8.3zod: 3.22.4- optionalDependencies:- typescript: 5.8.3- zod: 3.24.2-optionalDependencies:typescript: 5.8.3@@ -14161,21 +14151,6 @@ snapshots:transitivePeerDependencies:- zod- dependencies:- '@adraffy/ens-normalize': 1.11.1- '@noble/ciphers': 1.3.0- '@noble/curves': 1.9.6- '@noble/hashes': 1.8.0- '@scure/bip32': 1.7.0- '@scure/bip39': 1.6.0- abitype: 1.1.1([email protected])([email protected])- eventemitter3: 5.0.1- optionalDependencies:- typescript: 5.8.3- transitivePeerDependencies:- - zod-dependencies:'@adraffy/ens-normalize': 1.11.1@@ -14415,21 +14390,21 @@ snapshots:- [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):+ [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):dependencies:- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))hono: 4.9.10idb-keyval: 6.2.2mipd: 0.0.7([email protected])ox: 0.9.8([email protected])([email protected])zod: 4.1.11optionalDependencies:'@tanstack/react-query': 5.84.1([email protected])react: 18.3.1typescript: 5.8.3- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- '@types/react'- immer@@ -15301,7 +15276,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}@@ -15359,23 +15334,6 @@ snapshots:- utf-8-validate- zod- dependencies:- '@noble/curves': 1.9.2- '@noble/hashes': 1.8.0- '@scure/bip32': 1.7.0- '@scure/bip39': 1.6.0- abitype: 1.0.8([email protected])([email protected])- ox: 0.8.6([email protected])([email protected])- ws: 8.18.2([email protected])([email protected])- optionalDependencies:- typescript: 5.8.3- transitivePeerDependencies:- - bufferutil- - utf-8-validate- - zod-dependencies:'@noble/curves': 1.9.2@@ -15397,14 +15355,14 @@ snapshots:dependencies:xml-name-validator: 5.0.0- [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):+ [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@tanstack/react-query': 5.84.1([email protected])- '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))react: 18.3.1use-sync-external-store: 1.4.0([email protected])optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -15611,8 +15569,8 @@ snapshots:'@solana-program/token-2022': 0.4.2(@solana/[email protected]([email protected])([email protected])([email protected]([email protected])([email protected])))(@solana/[email protected]([email protected])([email protected]))'@solana/kit': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))'@solana/transaction-confirmation': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])zod: 3.25.76transitivePeerDependencies:- '@azure/app-configuration'diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex f09b34c1a..913e4492b 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -340,14 +340,15 @@ export async function concurrentJobDone(job: NuQJob<any>) {abTestJob(nextJob.job.data);- await scrapeQueue.addJob(+ await scrapeQueue.promoteJobFromBacklogOrAdd(nextJob.job.id,+ nextJob.job.data,{- ...nextJob.job.data,- concurrencyLimitHit: true,+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ ownerId: nextJob.job.data.team_id ?? undefined,+ groupId: nextJob.job.data.crawl_id ?? undefined,},- nextJob.job.priority,- nextJob.job.listenable,);}}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 885be5005..36a1722ed 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -43,6 +43,21 @@ async function _addScrapeJobToConcurrencyQueue(priority: number = 0,listenable: boolean = false,) {+ await scrapeQueue.addJob(+ jobId,+ {+ ...webScraperOptions,+ concurrencyLimited: true,+ },+ {+ priority,+ listenable,+ ownerId: webScraperOptions.team_id ?? undefined,+ groupId: webScraperOptions.crawl_id ?? undefined,+ backlogged: true,+ },+ );+await pushConcurrencyLimitedJob(webScraperOptions.team_id,{@@ -57,6 +72,44 @@ async function _addScrapeJobToConcurrencyQueue();}+async function _addScrapeJobsToConcurrencyQueue(+ jobs: {+ data: any;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[],+) {+ await scrapeQueue.addJobs(+ jobs.map(job => ({+ id: job.jobId,+ data: job.data,+ options: {+ priority: job.priority,+ listenable: job.listenable ?? false,+ ownerId: job.data.team_id ?? undefined,+ groupId: job.data.crawl_id ?? undefined,+ backlogged: true,+ },+ })),+ );++ for (const job of jobs) {+ await pushConcurrencyLimitedJob(+ job.data.team_id,+ {+ id: job.jobId,+ data: job.data,+ priority: job.priority,+ listenable: job.listenable ?? false,+ },+ job.data.crawl_id+ ? Infinity+ : (job.data.scrapeOptions?.timeout ?? 60 * 1000),+ );+ }+}+export async function _addScrapeJobToBullMQ(webScraperOptions: ScrapeJobData,jobId: string,@@ -86,7 +139,59 @@ export async function _addScrapeJobToBullMQ(}}- return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);+ return await scrapeQueue.addJob(jobId, webScraperOptions, {+ priority,+ listenable,+ ownerId: webScraperOptions.team_id ?? undefined,+ groupId: webScraperOptions.crawl_id ?? undefined,+ });+}++async function _addScrapeJobsToBullMQ(+ jobs: {+ data: any;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[],+): Promise<NuQJob<ScrapeJobData>[]> {+ for (const job of jobs) {+ if (job.data.mode === "single_urls") {+ abTestJob(job.data);+ }++ if (job.data && job.data.team_id) {+ await pushConcurrencyLimitActiveJob(+ job.data.team_id,+ job.jobId,+ 60 * 1000,+ ); // 60s default timeout++ if (job.data.crawl_id) {+ const sc = await getCrawl(job.data.crawl_id);+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {+ await pushCrawlConcurrencyLimitActiveJob(+ job.data.crawl_id,+ job.jobId,+ 60 * 1000,+ );+ }+ }+ }+ }++ return await scrapeQueue.addJobs(+ jobs.map(job => ({+ id: job.jobId,+ data: job.data,+ options: {+ priority: job.priority,+ listenable: job.listenable ?? false,+ ownerId: job.data.team_id ?? undefined,+ groupId: job.data.crawl_id ?? undefined,+ },+ })),+ );}async function addScrapeJobRaw(@@ -181,10 +286,20 @@ async function addScrapeJobRaw(webScraperOptions.concurrencyLimited = true;- await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId, priority, listenable);+ await _addScrapeJobToConcurrencyQueue(+ webScraperOptions,+ jobId,+ priority,+ listenable,+ );return null;} else {- return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority, listenable);+ return await _addScrapeJobToBullMQ(+ webScraperOptions,+ jobId,+ priority,+ listenable,+ );}}@@ -368,27 +483,22 @@ export async function addScrapeJobs(}}- await Promise.all(- addToCQ.map(async job => {- const size = JSON.stringify(job.data).length;- await _addScrapeJobToConcurrencyQueue(- { ...job.data, traceContext },- job.jobId,- job.priority,- job.listenable,- );- }),+ await _addScrapeJobsToConcurrencyQueue(+ addToCQ.map(job => ({+ jobId: job.jobId,+ data: { ...job.data, traceContext },+ priority: job.priority,+ listenable: job.listenable,+ })),);- await Promise.all(- addToBull.map(async job => {- await _addScrapeJobToBullMQ(- { ...job.data, traceContext },- job.jobId,- job.priority,- job.listenable,- );- }),+ await _addScrapeJobsToBullMQ(+ addToBull.map(job => ({+ jobId: job.jobId,+ data: { ...job.data, traceContext },+ priority: job.priority,+ listenable: job.listenable,+ })),);}}diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 13a49a555..9c857364c 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -4,6 +4,7 @@ import { Client, Pool } from "pg";import { type ScrapeJobData } from "../../types";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import amqp from "amqplib";+import { v5 as uuidv5, validate as isUUID } from "uuid";// === Basics@@ -16,7 +17,12 @@ nuqPool.on("error", err =>logger.error("Error in NuQ idle client", { err, module: "nuq" }),);-export type NuQJobStatus = "queued" | "active" | "completed" | "failed"; // must match nuq.job_status enum+export type NuQJobStatus =+ | "queued"+ | "active"+ | "completed"+ | "failed"+ | "backlog";export type NuQJob<Data = any, ReturnValue = any> = {id: string;status: NuQJobStatus;@@ -28,14 +34,39 @@ export type NuQJob<Data = any, ReturnValue = any> = {returnvalue?: ReturnValue;failedReason?: string;lock?: string;+ ownerId?: string;+ groupId?: string;};+type NuQJobOptions = {+ priority?: number;+ listenable?: boolean;+ ownerId?: string;+ groupId?: string;+ backlogged?: boolean;+};++type NuQOptions = {+ backlog?: boolean;+};++// owner IDs can sometimes be non-UUID, so let's normalize it to avoid query breakage - mogery+const normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";+function normalizeOwnerId(ownerId: string | undefined | null): string | null {+ if (typeof ownerId !== "string") return null;+ if (isUUID(ownerId)) return ownerId;+ return uuidv5(ownerId, normalizedUUIDNamespace);+}+const listenChannelId = process.env.NUQ_POD_NAME ?? "main";// === Queueclass NuQ<JobData = any, JobReturnValue = any> {- constructor(public readonly queueName: string) {}+ constructor(+ public readonly queueName: string,+ public readonly options: NuQOptions,+ ) {}// === Listener@@ -85,7 +116,7 @@ class NuQ<JobData = any, JobReturnValue = any> {let reconnectTimeout: NodeJS.Timeout | null = null;- const onClose = (function onClose() {+ const onClose = function onClose() {logger.info("NuQ listener channel closed", {module: "nuq/rabbitmq",});@@ -104,7 +135,7 @@ class NuQ<JobData = any, JobReturnValue = any> {250,);return;- }).bind(this);+ }.bind(this);connection.on("close", onClose);channel.on("close", onClose);@@ -114,7 +145,7 @@ class NuQ<JobData = any, JobReturnValue = any> {(msg => {if (msg === null) {onClose();- return;+ return;}logger.info("NuQ job received", {@@ -266,7 +297,7 @@ class NuQ<JobData = any, JobReturnValue = any> {await this.startSender();if (this.sender) {- await this.sender.channel.sendToQueue(+ this.sender.channel.sendToQueue(this.queueName + ".listen." + listenChannelId,Buffer.from(status, "utf8"),{@@ -286,7 +317,7 @@ class NuQ<JobData = any, JobReturnValue = any> {await this.startSender();if (this.sender) {- await this.sender.channel.sendToQueue(+ this.sender.channel.sendToQueue(this.queueName + ".prefetch",Buffer.from(JSON.stringify(job), "utf8"),{@@ -314,13 +345,28 @@ class NuQ<JobData = any, JobReturnValue = any> {"returnvalue","failedreason","lock",+ "owner_id",+ "group_id",+ ];++ private readonly jobBacklogReturning = [+ "id",+ "created_at",+ "priority",+ "data",+ "listen_channel_id",+ "owner_id",+ "group_id",];- private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {+ private rowToJob(+ row: any,+ backlogged?: boolean,+ ): NuQJob<JobData, JobReturnValue> | null {if (!row) return null;return {id: row.id,- status: row.status,+ status: backlogged ? "backlog" : row.status,createdAt: new Date(row.created_at),priority: row.priority,data: row.data,@@ -329,6 +375,8 @@ class NuQ<JobData = any, JobReturnValue = any> {returnvalue: row.returnvalue ?? undefined,failedReason: row.failedreason ?? undefined,lock: row.lock ?? undefined,+ ownerId: row.owner_id ?? undefined,+ groupId: row.group_id ?? undefined,};}@@ -503,16 +551,15 @@ class NuQ<JobData = any, JobReturnValue = any> {public async addJob(id: string,data: JobData,- priority: number = 0,- listenable: boolean = false,+ options: NuQJobOptions,): Promise<NuQJob<JobData, JobReturnValue>> {return withSpan("nuq.addJob", async span => {setSpanAttributes(span, {"nuq.queue_name": this.queueName,"nuq.job_id": id,- "nuq.priority": priority,+ "nuq.priority": options.priority ?? 0,"nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": listenable,+ "nuq.listenable": options.listenable ?? false,});const start = Date.now();@@ -520,8 +567,15 @@ class NuQ<JobData = any, JobReturnValue = any> {const result = this.rowToJob((await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id) VALUES ($1, $2, $3, $4) RETURNING ${this.jobReturning.join(", ")};`,- [id, data, priority, listenable ? listenChannelId : null],+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id) VALUES ($1, $2, $3, $4, $5, $6) RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,+ [+ id,+ data,+ options.priority ?? 0,+ options.listenable ? listenChannelId : null,+ normalizeOwnerId(options.ownerId),+ options.groupId ?? null,+ ],)).rows[0],)!;@@ -547,6 +601,189 @@ class NuQ<JobData = any, JobReturnValue = any> {});}+ public async addJobs(+ jobs: Array<{+ id: string;+ data: JobData;+ options: NuQJobOptions;+ }>,+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ return withSpan("nuq.addJobs", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.jobs_count": jobs.length,+ });++ if (jobs.length === 0) {+ return [];+ }++ const start = Date.now();+ try {+ // Separate jobs into backlogged and non-backlogged groups+ const regularJobs: typeof jobs = [];+ const backloggedJobs: typeof jobs = [];++ for (const job of jobs) {+ if (job.options.backlogged) {+ backloggedJobs.push(job);+ } else {+ regularJobs.push(job);+ }+ }++ const results: NuQJob<JobData, JobReturnValue>[] = [];++ // Batch size: 6 params per job, stay well under PG's 65535 param limit+ // 1000 jobs = 6000 params, leaving plenty of headroom+ const BATCH_SIZE = 1000;++ // Helper function to build and execute bulk insert with batching+ const bulkInsert = async (+ jobsToInsert: typeof jobs,+ tableSuffix: string,+ ) => {+ if (jobsToInsert.length === 0) return;++ // Process in batches+ for (+ let offset = 0;+ offset < jobsToInsert.length;+ offset += BATCH_SIZE+ ) {+ const batch = jobsToInsert.slice(offset, offset + BATCH_SIZE);++ // Build the VALUES clause and parameters array+ const valuesPlaceholders: string[] = [];+ const params: any[] = [];++ for (let i = 0; i < batch.length; i++) {+ const job = batch[i];+ const baseIdx = i * 6 + 1;++ valuesPlaceholders.push(+ `($${baseIdx}, $${baseIdx + 1}, $${baseIdx + 2}, $${baseIdx + 3}, $${baseIdx + 4}, $${baseIdx + 5})`,+ );++ params.push(+ job.id,+ job.data,+ job.options.priority ?? 0,+ job.options.listenable ? listenChannelId : null,+ normalizeOwnerId(job.options.ownerId),+ job.options.groupId ?? null,+ );+ }++ const query = `INSERT INTO ${this.queueName}${tableSuffix} (id, data, priority, listen_channel_id, owner_id, group_id) VALUES ${valuesPlaceholders.join(", ")} RETURNING ${(tableSuffix === "_backlog" ? this.jobBacklogReturning : this.jobReturning).join(", ")};`;++ const result = await nuqPool.query(query, params);++ // Convert rows to jobs and maintain order+ const jobMap = new Map(+ result.rows.map(row => [+ row.id,+ this.rowToJob(row, tableSuffix === "_backlog")!,+ ]),+ );++ for (const job of batch) {+ const insertedJob = jobMap.get(job.id);+ if (insertedJob) {+ results.push(insertedJob);+ }+ }+ }+ };++ // Insert regular jobs+ await bulkInsert(regularJobs, "");++ // Insert backlogged jobs+ await bulkInsert(backloggedJobs, "_backlog");++ setSpanAttributes(span, {+ "nuq.jobs_created": results.length,+ "nuq.regular_jobs_count": regularJobs.length,+ "nuq.backlogged_jobs_count": backloggedJobs.length,+ });++ return results;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJobs metrics", {+ module: "nuq/metrics",+ method: "nuqAddJobs",+ duration,+ jobsCount: jobs.length,+ });+ }+ });+ }++ public async promoteJobFromBacklogOrAdd(+ id: string,+ data: JobData,+ options: NuQJobOptions,+ ): Promise<NuQJob<JobData, JobReturnValue>> {+ return withSpan("nuq.promoteJobFromBacklogOrAdd", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": options.priority ?? 0,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,+ "nuq.listenable": options.listenable ?? false,+ });++ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `+ WITH ins AS (+ INSERT INTO ${this.queueName} (id, data, created_at, priority, listen_channel_id, owner_id, group_id)+ SELECT b.id, b.data, b.created_at, b.priority, b.listen_channel_id, b.owner_id, b.group_id+ FROM ${this.queueName}_backlog b+ WHERE b.id = $1+ LIMIT 1+ RETURNING ${this.jobReturning.join(", ")}+ ), del AS (+ DELETE FROM ${this.queueName}_backlog+ WHERE id = $1+ )+ SELECT * FROM ins+ `,+ [id],+ )+ ).rows[0],+ );++ if (!result) {+ return await this.addJob(id, data, {+ ...options,+ backlogged: false,+ });+ }++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqPromoteJobFromBacklogOrAdd metrics", {+ module: "nuq/metrics",+ method: "nuqPromoteJobFromBacklogOrAdd",+ duration,+ });+ }+ });+ }+private readonly nuqWaitMode =process.env.NUQ_WAIT_MODE === "listen" || process.env.NUQ_RABBITMQ_URL? ("listen" as const)@@ -944,7 +1181,9 @@ export async function nuqHealthCheck(): Promise<boolean> {// === Instances-export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {+ backlog: true,+});// === Cleanupdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 14b888a40..0d4804b82 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -37,7 +37,7 @@ import { getJobPriority } from "../../lib/job-priority";import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";import { hasFormatOfType } from "../../lib/format-utils";import { getACUCTeam } from "../../controllers/auth";-import { createWebhookSender, WebhookEvent } from "../webhook";+import { createWebhookSender, WebhookEvent } from "../webhook/index";import { CustomError } from "../../lib/custom-error";import { startWebScraperPipeline } from "../../main/runWebScraper";import { CostTracking } from "../../lib/cost-tracking";@@ -338,10 +338,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {// Store robots blocked URLs in Redis setfor (const [url, reason] of links.denialReasons) {if (reason === "URL blocked by robots.txt") {- await recordRobotsBlocked(- job.data.crawl_id,- url- );+ await recordRobotsBlocked(job.data.crawl_id, url);}}@@ -552,15 +549,12 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {error instanceof Error &&error.message === "URL blocked by robots.txt") {- await recordRobotsBlocked(- job.data.crawl_id,- job.data.url,- );+ await recordRobotsBlocked(job.data.crawl_id, job.data.url);}} catch (e) {logger.debug("Failed to record top-level robots block", { e });}-+if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;@@ -1182,7 +1176,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {await concurrentJobDone(job);}} catch (error) {- logger.debug("Job failed", { error });+ logger.warn("Job failed", { error });Sentry.captureException(error);if (error instanceof TransportableError) {throw new Error(serializeTransportableError(error));diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex f22a6b83a..df49dad3d 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -22,6 +22,8 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (listen_channel_id text, -- for listenable jobs over rabbitmqreturnvalue jsonb, -- only for selfhostfailedreason text, -- only for selfhost+ owner_id uuid,+ group_id uuid,CONSTRAINT queue_scrape_pkey PRIMARY KEY (id));@@ -36,6 +38,17 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scCREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog (+ id uuid NOT NULL DEFAULT gen_random_uuid(),+ data jsonb,+ created_at timestamp with time zone NOT NULL DEFAULT now(),+ priority int NOT NULL DEFAULT 0,+ listen_channel_id text, -- for listenable jobs over rabbitmq+ owner_id uuid,+ group_id uuid,+ CONSTRAINT queue_scrape_backlog_pkey PRIMARY KEY (id)+);+SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';$$);