Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): revert to pre-concurrency limit tracking
Gergő Móricz186335ebb7d0
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERSindex ecba85ef9..959141572 100644--- a/.github/CODEOWNERS+++ b/.github/CODEOWNERS@@ -12,7 +12,7 @@/apps/api/src/controllers/v2/crawl* @mogery### /batch/scrape-/apps/api/src/controllers/v2/batch-scrape* @mogery+/apps/api/src/controllers/v2/batch-scrape* @mogery### /extract/apps/api/src/controllers/v2/extract* @nickscamara@@ -36,7 +36,7 @@/apps/api/src/controllers/v1/crawl* @mogery### /batch/scrape-/apps/api/src/controllers/v1/batch-scrape* @mogery+/apps/api/src/controllers/v1/batch-scrape* @mogery### /extract/apps/api/src/controllers/v1/extract* @nickscamara@@ -77,6 +77,9 @@### remnants of WebScraper/WebCrawler/apps/api/src/scraper/WebScraper/* @mogery @nickscamara+## concurrency limits+/apps/api/src/lib/concurrency-limit.ts @mogery @nickscamara+## BullMQ-related code/apps/api/src/services/queue-worker.ts @mogery @nickscamara/apps/api/src/services/worker/scrape-worker.ts @mogerydiff --git a/apps/api/package.json b/apps/api/package.jsonindex 404e738e2..173187409 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -71,6 +71,7 @@"@ai-sdk/fireworks": "^0.2.4","@ai-sdk/google": "^1.2.3","@ai-sdk/google-vertex": "^2.2.15",+ "@pinecone-database/pinecone": "^6.1.2","@ai-sdk/groq": "^1.2.1","@ai-sdk/openai": "^1.3.12","@apidevtools/json-schema-ref-parser": "^11.7.3",@@ -136,7 +137,7 @@"tough-cookie": "^4.1.4","turndown": "^7.1.3","undici": "^7.10.0",- "uuid": "^13.0.0",+ "uuid": "^10.0.0","winston": "^3.14.2","ws": "^8.18.0","x402-express": "^0.6.5",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 54b9eebfd..43bf374bf 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -76,6 +76,9 @@ importers:'@opentelemetry/sdk-node':specifier: ^0.205.0version: 0.205.0(@opentelemetry/[email protected])+ '@pinecone-database/pinecone':+ specifier: ^6.1.2+ version: 6.1.2'@sentry/cli':specifier: ^2.33.1version: 2.33.1([email protected])@@ -224,8 +227,8 @@ importers:specifier: ^7.10.0version: 7.10.0uuid:- specifier: ^13.0.0- version: 13.0.0+ specifier: ^10.0.0+ version: 10.0.0winston:specifier: ^3.14.2version: 3.14.2@@ -2623,6 +2626,10 @@ packages:resolution: {integrity: sha512-IHnV6A+zxU7XwmKFinmYjUcwlyK9+xkG3/s9KcQhI9BjQKycrJ1JRO+FbNYPwZiPKW3je/DR0k7w8/gLa5eaxQ==}deprecated: 'The package is now available as "qr": npm install qr'+ '@pinecone-database/[email protected]':+ resolution: {integrity: sha512-ydIlbtgIIHFgBL08sPzua5ckmOgtjgDz8xg21CnP1fqnnEgDmOlnfd10MRKU+fvFRhDlh4Md37SwZDr0d4cBqg==}+ engines: {node: '>=18.0.0'}+'@pkgjs/[email protected]':resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}engines: {node: '>=14'}@@ -6744,8 +6751,8 @@ packages:resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}engines: {node: '>= 0.4.0'}- resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==}+ resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}hasBin: true@@ -7530,7 +7537,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7604,7 +7611,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7807,11 +7814,11 @@ snapshots:ethereum-cryptography: 2.2.1micro-ftch: 0.3.1- '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:'@metamask/rpc-errors': 7.0.2eventemitter3: 5.0.1transitivePeerDependencies:- supports-color@@ -9795,6 +9802,8 @@ snapshots:'@paulmillr/[email protected]': {}+ '@pinecone-database/[email protected]': {}+'@pkgjs/[email protected]':optional: true@@ -9851,7 +9860,7 @@ snapshots:dependencies:big.js: 6.2.2dayjs: 1.11.13transitivePeerDependencies:- bufferutil- typescript@@ -9864,7 +9873,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10010,7 +10019,7 @@ snapshots:'@walletconnect/logger': 2.1.2'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10063,7 +10072,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])bs58: 6.0.0valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10104,7 +10113,7 @@ snapshots:'@safe-global/[email protected]([email protected])([email protected])([email protected])([email protected])':dependencies:'@safe-global/safe-gateway-typescript-sdk': 3.23.1transitivePeerDependencies:- bufferutil- typescript@@ -11063,19 +11072,19 @@ snapshots:dependencies:'@types/yargs-parser': 21.0.3- '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':+ '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':dependencies:'@base-org/account': 1.1.1([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])'@coinbase/wallet-sdk': 4.3.6([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])- '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])'@safe-global/safe-apps-sdk': 9.1.0([email protected])([email protected])([email protected])([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))'@walletconnect/ethereum-provider': 2.21.1([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])cbw-sdk: '@coinbase/[email protected]'- porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))+ porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -11109,11 +11118,11 @@ snapshots:- wagmi- zod- '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:eventemitter3: 5.0.1mipd: 0.0.7([email protected])optionalDependencies:'@tanstack/query-core': 5.83.1@@ -11661,6 +11670,11 @@ snapshots:typescript: 5.8.3zod: 3.22.4+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+optionalDependencies:typescript: 5.8.3@@ -11671,6 +11685,11 @@ snapshots:typescript: 5.8.3zod: 3.22.4+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+optionalDependencies:typescript: 5.8.3@@ -14142,6 +14161,21 @@ snapshots:transitivePeerDependencies:- zod+ dependencies:+ '@adraffy/ens-normalize': 1.11.1+ '@noble/ciphers': 1.3.0+ '@noble/curves': 1.9.6+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.1.1([email protected])([email protected])+ eventemitter3: 5.0.1+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - zod+dependencies:'@adraffy/ens-normalize': 1.11.1@@ -14381,21 +14415,21 @@ snapshots:- [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):+ [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):dependencies:- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))hono: 4.9.10idb-keyval: 6.2.2mipd: 0.0.7([email protected])ox: 0.9.8([email protected])([email protected])zod: 4.1.11optionalDependencies:'@tanstack/react-query': 5.84.1([email protected])react: 18.3.1typescript: 5.8.3- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- '@types/react'- immer@@ -15267,7 +15301,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}@@ -15325,6 +15359,23 @@ snapshots:- utf-8-validate- zod+ dependencies:+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ ox: 0.8.6([email protected])([email protected])+ ws: 8.18.2([email protected])([email protected])+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - bufferutil+ - utf-8-validate+ - zod+dependencies:'@noble/curves': 1.9.2@@ -15346,14 +15397,14 @@ snapshots:dependencies:xml-name-validator: 5.0.0- [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):+ [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@tanstack/react-query': 5.84.1([email protected])- '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))react: 18.3.1use-sync-external-store: 1.4.0([email protected])optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -15560,8 +15611,8 @@ snapshots:'@solana-program/token-2022': 0.4.2(@solana/[email protected]([email protected])([email protected])([email protected]([email protected])([email protected])))(@solana/[email protected]([email protected])([email protected]))'@solana/kit': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))'@solana/transaction-confirmation': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])zod: 3.25.76transitivePeerDependencies:- '@azure/app-configuration'diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 19d096fed..ba9fec277 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -2,7 +2,6 @@ import { configDotenv } from "dotenv";configDotenv();import { TeamFlags } from "../../controllers/v1/types";-import { Client as PgClient } from "pg";// =========================================// Configuration@@ -44,14 +43,12 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {runNumber = 0;}- const concurrency = req.concurrency ?? 100;-const res = await fetch(process.env.IDMUX_URL + "/", {method: "POST",body: JSON.stringify({refName: process.env.GITHUB_REF_NAME!,runNumber,- concurrency,+ concurrency: req.concurrency ?? 100,...req,}),headers: {@@ -64,24 +61,7 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {}expect(res.ok).toBe(true);- const result: Identity = await res.json();-- const nuqClient = new PgClient({- connectionString: process.env.NUQ_DATABASE_URL,- });-- // update the concurrency limit for the team in the nuq spoof table- await nuqClient.query(- `- INSERT INTO nuq.queue_scrape_owner_concurrency_source (id, max_concurrency) VALUES ($1, $2)- ON CONFLICT (id) DO UPDATE SET max_concurrency = EXCLUDED.max_concurrency- `,- [result.teamId, concurrency],- );-- await nuqClient.end();-- return result;+ return await res.json();}export type Identity = {diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 26172d214..598a85682 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -112,7 +112,7 @@ const mockPreviewACUC: (const mockACUC: () => AuthCreditUsageChunk = () => ({api_key: "bypass",api_key_id: 0,- team_id: "00000000-0000-0000-0000-000000000000",+ team_id: "bypass",sub_id: "bypass",sub_current_period_start: new Date().toISOString(),sub_current_period_end: new Date(@@ -396,7 +396,7 @@ export async function authenticateUser(return withAuth(supaAuthenticateUser, {success: true,chunk: null,- team_id: "00000000-0000-0000-0000-000000000000",+ team_id: "bypass",})(req, res, mode);}diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.tsnew file mode 100644index 000000000..76d2c415d--- /dev/null+++ b/apps/api/src/controllers/v0/admin/cclog.ts@@ -0,0 +1,67 @@+import { redisEvictConnection } from "../../../services/redis";+import { supabase_service } from "../../../services/supabase";+import { logger as _logger } from "../../../lib/logger";+import { Request, Response } from "express";++async function cclog() {+ const logger = _logger.child({+ module: "cclog",+ });++ let cursor = 0;+ do {+ const result = await redisEvictConnection.scan(+ cursor,+ "MATCH",+ "concurrency-limiter:*",+ "COUNT",+ 100000,+ );+ cursor = parseInt(result[0], 10);+ const usable = result[1].filter(x => !x.includes("preview_"));++ logger.info("Stepped", { cursor, usable: usable.length });++ if (usable.length > 0) {+ const entries: {+ team_id: string;+ concurrency: number;+ created_at: Date;+ }[] = [];++ for (const x of usable) {+ const at = new Date();+ const concurrency = await redisEvictConnection.zrangebyscore(+ x,+ Date.now(),+ Infinity,+ );+ if (concurrency) {+ entries.push({+ team_id: x.split(":")[1],+ concurrency: concurrency.length,+ created_at: at,+ });+ }+ }++ try {+ await supabase_service.from("concurrency_log").insert(entries);+ } catch (e) {+ logger.error("Error inserting", { error: e });+ }+ }+ } while (cursor != 0);+}++export async function cclogController(req: Request, res: Response) {+ try {+ await cclog();+ res.status(200).json({ ok: true });+ } catch (e) {+ _logger.error("Error", { module: "cclog", error: e });+ res.status(500).json({+ message: "Error",+ });+ }+}diff --git a/apps/api/src/controllers/v0/admin/crawl-check.ts b/apps/api/src/controllers/v0/admin/crawl-check.tsindex b462a1e7d..75b42c5e4 100644--- a/apps/api/src/controllers/v0/admin/crawl-check.ts+++ b/apps/api/src/controllers/v0/admin/crawl-check.ts@@ -6,6 +6,7 @@ import {StoredCrawl,} from "../../../lib/crawl-redis";import { supabase_service } from "../../../services/supabase";+import { getConcurrencyLimitedJobs } from "../../../lib/concurrency-limit";import { scrapeQueue } from "../../../services/worker/nuq";type AnalyzedCrawlPass1 = {@@ -216,8 +217,23 @@ export async function crawlCheckController(req: Request, res: Response) {await Promise.all(activeCrawls.map(analyzeCrawlPass1))).filter(result => result.success);+ const teamIds = [...new Set(firstPass.map(result => result.teamId))];+ const teamConcurrencyQueuedJobs = Object.fromEntries(+ await Promise.all(+ teamIds.map(async teamId => [+ teamId,+ await getConcurrencyLimitedJobs(teamId),+ ]),+ ),+ );+const results = await Promise.all(- firstPass.map(result => analyzeCrawlPass2({ ...result }, new Set())),+ firstPass.map(result =>+ analyzeCrawlPass2(+ { ...result },+ teamConcurrencyQueuedJobs[result.teamId],+ ),+ ),);// Clean results which should be "solved"diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex e9e1e80ec..59c38eb89 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -33,7 +33,6 @@ import { ZodError } from "zod";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0ScrapeOptions } from "../v2/types";import { isSelfHosted } from "../../lib/deployment";-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";export async function crawlController(req: Request, res: Response) {try {@@ -147,6 +146,35 @@ export async function crawlController(req: Request, res: Response) {});}+ // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?+ // try {+ // const a = new WebScraperDataProvider();+ // await a.setOptions({+ // jobId: uuidv4(),+ // mode: "single_urls",+ // urls: [url],+ // crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },+ // pageOptions: pageOptions,+ // });++ // const docs = await a.getDocuments(false, (progress) => {+ // job.updateProgress({+ // current: progress.current,+ // total: progress.total,+ // current_step: "SCRAPING",+ // current_url: progress.currentDocumentUrl,+ // });+ // });+ // return res.json({+ // success: true,+ // documents: docs,+ // });+ // } catch (error) {+ // logger.error(error);+ // return res.status(500).json({ error: error.message });+ // }+ // }+const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(pageOptions,undefined,@@ -175,16 +203,6 @@ export async function crawlController(req: Request, res: Response) {sc.robots = await crawler.getRobotsTxt();} catch (_) {}- await crawlGroup.addGroup(id, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: undefined,- },- ],- ownerId: sc.team_id,- });-await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex ed56d5397..c12ff7a9f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -25,7 +25,6 @@ import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -137,15 +136,6 @@ export async function batchScrapeController(};if (!req.body.appendToId) {- await crawlGroup.addGroup(id, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency,- },- ],- ownerId: sc.team_id,- });await saveCrawl(id, sc);await markCrawlActive(id);}diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.tsindex 26b11f74e..517aed515 100644--- a/apps/api/src/controllers/v1/concurrency-check.ts+++ b/apps/api/src/controllers/v1/concurrency-check.ts@@ -1,59 +1,27 @@import {- AuthCreditUsageChunkFromTeam,ConcurrencyCheckParams,ConcurrencyCheckResponse,RequestWithAuth,} from "./types";import { Response } from "express";-import { scrapeQueue } from "../../services/worker/nuq";-import { getACUCTeam } from "../auth";-import { RateLimiterMode } from "../../types";+import { redisEvictConnection } from "../../../src/services/redis";// Basically just middleware and error wrappingexport async function concurrencyCheckController(req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,res: Response<ConcurrencyCheckResponse>,) {- if (!req.acuc) {- return res.status(401).json({- success: false,- error: "Unauthorized",- });- }-- const ownerConcurrency = await scrapeQueue.getOwnerConcurrency(- req.auth.team_id,+ const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;+ const now = Date.now();+ const activeJobsOfTeam = await redisEvictConnection.zrangebyscore(+ concurrencyLimiterKey,+ now,+ Infinity,);- let maxConcurrency: number | null = ownerConcurrency?.maxConcurrency ?? null;-- if (maxConcurrency === null) {- let otherACUC: AuthCreditUsageChunkFromTeam | null = null;- if (!req.acuc.is_extract) {- otherACUC = await getACUCTeam(- req.auth.team_id,- false,- true,- RateLimiterMode.Extract,- );- } else {- otherACUC = await getACUCTeam(- req.auth.team_id,- false,- true,- RateLimiterMode.Crawl,- );- }-- maxConcurrency = Math.max(- req.acuc.concurrency,- otherACUC?.concurrency ?? 0,- );- }-return res.status(200).json({success: true,- concurrency: ownerConcurrency?.currentConcurrency ?? 0,- maxConcurrency,+ concurrency: activeJobsOfTeam.length,+ maxConcurrency: req.acuc?.concurrency ?? 0,});}diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex d96d5c1b9..6c4c6155b 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -18,6 +18,7 @@ import {} from "../../lib/crawl-redis";import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";+import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";import { getErrorContactMessage } from "../../lib/deployment";@@ -110,9 +111,10 @@ async function crawlStatusWS(setTimeout(loop, 1000);- let [_doneJobIDs, jobIDs] = await Promise.all([+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([getDoneJobsOrdered(req.params.jobId),getCrawlJobs(req.params.jobId),+ getConcurrencyLimitedJobs(req.auth.team_id),]);doneJobIDs = _doneJobIDs;@@ -122,10 +124,15 @@ async function crawlStatusWS(const validJobIDs: string[] = [];for (const id of jobIDs) {- const job = jobs.get(id);- if (job && job.status !== "failed") {- validJobStatuses.push([id, job.status]);+ if (throttledJobsSet.has(id)) {+ validJobStatuses.push([id, "queued"]);validJobIDs.push(id);+ } else {+ const job = jobs.get(id);+ if (job && job.status !== "failed") {+ validJobStatuses.push([id, job.status]);+ validJobIDs.push(id);+ }}}diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex cba3c6342..673d9d315 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -387,7 +387,7 @@ export async function crawlStatusController(data: scrapes,next:(outputBulkA.total ?? 0) > start + iteratedOver- ? `${req.protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};} else {@@ -445,7 +445,7 @@ export async function crawlStatusController(data: scrapes,next:(outputBulkA.total ?? 0) > start + iteratedOver- ? `${req.protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};}diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 669c0908f..e83489826 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -13,11 +13,10 @@ import {StoredCrawl,markCrawlActive,} from "../../lib/crawl-redis";-import { addScrapeJob } from "../../services/queue-jobs";+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -136,21 +135,11 @@ export async function crawlController(});}- await crawlGroup.addGroup(id, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency,- },- ],- ownerId: sc.team_id,- });-await saveCrawl(id, sc);await markCrawlActive(id);- await addScrapeJob(+ await _addScrapeJobToBullMQ({mode: "kickoff" as const,url: req.body.url,diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.tsindex 46c46107d..63e0157fc 100644--- a/apps/api/src/controllers/v1/queue-status.ts+++ b/apps/api/src/controllers/v1/queue-status.ts@@ -3,7 +3,12 @@ import { getACUCTeam } from "../auth";import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";import { Response } from "express";import { redisEvictConnection } from "../../services/redis";-import { scrapeQueue } from "../../services/worker/nuq";+import {+ cleanOldConcurrencyLimitedJobs,+ cleanOldConcurrencyLimitEntries,+ getConcurrencyLimitActiveJobsCount,+ getConcurrencyQueueJobsCount,+} from "../../lib/concurrency-limit";type QueueStatusResponse = {success: boolean;@@ -35,7 +40,12 @@ export async function queueStatusController();}- const jobCounts = await scrapeQueue.getOwnerJobCounts(req.auth.team_id);+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(+ req.auth.team_id,+ );+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id,@@ -44,9 +54,9 @@ export async function queueStatusController(return res.status(200).json({success: true,- jobsInQueue: jobCounts.active + jobCounts.queued,- activeJobsInQueue: jobCounts.active,- waitingJobsInQueue: jobCounts.queued,+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,+ activeJobsInQueue: activeJobsOfTeam,+ waitingJobsInQueue: queuedJobsOfTeam,maxConcurrency: Math.max(req.acuc?.concurrency ?? 1,otherACUC?.concurrency ?? 1,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex d0b362c21..04a478bb1 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -25,7 +25,6 @@ import { logger as _logger } from "../../lib/logger";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -130,15 +129,6 @@ export async function batchScrapeController(};if (!req.body.appendToId) {- await crawlGroup.addGroup(id, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency,- },- ],- ownerId: sc.team_id,- });await saveCrawl(id, sc);await markCrawlActive(id);}diff --git a/apps/api/src/controllers/v2/concurrency-check.ts b/apps/api/src/controllers/v2/concurrency-check.tsindex 04b6980ec..17edca1c4 100644--- a/apps/api/src/controllers/v2/concurrency-check.ts+++ b/apps/api/src/controllers/v2/concurrency-check.ts@@ -5,9 +5,9 @@ import {} from "./types";import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";+import { redisEvictConnection } from "../../../src/services/redis";import { getACUCTeam } from "../auth";import { RateLimiterMode } from "../../types";-import { scrapeQueue } from "../../services/worker/nuq";// Basically just middleware and error wrappingexport async function concurrencyCheckController(@@ -21,39 +21,34 @@ export async function concurrencyCheckController(});}- const ownerConcurrency = await scrapeQueue.getOwnerConcurrency(- req.auth.team_id,- );-- let maxConcurrency: number | null = ownerConcurrency?.maxConcurrency ?? null;-- if (maxConcurrency === null) {- let otherACUC: AuthCreditUsageChunkFromTeam | null = null;- if (!req.acuc.is_extract) {- otherACUC = await getACUCTeam(- req.auth.team_id,- false,- true,- RateLimiterMode.Extract,- );- } else {- otherACUC = await getACUCTeam(- req.auth.team_id,- false,- true,- RateLimiterMode.Crawl,- );- }-- maxConcurrency = Math.max(- req.acuc.concurrency,- otherACUC?.concurrency ?? 0,+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;+ if (!req.acuc.is_extract) {+ otherACUC = await getACUCTeam(+ req.auth.team_id,+ false,+ true,+ RateLimiterMode.Extract,+ );+ } else {+ otherACUC = await getACUCTeam(+ req.auth.team_id,+ false,+ true,+ RateLimiterMode.Crawl,);}+ const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;+ const now = Date.now();+ const activeJobsOfTeam = await redisEvictConnection.zrangebyscore(+ concurrencyLimiterKey,+ now,+ Infinity,+ );+return res.status(200).json({success: true,- concurrency: ownerConcurrency?.currentConcurrency ?? 0,- maxConcurrency,+ concurrency: activeJobsOfTeam.length,+ maxConcurrency: Math.max(req.acuc.concurrency, otherACUC?.concurrency ?? 0),});}diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex df5bfe5df..91c1dd809 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -18,6 +18,7 @@ import {} from "../../lib/crawl-redis";import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";+import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";import { getErrorContactMessage } from "../../lib/deployment";@@ -113,9 +114,10 @@ async function crawlStatusWS(setTimeout(loop, 1000);- let [_doneJobIDs, jobIDs] = await Promise.all([+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([getDoneJobsOrdered(req.params.jobId),getCrawlJobs(req.params.jobId),+ getConcurrencyLimitedJobs(req.auth.team_id),]);doneJobIDs = _doneJobIDs;@@ -125,10 +127,15 @@ async function crawlStatusWS(const validJobIDs: string[] = [];for (const id of jobIDs) {- const job = jobs.get(id);- if (job && job.status !== "failed") {- validJobStatuses.push([id, job.status]);+ if (throttledJobsSet.has(id)) {+ validJobStatuses.push([id, "queued"]);validJobIDs.push(id);+ } else {+ const job = jobs.get(id);+ if (job && job.status !== "failed") {+ validJobStatuses.push([id, job.status]);+ validJobIDs.push(id);+ }}}diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex e8ccd6f83..16e6d845c 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -406,7 +406,7 @@ export async function crawlStatusController(data: scrapes,next:(outputBulkA.total ?? 0) > start + iteratedOver- ? `${req.protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};} else {@@ -464,7 +464,7 @@ export async function crawlStatusController(data: scrapes,next:(outputBulkA.total ?? 0) > start + iteratedOver- ? `${req.protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};}diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 3825cc706..2a74b00fa 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -13,14 +13,13 @@ import {StoredCrawl,markCrawlActive,} from "../../lib/crawl-redis";-import { addScrapeJob } from "../../services/queue-jobs";+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";import { modifyCrawlUrl } from "../../utils/url-utils";-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -203,20 +202,11 @@ export async function crawlController(});}- await crawlGroup.addGroup(id, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency,- },- ],- ownerId: sc.team_id,- });await saveCrawl(id, sc);await markCrawlActive(id);- await addScrapeJob(+ await _addScrapeJobToBullMQ({url: req.body.url,mode: "kickoff" as const,diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.tsindex 1c11851f4..e52542dce 100644--- a/apps/api/src/controllers/v2/queue-status.ts+++ b/apps/api/src/controllers/v2/queue-status.ts@@ -4,7 +4,12 @@ import { RequestWithAuth } from "./types";import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";import { redisEvictConnection } from "../../services/redis";-import { scrapeQueue } from "../../services/worker/nuq";+import {+ cleanOldConcurrencyLimitedJobs,+ cleanOldConcurrencyLimitEntries,+ getConcurrencyLimitActiveJobsCount,+ getConcurrencyQueueJobsCount,+} from "../../lib/concurrency-limit";type QueueStatusResponse = {success: boolean;@@ -36,7 +41,12 @@ export async function queueStatusController();}- const jobCounts = await scrapeQueue.getOwnerJobCounts(req.auth.team_id);+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(+ req.auth.team_id,+ );+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id,@@ -45,9 +55,9 @@ export async function queueStatusController(return res.status(200).json({success: true,- jobsInQueue: jobCounts.active + jobCounts.queued,- activeJobsInQueue: jobCounts.active,- waitingJobsInQueue: jobCounts.queued,+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,+ activeJobsInQueue: activeJobsOfTeam,+ waitingJobsInQueue: queuedJobsOfTeam,maxConcurrency: Math.max(req.acuc?.concurrency ?? 1,otherACUC?.concurrency ?? 1,diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsnew file mode 100644index 000000000..f09b34c1a--- /dev/null+++ b/apps/api/src/lib/concurrency-limit.ts@@ -0,0 +1,355 @@+import { RateLimiterMode } from "../types";+import { redisEvictConnection } from "../services/redis";+import { getACUCTeam } from "../controllers/auth";+import { getCrawl, StoredCrawl } from "./crawl-redis";+import { logger } from "./logger";+import { abTestJob } from "../services/ab-test";+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";++const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;+const constructQueueKey = (team_id: string) =>+ "concurrency-limit-queue:" + team_id;++const constructCrawlKey = (crawl_id: string) =>+ "crawl-concurrency-limiter:" + crawl_id;++export async function cleanOldConcurrencyLimitEntries(+ team_id: string,+ now: number = Date.now(),+) {+ await redisEvictConnection.zremrangebyscore(+ constructKey(team_id),+ -Infinity,+ now,+ );+}++export async function getConcurrencyLimitActiveJobsCount(+ team_id: string,+): Promise<number> {+ return await redisEvictConnection.zcount(+ constructKey(team_id),+ Date.now(),+ Infinity,+ );+}++export async function getConcurrencyLimitActiveJobs(+ team_id: string,+ now: number = Date.now(),+): Promise<string[]> {+ return await redisEvictConnection.zrangebyscore(+ constructKey(team_id),+ now,+ Infinity,+ );+}++export async function pushConcurrencyLimitActiveJob(+ team_id: string,+ id: string,+ timeout: number,+ now: number = Date.now(),+) {+ await redisEvictConnection.zadd(constructKey(team_id), now + timeout, id);+}++async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {+ await redisEvictConnection.zrem(constructKey(team_id), id);+}++type ConcurrencyLimitedJob = {+ id: string;+ data: any;+ priority: number;+ listenable: boolean;+};++export async function cleanOldConcurrencyLimitedJobs(+ team_id: string,+ now: number = Date.now(),+) {+ await redisEvictConnection.zremrangebyscore(+ constructQueueKey(team_id),+ -Infinity,+ now,+ );+}++export async function pushConcurrencyLimitedJob(+ team_id: string,+ job: ConcurrencyLimitedJob,+ timeout: number,+ now: number = Date.now(),+) {+ const queueKey = constructQueueKey(team_id);+ await redisEvictConnection.zadd(queueKey, now + timeout, JSON.stringify(job));+ await redisEvictConnection.sadd("concurrency-limit-queues", queueKey);+}++export async function getConcurrencyLimitedJobs(team_id: string) {+ return new Set(+ (await redisEvictConnection.zrange(constructQueueKey(team_id), 0, -1)).map(+ x => JSON.parse(x).id,+ ),+ );+}++export async function getConcurrencyQueueJobsCount(+ team_id: string,+): Promise<number> {+ return await redisEvictConnection.zcount(+ constructQueueKey(team_id),+ Date.now(),+ Infinity,+ );+}++async function cleanOldCrawlConcurrencyLimitEntries(+ crawl_id: string,+ now: number = Date.now(),+) {+ await redisEvictConnection.zremrangebyscore(+ constructCrawlKey(crawl_id),+ -Infinity,+ now,+ );+}++export async function getCrawlConcurrencyLimitActiveJobs(+ crawl_id: string,+ now: number = Date.now(),+): Promise<string[]> {+ return await redisEvictConnection.zrangebyscore(+ constructCrawlKey(crawl_id),+ now,+ Infinity,+ );+}++export async function pushCrawlConcurrencyLimitActiveJob(+ crawl_id: string,+ id: string,+ timeout: number,+ now: number = Date.now(),+) {+ await redisEvictConnection.zadd(+ constructCrawlKey(crawl_id),+ now + timeout,+ id,+ );+}++async function removeCrawlConcurrencyLimitActiveJob(+ crawl_id: string,+ id: string,+) {+ await redisEvictConnection.zrem(constructCrawlKey(crawl_id), id);+}++/**+ * Grabs the next job from the team's concurrency limit queue. Handles crawl concurrency limits.+ *+ * This function may only be called once the outer code has verified that the team has not reached its concurrency limit.+ *+ * @param teamId+ * @returns A job that can be run, or null if there are no more jobs to run.+ */+async function getNextConcurrentJob(+ teamId: string,+ i = 0,+): Promise<{+ job: ConcurrencyLimitedJob;+ timeout: number;+} | null> {+ let finalJobs: {+ job: ConcurrencyLimitedJob;+ _member: string;+ timeout: number;+ }[] = [];++ const crawlCache = new Map<string, StoredCrawl>();+ let cursor: string = "0";++ do {+ const scanResult = await redisEvictConnection.zscan(+ constructQueueKey(teamId),+ cursor,+ "COUNT",+ 20,+ );+ cursor = scanResult[0];+ const results = scanResult[1];++ for (let i = 0; i < results.length; i += 2) {+ const res = {+ job: JSON.parse(results[i]),+ _member: results[i],+ timeout:+ results[i + 1] === "inf" ? Infinity : parseFloat(results[i + 1]),+ };++ // If the job is associated with a crawl ID, we need to check if the crawl has a max concurrency limit+ if (res.job.data.crawl_id) {+ const sc =+ crawlCache.get(res.job.data.crawl_id) ??+ (await getCrawl(res.job.data.crawl_id));+ if (sc !== null) {+ crawlCache.set(res.job.data.crawl_id, sc);+ }++ const maxCrawlConcurrency =+ sc === null+ ? null+ : typeof sc.crawlerOptions?.delay === "number" &&+ sc.crawlerOptions.delay > 0+ ? 1+ : (sc.maxConcurrency ?? null);++ if (maxCrawlConcurrency !== null) {+ // If the crawl has a max concurrency limit, we need to check if the crawl has reached the limit+ const currentActiveConcurrency = (+ await getCrawlConcurrencyLimitActiveJobs(res.job.data.crawl_id)+ ).length;+ if (currentActiveConcurrency < maxCrawlConcurrency) {+ // If we're under the max concurrency limit, we can run the job+ finalJobs.push(res);+ }+ } else {+ // If the crawl has no max concurrency limit, we can run the job+ finalJobs.push(res);+ }+ } else {+ // If the job is not associated with a crawl ID, we can run the job+ finalJobs.push(res);+ }+ }+ } while (finalJobs.length === 0 && cursor !== "0");++ let finalJob: (typeof finalJobs)[number] | null = null;+ if (finalJobs.length > 0) {+ for (const job of finalJobs) {+ const res = await redisEvictConnection.zrem(+ constructQueueKey(teamId),+ job._member,+ );+ if (res !== 0) {+ finalJob = job;+ break;+ }+ }++ if (finalJob === null) {+ // It's normal for this to happen, but if it happens too many times, we should log a warning+ if (i > 100) {+ logger.error(+ "Failed to remove job from concurrency limit queue, hard bailing",+ {+ teamId,+ jobIds: finalJobs.map(x => x.job.id),+ zeroDataRetention: finalJobs.some(+ x => x.job.data?.zeroDataRetention,+ ),+ i,+ },+ );+ return null;+ } else if (i > 15) {+ logger.warn("Failed to remove job from concurrency limit queue", {+ teamId,+ jobIds: finalJobs.map(x => x.job.id),+ zeroDataRetention: finalJobs.some(x => x.job.data?.zeroDataRetention),+ i,+ });+ }++ return await new Promise((resolve, reject) =>+ setTimeout(+ () => {+ getNextConcurrentJob(teamId, i + 1)+ .then(resolve)+ .catch(reject);+ },+ Math.floor(Math.random() * 300),+ ),+ ); // Stagger the workers off to break up the clump that causes the race condition+ } else {+ logger.debug("Removed job from concurrency limit queue", {+ teamId,+ jobId: finalJob.job.id,+ zeroDataRetention: finalJob.job.data?.zeroDataRetention,+ i,+ });+ }+ }++ return finalJob;+}++/**+ * Called when a job associated with a concurrency queue is done.+ *+ * @param job The BullMQ job that is done.+ */+export async function concurrentJobDone(job: NuQJob<any>) {+ if (job.id && job.data && job.data.team_id) {+ await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);+ await cleanOldConcurrencyLimitEntries(job.data.team_id);++ if (job.data.crawl_id) {+ await removeCrawlConcurrencyLimitActiveJob(job.data.crawl_id, job.id);+ await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);+ }++ const maxTeamConcurrency =+ (+ await getACUCTeam(+ job.data.team_id,+ false,+ true,+ job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+ const currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(job.data.team_id)+ ).length;++ if (currentActiveConcurrency < maxTeamConcurrency) {+ const nextJob = await getNextConcurrentJob(job.data.team_id);+ if (nextJob !== null) {+ await pushConcurrencyLimitActiveJob(+ job.data.team_id,+ nextJob.job.id,+ 60 * 1000,+ );++ if (nextJob.job.data.crawl_id) {+ await pushCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ 60 * 1000,+ );++ const sc = await getCrawl(nextJob.job.data.crawl_id);+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {+ await new Promise(resolve =>+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),+ );+ }+ }++ abTestJob(nextJob.job.data);++ await scrapeQueue.addJob(+ nextJob.job.id,+ {+ ...nextJob.job.data,+ concurrencyLimitHit: true,+ },+ nextJob.job.priority,+ nextJob.job.listenable,+ );+ }+ }+ }+}diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 1b256ea67..ff4ee837f 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -49,8 +49,14 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});}-export async function recordRobotsBlocked(crawlId: string, url: string) {- await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);+export async function recordRobotsBlocked(+ crawlId: string,+ url: string,+) {+ await redisEvictConnection.sadd(+ "crawl:" + crawlId + ":robots_blocked",+ url,+ );await redisEvictConnection.expire("crawl:" + crawlId + ":robots_blocked",24 * 60 * 60,diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.tsindex dd0f1da67..bfee651cf 100644--- a/apps/api/src/lib/search-index-client.ts+++ b/apps/api/src/lib/search-index-client.ts@@ -1,6 +1,6 @@/*** HTTP Client for Search Index Service- *+ ** This client communicates with the standalone search index service* (firecrawl-search backend) via HTTP.*/@@ -8,13 +8,13 @@import { logger as _logger } from "./logger";import type { Logger } from "winston";-interface SearchIndexClientConfig {+export interface SearchIndexClientConfig {baseUrl: string;apiSecret?: string;timeout?: number;}-interface IndexDocumentRequest {+export interface IndexDocumentRequest {url: string;resolvedUrl: string;title?: string;@@ -43,7 +43,7 @@ export interface SearchRequest {};}-interface SearchResult {+export interface SearchResult {documentId: string;url: string;title: string | null;@@ -58,7 +58,7 @@ interface SearchResult {lastCrawledAt: string;}-interface SearchResponse {+export interface SearchResponse {results: SearchResult[];total: number;query: string;@@ -75,13 +75,13 @@ export class SearchIndexClient {private baseUrl: string;private apiSecret?: string;private timeout: number;-+constructor(config: SearchIndexClientConfig) {this.baseUrl = config.baseUrl.replace(/\/$/, ""); // Remove trailing slashthis.apiSecret = config.apiSecret;this.timeout = config.timeout || 30000;}-+/*** Check if search index service is enabled*/@@ -91,7 +91,7 @@ export class SearchIndexClient {process.env.ENABLE_SEARCH_INDEX === "true");}-+/*** Make HTTP request to search service*/@@ -103,36 +103,36 @@ export class SearchIndexClient {): Promise<T> {const log = logger ?? _logger.child({ module: "search-index-client" });const url = `${this.baseUrl}${path}`;-+const headers: Record<string, string> = {"Content-Type": "application/json",};-+if (this.apiSecret) {headers["X-API-Secret"] = this.apiSecret;}-+const controller = new AbortController();const timeoutId = setTimeout(() => controller.abort(), this.timeout);-+try {log.debug("Making request to search service", {method,url,hasBody: !!body,});-+const response = await fetch(url, {method,headers,body: body ? JSON.stringify(body) : undefined,signal: controller.signal,});-+clearTimeout(timeoutId);-+const data = await response.json();-+if (!response.ok) {log.error("Search service request failed", {status: response.status,@@ -142,19 +142,16 @@ export class SearchIndexClient {data.error || `Search service returned ${response.status}`,);}-+return data;} catch (error) {clearTimeout(timeoutId);-+if (error.name === "AbortError") {- log.error("Search service request timed out", {- url,- timeout: this.timeout,- });+ log.error("Search service request timed out", { url, timeout: this.timeout });throw new Error("Search service request timed out");}-+log.error("Search service request failed", {error: (error as Error).message,url,@@ -162,7 +159,7 @@ export class SearchIndexClient {throw error;}}-+/*** Index a document (async - queues for processing)*/@@ -171,7 +168,7 @@ export class SearchIndexClient {logger?: Logger,): Promise<{ success: boolean; message: string }> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<any>("POST",@@ -179,12 +176,12 @@ export class SearchIndexClient {request,log,);-+log.info("Document queued for indexing", {url: request.url,success: response.success,});-+return {success: response.success,message: response.message || "Document queued for indexing",@@ -194,7 +191,7 @@ export class SearchIndexClient {error: (error as Error).message,url: request.url,});-+// Don't throw - indexing failures shouldn't break scrapingreturn {success: false,@@ -202,7 +199,7 @@ export class SearchIndexClient {};}}-+/*** Search indexed documents*/@@ -211,30 +208,32 @@ export class SearchIndexClient {logger?: Logger,): Promise<SearchResponse> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {- const response = await this.request<{- success: boolean;- data: SearchResponse;- }>("POST", "/api/search", request, log);-+ const response = await this.request<{ success: boolean; data: SearchResponse }>(+ "POST",+ "/api/search",+ request,+ log,+ );+if (!response.success || !response.data) {throw new Error("Invalid response from search service");}-+log.info("Search completed", {query: request.query,results: response.data.results.length,took: response.data.took,});-+return response.data;} catch (error) {log.error("Search request failed", {error: (error as Error).message,query: request.query,});-+// Return empty results on errorreturn {results: [],@@ -250,7 +249,7 @@ export class SearchIndexClient {};}}-+/*** Get search index statistics*/@@ -259,7 +258,7 @@ export class SearchIndexClient {queue: any;}> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<{ success: boolean; data: any }>("GET",@@ -267,30 +266,30 @@ export class SearchIndexClient {undefined,log,);-+if (!response.success || !response.data) {throw new Error("Invalid response from search service");}-+return response.data;} catch (error) {log.error("Failed to get search stats", {error: (error as Error).message,});-+return {index: {},queue: {},};}}-+/*** Health check*/async health(logger?: Logger): Promise<boolean> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<{ success: boolean }>("GET",@@ -298,7 +297,7 @@ export class SearchIndexClient {undefined,log,);-+return response.success;} catch (error) {log.warn("Search service health check failed", {@@ -319,22 +318,22 @@ export function getSearchIndexClient(): SearchIndexClient | null {if (!SearchIndexClient.isEnabled()) {return null;}-+if (!searchIndexClient) {const baseUrl = process.env.SEARCH_SERVICE_URL!;const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;-+searchIndexClient = new SearchIndexClient({baseUrl,apiSecret,timeout: 30000,});-+_logger.info("Search index client initialized", {baseUrl: baseUrl.substring(0, 30) + "...",});}-+return searchIndexClient;}@@ -346,11 +345,11 @@ export async function indexDocumentIfEnabled(logger?: Logger,): Promise<void> {const client = getSearchIndexClient();-+if (!client) {return;}-+try {await client.indexDocument(request, logger);} catch (error) {@@ -361,3 +360,4 @@ export async function indexDocumentIfEnabled(});}}+diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 205b3aae3..370e98291 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -3,6 +3,7 @@ import { redisHealthController } from "../controllers/v0/admin/redis-health";import { authMiddleware, blocklistMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";+import { cclogController } from "../controllers/v0/admin/cclog";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";@@ -30,6 +31,11 @@ adminRouter.get(wrap(checkFireEngine),);+adminRouter.get(+ `/admin/${process.env.BULL_AUTH_KEY}/cclog`,+ wrap(cclogController),+);+adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`,wrap(zdrcleanerController),diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 22b21f62c..e662e999b 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -34,10 +34,9 @@ import {toV0CrawlerOptions,} from "../../controllers/v2/types";import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";-import { addScrapeJob } from "../queue-jobs";+import { _addScrapeJobToBullMQ } from "../queue-jobs";import { BullMQOtel } from "bullmq-otel";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";-import { crawlGroup, scrapeQueue } from "../worker/nuq";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -491,19 +490,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {// });// }- await crawlGroup.addGroup(crawlId, {- concurrency: [- {- queue: scrapeQueue,- maxConcurrency: sc.maxConcurrency ?? undefined,- },- ],- ownerId: sc.team_id,- });-await saveCrawl(crawlId, sc);- await addScrapeJob(+ await _addScrapeJobToBullMQ({url: url,mode: "kickoff" as const,diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 6648c3f89..885be5005 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,8 +1,21 @@import { v4 as uuidv4 } from "uuid";-import { ScrapeJobData } from "../types";+import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";+import {+ cleanOldConcurrencyLimitEntries,+ getConcurrencyLimitActiveJobs,+ getConcurrencyQueueJobsCount,+ getCrawlConcurrencyLimitActiveJobs,+ pushConcurrencyLimitActiveJob,+ pushConcurrencyLimitedJob,+ pushCrawlConcurrencyLimitActiveJob,+} from "../lib/concurrency-limit";import { logger as _logger } from "../lib/logger";+import { sendNotificationWithCustomDays } from "./notification/email_notification";+import { shouldSendConcurrencyLimitNotification } from "./notification/notification-check";+import { getACUCTeam } from "../controllers/auth";import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";import { Document } from "../controllers/v1/types";+import { getCrawl } from "../lib/crawl-redis";import { Logger } from "winston";import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";@@ -10,6 +23,171 @@ import { abTestJob } from "./ab-test";import { NuQJob, scrapeQueue } from "./worker/nuq";import { serializeTraceContext } from "../lib/otel-tracer";+/**+ * Checks if a job is a crawl or batch scrape based on its options+ * @param options The job options containing crawlerOptions and crawl_id+ * @returns true if the job is either a crawl or batch scrape+ */+function isCrawlOrBatchScrape(options: {+ crawlerOptions?: any;+ crawl_id?: string;+}): boolean {+ // If crawlerOptions exists, it's a crawl+ // If crawl_id exists but no crawlerOptions, it's a batch scrape+ return !!options.crawlerOptions || !!options.crawl_id;+}++async function _addScrapeJobToConcurrencyQueue(+ webScraperOptions: any,+ jobId: string,+ priority: number = 0,+ listenable: boolean = false,+) {+ await pushConcurrencyLimitedJob(+ webScraperOptions.team_id,+ {+ id: jobId,+ data: webScraperOptions,+ priority,+ listenable,+ },+ webScraperOptions.crawl_id+ ? Infinity+ : (webScraperOptions.scrapeOptions?.timeout ?? 60 * 1000),+ );+}++export async function _addScrapeJobToBullMQ(+ webScraperOptions: ScrapeJobData,+ jobId: string,+ priority: number = 0,+ listenable: boolean = false,+): Promise<NuQJob<ScrapeJobData>> {+ if (webScraperOptions.mode === "single_urls") {+ abTestJob(webScraperOptions);+ }++ if (webScraperOptions && webScraperOptions.team_id) {+ await pushConcurrencyLimitActiveJob(+ webScraperOptions.team_id,+ jobId,+ 60 * 1000,+ ); // 60s default timeout++ if (webScraperOptions.crawl_id) {+ const sc = await getCrawl(webScraperOptions.crawl_id);+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {+ await pushCrawlConcurrencyLimitActiveJob(+ webScraperOptions.crawl_id,+ jobId,+ 60 * 1000,+ );+ }+ }+ }++ return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);+}++async function addScrapeJobRaw(+ webScraperOptions: ScrapeJobData,+ jobId: string,+ priority: number = 0,+ directToBullMQ: boolean = false,+ listenable: boolean = false,+): Promise<NuQJob<ScrapeJobData> | null> {+ let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;+ let currentActiveConcurrency = 0;+ let maxConcurrency = 0;++ if (directToBullMQ) {+ concurrencyLimited = "no";+ } else {+ if (webScraperOptions.crawl_id) {+ const crawl = await getCrawl(webScraperOptions.crawl_id);+ const concurrencyLimit = !crawl+ ? null+ : crawl.crawlerOptions?.delay === undefined &&+ crawl.maxConcurrency === undefined+ ? null+ : (crawl.maxConcurrency ?? 1);++ if (concurrencyLimit !== null) {+ const crawlConcurrency = (+ await getCrawlConcurrencyLimitActiveJobs(webScraperOptions.crawl_id)+ ).length;+ const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);+ if (freeSlots === 0) {+ concurrencyLimited = "yes-crawl";+ }+ }+ }++ if (concurrencyLimited === null) {+ const now = Date.now();+ const maxConcurrency =+ (+ await getACUCTeam(+ webScraperOptions.team_id,+ false,+ true,+ webScraperOptions.mode === "single_urls" &&+ webScraperOptions.from_extract+ ? RateLimiterMode.Extract+ : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+ await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);+ const currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)+ ).length;+ concurrencyLimited =+ currentActiveConcurrency >= maxConcurrency ? "yes" : "no";+ }+ }++ if (concurrencyLimited === "yes" || concurrencyLimited === "yes-crawl") {+ if (concurrencyLimited === "yes") {+ // Detect if they hit their concurrent limit+ // If above by 2x, send them an email+ // No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x+ const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(+ webScraperOptions.team_id,+ );+ if (concurrencyQueueJobs > maxConcurrency) {+ // logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);++ // Only send notification if it's not a crawl or batch scrape+ const shouldSendNotification =+ await shouldSendConcurrencyLimitNotification(+ webScraperOptions.team_id,+ );+ if (shouldSendNotification) {+ sendNotificationWithCustomDays(+ webScraperOptions.team_id,+ NotificationType.CONCURRENCY_LIMIT_REACHED,+ 15,+ false,+ true,+ ).catch(error => {+ _logger.error(+ "Error sending notification (concurrency limit reached)",+ { error },+ );+ });+ }+ }+ }++ webScraperOptions.concurrencyLimited = true;++ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId, priority, listenable);+ return null;+ } else {+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority, listenable);+ }+}+export async function addScrapeJob(webScraperOptions: ScrapeJobData,jobId: string = uuidv4(),@@ -24,22 +202,13 @@ export async function addScrapeJob(traceContext,};- if (webScraperOptions.mode === "single_urls") {- abTestJob(webScraperOptions);- }-- return await scrapeQueue.addJob(jobId, webScraperOptions, {+ return await addScrapeJobRaw(+ optionsWithTrace,+ jobId,priority,+ directToBullMQ,listenable,- ownerId: webScraperOptions.team_id,- groupId: webScraperOptions.crawl_id ?? undefined,- timesOutAt: webScraperOptions.crawl_id- ? undefined- : new Date(- Date.now() +- ((webScraperOptions as any).scrapeOptions?.timeout ?? 300) * 1000,- ),- });+ );}export async function addScrapeJobs(@@ -55,27 +224,173 @@ export async function addScrapeJobs(// Capture trace context for all jobsconst traceContext = serializeTraceContext();+ const jobsByTeam = new Map<+ string,+ {+ jobId: string;+ data: ScrapeJobData;+ priority: number;+ listenable?: boolean;+ }[]+ >();+for (const job of jobs) {- if (job.data.mode === "single_urls") {- abTestJob(job.data);+ if (!jobsByTeam.has(job.data.team_id)) {+ jobsByTeam.set(job.data.team_id, []);}+ jobsByTeam.get(job.data.team_id)!.push(job);}- await scrapeQueue.addJobs(- jobs.map(job => ({- data: {- ...job.data,- traceContext,- },- id: job.jobId,- options: {- priority: job.priority,- listenable: job.listenable,- ownerId: job.data.team_id,- groupId: job.data.crawl_id ?? undefined,- },- })),- );+ for (const [teamId, teamJobs] of jobsByTeam) {+ // == Buckets for jobs ==+ let jobsForcedToCQ: {+ data: ScrapeJobData;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[] = [];++ let jobsPotentiallyInCQ: {+ data: ScrapeJobData;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[] = [];++ // == Select jobs by crawl ID ==+ const jobsByCrawlID = new Map<+ string,+ {+ data: ScrapeJobData;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[]+ >();++ const jobsWithoutCrawlID: {+ data: ScrapeJobData;+ jobId: string;+ priority: number;+ listenable?: boolean;+ }[] = [];++ for (const job of teamJobs) {+ if (job.data.crawl_id) {+ if (!jobsByCrawlID.has(job.data.crawl_id)) {+ jobsByCrawlID.set(job.data.crawl_id, []);+ }+ jobsByCrawlID.get(job.data.crawl_id)!.push(job);+ } else {+ jobsWithoutCrawlID.push(job);+ }+ }++ // == Select jobs by crawl ID ==+ for (const [crawlID, crawlJobs] of jobsByCrawlID) {+ const crawl = await getCrawl(crawlID);+ const concurrencyLimit = !crawl+ ? null+ : crawl.crawlerOptions?.delay === undefined &&+ crawl.maxConcurrency === undefined+ ? null+ : (crawl.maxConcurrency ?? 1);++ if (concurrencyLimit === null) {+ // All jobs may be in the CQ depending on the global team concurrency limit+ jobsPotentiallyInCQ.push(...crawlJobs);+ } else {+ const crawlConcurrency = (+ await getCrawlConcurrencyLimitActiveJobs(crawlID)+ ).length;+ const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);++ // The first n jobs may be in the CQ depending on the global team concurrency limit+ jobsPotentiallyInCQ.push(...crawlJobs.slice(0, freeSlots));++ // Every job after that must be in the CQ, as the crawl concurrency limit has been reached+ jobsForcedToCQ.push(...crawlJobs.slice(freeSlots));+ }+ }++ // All jobs without a crawl ID may be in the CQ depending on the global team concurrency limit+ jobsPotentiallyInCQ.push(...jobsWithoutCrawlID);++ const now = Date.now();+ const maxConcurrency =+ (+ await getACUCTeam(+ teamId,+ false,+ true,+ jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract+ ? RateLimiterMode.Extract+ : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+ await cleanOldConcurrencyLimitEntries(teamId, now);++ const currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(teamId, now)+ ).length;++ const countCanBeDirectlyAdded = Math.max(+ maxConcurrency - currentActiveConcurrency,+ 0,+ );++ const addToBull = jobsPotentiallyInCQ.slice(0, countCanBeDirectlyAdded);+ const addToCQ = jobsPotentiallyInCQ+ .slice(countCanBeDirectlyAdded)+ .concat(jobsForcedToCQ);++ // equals 2x the max concurrency+ if (jobsPotentiallyInCQ.length - countCanBeDirectlyAdded > maxConcurrency) {+ // logger.info(`Concurrency limited 2x (multiple) - Concurrency queue jobs: ${addToCQ.length} Max concurrency: ${maxConcurrency} Team ID: ${jobs[0].data.team_id}`);+ // Only send notification if it's not a crawl or batch scrape+ if (!isCrawlOrBatchScrape(jobs[0].data)) {+ const shouldSendNotification =+ await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);+ if (shouldSendNotification) {+ sendNotificationWithCustomDays(+ jobs[0].data.team_id,+ NotificationType.CONCURRENCY_LIMIT_REACHED,+ 15,+ false,+ true,+ ).catch(error => {+ _logger.error(+ "Error sending notification (concurrency limit reached)",+ { error },+ );+ });+ }+ }+ }++ await Promise.all(+ addToCQ.map(async job => {+ const size = JSON.stringify(job.data).length;+ await _addScrapeJobToConcurrencyQueue(+ { ...job.data, traceContext },+ job.jobId,+ job.priority,+ job.listenable,+ );+ }),+ );++ await Promise.all(+ addToBull.map(async job => {+ await _addScrapeJobToBullMQ(+ { ...job.data, traceContext },+ job.jobId,+ job.priority,+ job.listenable,+ );+ }),+ );+ }}export async function waitForJob(diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.tsnew file mode 100644index 000000000..8b199f323--- /dev/null+++ b/apps/api/src/services/search-index-db.ts@@ -0,0 +1,104 @@+/**+ * Search Index Database Service+ *+ * Separate Supabase client for the search index database.+ * This allows independent scaling and isolation from the main index DB.+ */++import { createClient, SupabaseClient } from "@supabase/supabase-js";+import { logger as _logger } from "../lib/logger";+import { configDotenv } from "dotenv";++configDotenv();++/**+ * SearchIndexSupabaseService - Manages connection to search index database+ */+class SearchIndexSupabaseService {+ private client: SupabaseClient | null = null;++ constructor() {+ const supabaseUrl = process.env.SEARCH_INDEX_SUPABASE_URL;+ const supabaseServiceToken = process.env.SEARCH_INDEX_SUPABASE_SERVICE_TOKEN;++ // Only initialize if both URL and token are provided+ if (!supabaseUrl || !supabaseServiceToken) {+ _logger.warn("Search index database not configured. Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN to enable.");+ this.client = null;+ } else {+ this.client = createClient(supabaseUrl, supabaseServiceToken);+ _logger.info("Search index database client initialized", {+ url: supabaseUrl.substring(0, 30) + "...",+ });+ }+ }++ /**+ * Get the Supabase client for search index+ */+ getClient(): SupabaseClient | null {+ return this.client;+ }++ /**+ * Check if search index is enabled+ */+ isEnabled(): boolean {+ return this.client !== null;+ }+}++const searchIndexService = new SearchIndexSupabaseService();++/**+ * Proxy to provide clean error messages when search index is not configured+ */+export const search_index_supabase_service: SupabaseClient = new Proxy(+ searchIndexService,+ {+ get: function (target, prop, receiver) {+ const client = target.getClient();++ // If client is not initialized, throw immediately to prevent nested access errors+ if (client === null) {+ throw new Error(+ "Search index database is not configured. " ++ "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."+ );+ }++ // Direct access to service properties+ if (prop in target) {+ return Reflect.get(target, prop, receiver);+ }++ // Delegate to Supabase client+ return Reflect.get(client, prop, receiver);+ },+ }+) as unknown as SupabaseClient;++/**+ * Check if search index database is enabled+ */+export function isSearchIndexEnabled(): boolean {+ return (+ process.env.ENABLE_SEARCH_INDEX === "true" &&+ searchIndexService.isEnabled()+ );+}++/**+ * Get search index database client (throws if not configured)+ */+export function getSearchIndexClient(): SupabaseClient {+ const client = searchIndexService.getClient();+ if (!client) {+ throw new Error(+ "Search index database is not configured. " ++ "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN."+ );+ }+ return client;+}+diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 4131f4388..af3489d44 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -18,7 +18,7 @@ import { v4 as uuidv4 } from "uuid";import { addScrapeJobs } from "../queue-jobs";import { getJobs } from "../../controllers/v1/crawl-status";import { logJob } from "../logging/log_job";-import { createWebhookSender, WebhookEvent } from "../webhook/index";+import { createWebhookSender, WebhookEvent } from "../webhook";import { hasFormatOfType } from "../../lib/format-utils";import type { NuQJob } from "./nuq";import { ScrapeJobData } from "../../types";diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex ead9341e1..286265a6b 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -1,45 +1,35 @@import "dotenv/config";-import {- scrapeQueue,- nuqGetLocalMetrics,- nuqHealthCheck,- nuqShutdown,-} from "./nuq";+import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck, nuqShutdown } from "./nuq";import Express from "express";import { logger } from "../../lib/logger";(async () => {- const app = Express();+ const app = Express();- app.get("/metrics", (_, res) =>- res.contentType("text/plain").send(nuqGetLocalMetrics()),- );- app.get("/health", async (_, res) => {- if (await nuqHealthCheck()) {- res.status(200).send("OK");- } else {- res.status(500).send("Not OK");- }- });+ app.get("/metrics", (_, res) => res.contentType("text/plain").send(nuqGetLocalMetrics()));+ app.get("/health", async (_, res) => {+ if (await nuqHealthCheck()) {+ res.status(200).send("OK");+ } else {+ res.status(500).send("Not OK");+ }+ });- const server = app.listen(- Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),- () => {- logger.info("NuQ prefetch worker metrics server started");- },- );+ const server = app.listen(Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011), () => {+ logger.info("NuQ prefetch worker metrics server started");+ });- async function shutdown() {- server.close();- await nuqShutdown();- process.exit(0);- }+ async function shutdown() {+ server.close();+ await nuqShutdown();+ process.exit(0);+ }- process.on("SIGINT", shutdown);- process.on("SIGTERM", shutdown);+ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);- while (true) {- await scrapeQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 750));- }+ while (true) {+ await scrapeQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 250));+ }})();diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex cd7b6f781..13a49a555 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -4,7 +4,6 @@ import { Client, Pool } from "pg";import { type ScrapeJobData } from "../../types";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import amqp from "amqplib";-import { v5 as uuidv5, validate as uuidValidate } from "uuid";// === Basics@@ -29,42 +28,14 @@ export type NuQJob<Data = any, ReturnValue = any> = {returnvalue?: ReturnValue;failedReason?: string;lock?: string;- ownerId?: string;- groupId?: string;- timesOutAt?: Date;};const listenChannelId = process.env.NUQ_POD_NAME ?? "main";// === Queue-type NuQOptions = {- concurrencyLimit?: false | "per-owner" | "per-owner-per-group";-};--type NuQJobOptions = {- listenable?: boolean;- priority?: number;- ownerId?: string;- groupId?: string;- timesOutAt?: Date;-};--function normalizeOwnerId(ownerId: string | undefined | null) {- const bareOwnerId = ownerId ?? undefined;- const normalizedOwnerId = bareOwnerId- ? uuidValidate(bareOwnerId)- ? bareOwnerId- : uuidv5(bareOwnerId, "b208cbac-8bdf-4599-bf17-da78426e3f7c") // preview namespace- : null;- return normalizedOwnerId;-}-class NuQ<JobData = any, JobReturnValue = any> {- constructor(- public readonly queueName: string,- public readonly options: NuQOptions = { concurrencyLimit: false },- ) {}+ constructor(public readonly queueName: string) {}// === Listener@@ -114,7 +85,7 @@ class NuQ<JobData = any, JobReturnValue = any> {let reconnectTimeout: NodeJS.Timeout | null = null;- const onClose = function onClose() {+ const onClose = (function onClose() {logger.info("NuQ listener channel closed", {module: "nuq/rabbitmq",});@@ -133,7 +104,7 @@ class NuQ<JobData = any, JobReturnValue = any> {250,);return;- }.bind(this);+ }).bind(this);connection.on("close", onClose);channel.on("close", onClose);@@ -143,7 +114,7 @@ class NuQ<JobData = any, JobReturnValue = any> {(msg => {if (msg === null) {onClose();- return;+ return;}logger.info("NuQ job received", {@@ -295,7 +266,7 @@ class NuQ<JobData = any, JobReturnValue = any> {await this.startSender();if (this.sender) {- this.sender.channel.sendToQueue(+ await this.sender.channel.sendToQueue(this.queueName + ".listen." + listenChannelId,Buffer.from(status, "utf8"),{@@ -315,7 +286,7 @@ class NuQ<JobData = any, JobReturnValue = any> {await this.startSender();if (this.sender) {- this.sender.channel.sendToQueue(+ await this.sender.channel.sendToQueue(this.queueName + ".prefetch",Buffer.from(JSON.stringify(job), "utf8"),{@@ -343,9 +314,6 @@ class NuQ<JobData = any, JobReturnValue = any> {"returnvalue","failedreason","lock",- "owner_id",- "group_id",- "times_out_at",];private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {@@ -361,9 +329,6 @@ class NuQ<JobData = any, JobReturnValue = any> {returnvalue: row.returnvalue ?? undefined,failedReason: row.failedreason ?? undefined,lock: row.lock ?? undefined,- ownerId: row.owner_id ?? undefined,- groupId: row.group_id ?? undefined,- timesOutAt: row.times_out_at ? new Date(row.times_out_at) : undefined,};}@@ -485,60 +450,6 @@ class NuQ<JobData = any, JobReturnValue = any> {}}- public async getJobCountsOfGroup(- groupId: string,- _logger: Logger = logger,- ): Promise<Record<NuQJobStatus, number>> {- const start = Date.now();- try {- const stats = await nuqPool.query(- `- SELECT status, COUNT(*) as count FROM nuq.queue_scrape WHERE group_id = $1 GROUP BY status;- `,- [groupId],- );-- return Object.fromEntries(stats.rows.map(x => [x.status, x.count]));- } finally {- _logger.info("nuqGetJobCountsOfGroup metrics", {- module: "nuq/metrics",- method: "nuqGetJobCountsOfGroup",- duration: Date.now() - start,- groupId,- });- }- }-- public async getJobsOfGroupWithStatus(- groupId: string,- status: NuQJobStatus,- limit: number,- offset: number,- _logger: Logger = logger,- ): Promise<NuQJob<JobData, JobReturnValue>[]> {- const start = Date.now();- try {- return (- await nuqPool.query(- `- SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName}- WHERE group_id = $1 AND status = $2- ORDER BY finished_at ASC- LIMIT $3 OFFSET $4- `,- [groupId, status, limit, offset],- )- ).rows.map(x => this.rowToJob(x)!);- } finally {- _logger.info("nuqGetJobsOfGroupWithStatus metrics", {- module: "nuq/metrics",- method: "nuqGetJobsOfGroupWithStatus",- duration: Date.now() - start,- groupId,- });- }- }-public async removeJob(id: string,_logger: Logger = logger,@@ -589,72 +500,19 @@ class NuQ<JobData = any, JobReturnValue = any> {}// === Producer- public async tryAddJob(- id: string,- data: JobData,- options: NuQJobOptions = {},- ): Promise<NuQJob<JobData, JobReturnValue> | null> {- return withSpan("nuq.tryAddJob", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.job_id": id,- "nuq.priority": options.priority ?? 0,- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": options.listenable ?? false,- });-- const start = Date.now();- try {- const result = this.rowToJob(- (- await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING RETURNING ${this.jobReturning.join(", ")};`,- [- id,- data,- options.priority ?? 0,- options.listenable ? listenChannelId : null,- normalizeOwnerId(options.ownerId) ?? null,- options.groupId ?? null,- options.timesOutAt ? options.timesOutAt.toISOString() : null,- ],- )- ).rows[0],- )!;-- setSpanAttributes(span, {- "nuq.job_created": result !== null,- });-- return result;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- logger.info("nuqAddJob metrics", {- module: "nuq/metrics",- method: "nuqAddJob",- duration,- scrapeId: id,- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,- });- }- });- }-public async addJob(id: string,data: JobData,- options: NuQJobOptions = {},+ priority: number = 0,+ listenable: boolean = false,): Promise<NuQJob<JobData, JobReturnValue>> {return withSpan("nuq.addJob", async span => {setSpanAttributes(span, {"nuq.queue_name": this.queueName,"nuq.job_id": id,- "nuq.priority": options.priority ?? 0,+ "nuq.priority": priority,"nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": options.listenable ?? false,+ "nuq.listenable": listenable,});const start = Date.now();@@ -662,16 +520,8 @@ class NuQ<JobData = any, JobReturnValue = any> {const result = this.rowToJob((await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING ${this.jobReturning.join(", ")};`,- [- id,- data,- options.priority ?? 0,- options.listenable ? listenChannelId : null,- normalizeOwnerId(options.ownerId) ?? null,- options.groupId ?? null,- options.timesOutAt ? options.timesOutAt.toISOString() : null,- ],+ `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id) VALUES ($1, $2, $3, $4) RETURNING ${this.jobReturning.join(", ")};`,+ [id, data, priority, listenable ? listenChannelId : null],)).rows[0],)!;@@ -697,87 +547,6 @@ class NuQ<JobData = any, JobReturnValue = any> {});}- public async addJobs(- jobs: Array<{- id: string;- data: JobData;- options?: NuQJobOptions;- }>,- _logger: Logger = logger,- ): Promise<NuQJob<JobData, JobReturnValue>[]> {- if (jobs.length === 0) return [];-- return withSpan("nuq.addJobs", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.job_count": jobs.length,- });-- const start = Date.now();- try {- // Prepare arrays for bulk insert- const ids: string[] = [];- const dataArray: JobData[] = [];- const priorities: number[] = [];- const listenChannelIds: (string | null)[] = [];- const ownerIds: (string | null)[] = [];- const groupIds: (string | null)[] = [];- const timesOutAts: (string | null)[] = [];-- for (const job of jobs) {- ids.push(job.id);- dataArray.push(job.data);- priorities.push(job.options?.priority ?? 0);- listenChannelIds.push(- job.options?.listenable ? listenChannelId : null,- );- ownerIds.push(normalizeOwnerId(job.options?.ownerId));- groupIds.push(job.options?.groupId ?? null);- timesOutAts.push(- job.options?.timesOutAt- ? job.options.timesOutAt.toISOString()- : null,- );- }-- // Bulk insert using UNNEST- const result = await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at)- SELECT * FROM UNNEST($1::uuid[], $2::jsonb[], $3::int[], $4::text[], $5::uuid[], $6::uuid[], $7::timestamptz[])- RETURNING ${this.jobReturning.join(", ")};`,- [- ids,- dataArray,- priorities,- listenChannelIds,- ownerIds,- groupIds,- timesOutAts,- ],- );-- const createdJobs = result.rows.map(row => this.rowToJob(row)!);-- setSpanAttributes(span, {- "nuq.jobs_created": createdJobs.length,- });-- return createdJobs;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- _logger.info("nuqAddJobs metrics", {- module: "nuq/metrics",- method: "nuqAddJobs",- duration,- jobCount: jobs.length,- });- }- });- }-private readonly nuqWaitMode =process.env.NUQ_WAIT_MODE === "listen" || process.env.NUQ_RABBITMQ_URL? ("listen" as const)@@ -896,229 +665,14 @@ class NuQ<JobData = any, JobReturnValue = any> {public async prefetchJobs(_logger: Logger = logger): Promise<number> {const start = Date.now();try {- let shardingIndex = parseInt(- (process.env.NUQ_POD_NAME ?? "0").split("-").slice(-1)[0],- 10,- );- let shardingCount = parseInt(- process.env.NUQ_PREFETCH_REPLICAS ?? "1",- 10,- );-- if (- isNaN(shardingIndex) ||- isNaN(shardingCount) ||- shardingIndex >= shardingCount ||- shardingIndex < 0 ||- shardingCount < 1- ) {- _logger.warn(- "Sharding config invalid, falling back to guaranteed working defaults.",- {- shardingCount,- shardingIndex,- module: "nuq",- method: "nuqPrefetchJobs",- },- );- shardingIndex = 0;- shardingCount = 1;- }-- let updateQuery: string;- if (this.options.concurrencyLimit === "per-owner") {- updateQuery = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- owner_limited_jobs AS (- SELECT- q.id,- q.owner_id,- ROW_NUMBER() OVER (- PARTITION BY q.owner_id- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as owner_rank,- GREATEST(- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id))- - COALESCE(oac.active_count, 0),- 0- ) as owner_limit- FROM ${this.queueName} q- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_id- LEFT JOIN owner_active_counts oac ON oac.owner_id = q.owner_id- WHERE q.status = 'queued'::nuq.job_status- AND (abs(hashtext(q.id::text)) % ${shardingCount}) = ${shardingIndex}- ),- selected_jobs_with_metadata AS (- SELECT id, owner_id- FROM owner_limited_jobs- WHERE owner_rank <= owner_limit- ),- locked_jobs AS (- SELECT q.id, q.owner_id- FROM ${this.queueName} q- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)- AND q.status = 'queued'::nuq.job_status- FOR UPDATE SKIP LOCKED- ),- missing_owners AS (- SELECT DISTINCT owner_id- FROM locked_jobs- WHERE owner_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = owner_id- )- ),- ensure_owner_rows AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)- FROM missing_owners- ON CONFLICT (id) DO NOTHING- ),- updated AS (- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM locked_jobs)- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}- )- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;- `;- } else if (this.options.concurrencyLimit === "per-owner-per-group") {- updateQuery = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- group_active_counts AS (- SELECT- group_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL- GROUP BY group_id- ),- group_limited_jobs AS (- SELECT- q.id,- q.owner_id,- q.group_id,- q.priority,- q.created_at,- ROW_NUMBER() OVER (- PARTITION BY q.owner_id, q.group_id- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as group_rank,- GREATEST(- COALESCE(gc.max_concurrency, 999999)- - COALESCE(gac.active_count, 0),- 0- ) as group_limit- FROM ${this.queueName} q- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_id- LEFT JOIN group_active_counts gac ON gac.group_id = q.group_id- WHERE q.status = 'queued'::nuq.job_status- AND (abs(hashtext(q.id::text)) % ${shardingCount}) = ${shardingIndex}- ),- owner_limited_jobs AS (- SELECT- glj.id,- glj.owner_id,- glj.group_id,- ROW_NUMBER() OVER (- PARTITION BY glj.owner_id- ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC- ) as owner_rank,- GREATEST(- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id))- - COALESCE(oac.active_count, 0),- 0- ) as owner_limit- FROM group_limited_jobs glj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id- LEFT JOIN owner_active_counts oac ON oac.owner_id = glj.owner_id- WHERE glj.group_rank <= glj.group_limit- ),- selected_jobs_with_metadata AS (- SELECT id, owner_id, group_id- FROM owner_limited_jobs- WHERE owner_rank <= owner_limit- ),- locked_jobs AS (- SELECT q.id, q.owner_id, q.group_id- FROM ${this.queueName} q- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)- AND q.status = 'queued'::nuq.job_status- FOR UPDATE SKIP LOCKED- ),- missing_owners AS (- SELECT DISTINCT owner_id- FROM locked_jobs- WHERE owner_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = owner_id- )- ),- ensure_owner_rows AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)- FROM missing_owners- ON CONFLICT (id) DO NOTHING- ),- missing_groups AS (- SELECT DISTINCT group_id- FROM locked_jobs- WHERE group_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_group_concurrency gc- WHERE gc.id = group_id- )- ),- ensure_group_rows AS (- INSERT INTO ${this.queueName}_group_concurrency (id, max_concurrency)- SELECT group_id, NULL- FROM missing_groups- ON CONFLICT (id) DO NOTHING- ),- updated AS (- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM locked_jobs)- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}- )- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;- `;- } else {- updateQuery = `- WITH selected_jobs AS (- SELECT j.id- FROM ${this.queueName} j- WHERE j.status = 'queued'::nuq.job_status- ORDER BY j.priority ASC, j.created_at ASC- FOR UPDATE SKIP LOCKED- LIMIT 100- )- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM selected_jobs)- RETURNING ${this.jobReturning.join(", ")};- `;- }-- const result = await nuqPool.query(updateQuery);-- const jobs = result.rows.map(row => this.rowToJob(row)!);+ const jobs = (+ await nuqPool.query(+ `+ WITH next AS (SELECT id FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 500)+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};+ `,+ )+ ).rows.map(row => this.rowToJob(row)!);for (const job of jobs) {await this.sendJobPrefetch(@@ -1170,198 +724,16 @@ class NuQ<JobData = any, JobReturnValue = any> {}}- let updateQuery: string;- if (this.options.concurrencyLimit === "per-owner") {- updateQuery = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- owner_limited_jobs AS (- SELECT- q.id,- q.owner_id,- ROW_NUMBER() OVER (- PARTITION BY q.owner_id- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as owner_rank,- GREATEST(- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id))- - COALESCE(oac.active_count, 0),- 0- ) as owner_limit- FROM ${this.queueName} q- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_id- LEFT JOIN owner_active_counts oac ON oac.owner_id = q.owner_id- WHERE q.status = 'queued'::nuq.job_status- ),- selected_jobs_with_metadata AS (- SELECT id, owner_id- FROM owner_limited_jobs- WHERE owner_rank <= owner_limit- LIMIT 1- ),- locked_jobs AS (- SELECT q.id, q.owner_id- FROM ${this.queueName} q- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)- AND q.status = 'queued'::nuq.job_status- FOR UPDATE SKIP LOCKED- ),- missing_owners AS (- SELECT DISTINCT owner_id- FROM locked_jobs- WHERE owner_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = owner_id- )- ),- ensure_owner_rows AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)- FROM missing_owners- ON CONFLICT (id) DO NOTHING- ),- updated AS (- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM locked_jobs)- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}- )- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;- `;- } else if (this.options.concurrencyLimit === "per-owner-per-group") {- updateQuery = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- group_active_counts AS (- SELECT- group_id,- COUNT(*)::int8 as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL- GROUP BY group_id- ),- group_limited_jobs AS (- SELECT- q.id,- q.owner_id,- q.group_id,- q.priority,- q.created_at,- ROW_NUMBER() OVER (- PARTITION BY q.owner_id, q.group_id- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC- ) as group_rank,- GREATEST(- COALESCE(gc.max_concurrency, 999999)- - COALESCE(gac.active_count, 0),- 0- ) as group_limit- FROM ${this.queueName} q- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_id- LEFT JOIN group_active_counts gac ON gac.group_id = q.group_id- WHERE q.status = 'queued'::nuq.job_status- ),- owner_limited_jobs AS (- SELECT- glj.id,- glj.owner_id,- glj.group_id,- ROW_NUMBER() OVER (- PARTITION BY glj.owner_id- ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC- ) as owner_rank,- GREATEST(- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id))- - COALESCE(oac.active_count, 0),- 0- ) as owner_limit- FROM group_limited_jobs glj- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id- LEFT JOIN owner_active_counts oac ON oac.owner_id = glj.owner_id- WHERE glj.group_rank <= glj.group_limit- ),- selected_jobs_with_metadata AS (- SELECT id, owner_id, group_id- FROM owner_limited_jobs- WHERE owner_rank <= owner_limit- LIMIT 1- ),- locked_jobs AS (- SELECT q.id, q.owner_id, q.group_id- FROM ${this.queueName} q- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)- AND q.status = 'queued'::nuq.job_status- FOR UPDATE SKIP LOCKED- ),- missing_owners AS (- SELECT DISTINCT owner_id- FROM locked_jobs- WHERE owner_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_owner_concurrency oc- WHERE oc.id = owner_id- )- ),- ensure_owner_rows AS (- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)- FROM missing_owners- ON CONFLICT (id) DO NOTHING- ),- missing_groups AS (- SELECT DISTINCT group_id- FROM locked_jobs- WHERE group_id IS NOT NULL- AND NOT EXISTS (- SELECT 1 FROM ${this.queueName}_group_concurrency gc- WHERE gc.id = group_id- )- ),- ensure_group_rows AS (- INSERT INTO ${this.queueName}_group_concurrency (id, max_concurrency)- SELECT group_id, NULL- FROM missing_groups- ON CONFLICT (id) DO NOTHING- ),- updated AS (- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM locked_jobs)- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}- )- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;- `;- } else {- updateQuery = `- WITH selected_jobs AS (- SELECT j.id- FROM ${this.queueName} j- WHERE j.status = 'queued'::nuq.job_status- ORDER BY j.priority ASC, j.created_at ASC- FOR UPDATE SKIP LOCKED- LIMIT 1+ return this.rowToJob(+ (+ await nuqPool.query(+ `+ WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1)+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};+ `,)- UPDATE ${this.queueName} q- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()- WHERE q.id IN (SELECT id FROM selected_jobs)- RETURNING ${this.jobReturning.join(", ")};- `;- }-- return this.rowToJob((await nuqPool.query(updateQuery)).rows[0])!;+ ).rows[0],+ )!;} finally {logger.info("nuqGetJobToProcess metrics", {module: "nuq/metrics",@@ -1410,36 +782,26 @@ class NuQ<JobData = any, JobReturnValue = any> {const start = Date.now();try {- const updateQuery = `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;-- const result = await nuqPool.query(updateQuery, [- id,- lock,- returnvalue,- ]);+ const result = await nuqPool.query(+ `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,+ [id, lock, returnvalue],+ );const success = result.rowCount !== 0;if (success) {const job = result.rows[0];-- if (job) {- if (- this.nuqWaitMode === "listen" &&- !process.env.NUQ_RABBITMQ_URL- ) {- await nuqPool.query(- `SELECT pg_notify('${this.queueName}', $1);`,- [job.id + "|completed"],- );- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {- await this.sendJobEnd(- job.id,- "completed",- job.listen_channel_id,- _logger,- );- }+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [+ job.id + "|completed",+ ]);+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ await this.sendJobEnd(+ job.id,+ "completed",+ job.listen_channel_id,+ _logger,+ );}}@@ -1478,36 +840,26 @@ class NuQ<JobData = any, JobReturnValue = any> {const start = Date.now();try {- const updateQuery = `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;-- const result = await nuqPool.query(updateQuery, [- id,- lock,- failedReason,- ]);+ const result = await nuqPool.query(+ `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,+ [id, lock, failedReason],+ );const success = result.rowCount !== 0;if (success) {const job = result.rows[0];-- if (job) {- if (- this.nuqWaitMode === "listen" &&- !process.env.NUQ_RABBITMQ_URL- ) {- await nuqPool.query(- `SELECT pg_notify('${this.queueName}', $1);`,- [job.id + "|failed"],- );- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {- await this.sendJobEnd(- job.id,- "failed",- job.listen_channel_id,- _logger,- );- }+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [+ job.id + "|failed",+ ]);+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ await this.sendJobEnd(+ job.id,+ "failed",+ job.listen_channel_id,+ _logger,+ );}}@@ -1531,223 +883,12 @@ class NuQ<JobData = any, JobReturnValue = any> {});}- public async getOwnerJobCounts(- ownerId: string,- _logger: Logger = logger,- ): Promise<{ active: number; queued: number }> {- const start = Date.now();- try {- const result = await nuqPool.query(- `SELECT- COALESCE(SUM(CASE WHEN status = 'active'::nuq.job_status THEN 1 ELSE 0 END), 0)::int as active,- COALESCE(SUM(CASE WHEN status = 'queued'::nuq.job_status THEN 1 ELSE 0 END), 0)::int as queued- FROM ${this.queueName}- WHERE owner_id = $1;`,- [ownerId],- );-- return {- active: result.rows[0]?.active ?? 0,- queued: result.rows[0]?.queued ?? 0,- };- } finally {- _logger.info("nuqGetOwnerJobCounts metrics", {- module: "nuq/metrics",- method: "nuqGetOwnerJobCounts",- duration: Date.now() - start,- ownerId,- });- }- }-- public async getOwnerConcurrency(- ownerId: string,- _logger: Logger = logger,- ): Promise<{- currentConcurrency: number;- maxConcurrency: number;- } | null> {- const start = Date.now();- try {- const result = await nuqPool.query(- `SELECT- (SELECT COUNT(*)::int8 FROM ${this.queueName} WHERE owner_id = $1 AND status = 'active'::nuq.job_status) as current_concurrency,- max_concurrency- FROM ${this.queueName}_owner_concurrency- WHERE id = $1;`,- [ownerId],- );-- if (result.rows.length === 0) {- return null;- }-- return {- currentConcurrency: result.rows[0].current_concurrency,- maxConcurrency: result.rows[0].max_concurrency,- };- } finally {- _logger.info("nuqGetOwnerConcurrency metrics", {- module: "nuq/metrics",- method: "nuqGetOwnerConcurrency",- duration: Date.now() - start,- ownerId,- });- }- }-// === Metricspublic async getMetrics(): Promise<string> {const start = Date.now();-- let query: string;- if (this.options.concurrencyLimit === "per-owner") {- query = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*) as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- owner_status AS (- SELECT- oc.id,- GREATEST(0, oc.max_concurrency - COALESCE(oac.active_count, 0)) as available_slots- FROM ${this.queueName}_owner_concurrency oc- LEFT JOIN owner_active_counts oac ON oac.owner_id = oc.id- ),- queued_per_owner AS (- SELECT- owner_id,- COUNT(*) as queued_count- FROM ${this.queueName}- WHERE status = 'queued'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- owner_breakdown AS (- SELECT- LEAST(qpo.queued_count, COALESCE(os.available_slots, 999999)) as can_run,- GREATEST(0, qpo.queued_count - COALESCE(os.available_slots, 999999)) as blocked- FROM queued_per_owner qpo- LEFT JOIN owner_status os ON qpo.owner_id = os.id- ),- non_queued AS (- SELECT status::text, COUNT(*) as count- FROM ${this.queueName}- WHERE status != 'queued'::nuq.job_status- GROUP BY status- ),- totals AS (- SELECT- COALESCE(SUM(can_run), 0) as total_queued,- COALESCE(SUM(blocked), 0) as total_limited- FROM owner_breakdown- )- SELECT status, count FROM non_queued- UNION ALL- SELECT 'queued' as status, total_queued::bigint as count- FROM totals- WHERE total_queued > 0- UNION ALL- SELECT 'concurrency-limited' as status, total_limited::bigint as count- FROM totals- WHERE total_limited > 0- ORDER BY count DESC;- `;- } else if (this.options.concurrencyLimit === "per-owner-per-group") {- query = `- WITH owner_active_counts AS (- SELECT- owner_id,- COUNT(*) as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL- GROUP BY owner_id- ),- group_active_counts AS (- SELECT- group_id,- COUNT(*) as active_count- FROM ${this.queueName}- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL- GROUP BY group_id- ),- owner_status AS (- SELECT- oc.id,- GREATEST(0, oc.max_concurrency - COALESCE(oac.active_count, 0)) as available_slots- FROM ${this.queueName}_owner_concurrency oc- LEFT JOIN owner_active_counts oac ON oac.owner_id = oc.id- ),- group_status AS (- SELECT- gc.id,- GREATEST(0, gc.max_concurrency - COALESCE(gac.active_count, 0)) as available_slots- FROM ${this.queueName}_group_concurrency gc- LEFT JOIN group_active_counts gac ON gac.group_id = gc.id- ),- queued_per_owner_group AS (- SELECT- owner_id,- group_id,- COUNT(*) as queued_count- FROM ${this.queueName}- WHERE status = 'queued'::nuq.job_status- GROUP BY owner_id, group_id- ),- group_capacity AS (- SELECT- qpog.owner_id,- qpog.group_id,- qpog.queued_count,- LEAST(qpog.queued_count, COALESCE(gs.available_slots, 999999)) as group_can_run,- GREATEST(0, qpog.queued_count - COALESCE(gs.available_slots, 999999)) as group_blocked- FROM queued_per_owner_group qpog- LEFT JOIN group_status gs ON qpog.group_id = gs.id- ),- owner_breakdown AS (- SELECT- gc.owner_id,- SUM(gc.group_can_run) as total_demand,- COALESCE(os.available_slots, 999999) as owner_capacity,- LEAST(SUM(gc.group_can_run), COALESCE(os.available_slots, 999999)) as owner_can_run,- GREATEST(0, SUM(gc.group_can_run) - COALESCE(os.available_slots, 999999)) as owner_blocked,- SUM(gc.group_blocked) as group_blocked- FROM group_capacity gc- LEFT JOIN owner_status os ON gc.owner_id = os.id- GROUP BY gc.owner_id, os.available_slots- ),- non_queued AS (- SELECT status::text, COUNT(*) as count- FROM ${this.queueName}- WHERE status != 'queued'::nuq.job_status- GROUP BY status- ),- totals AS (- SELECT- COALESCE(SUM(owner_can_run), 0) as total_queued,- COALESCE(SUM(owner_blocked + group_blocked), 0) as total_limited- FROM owner_breakdown- )- SELECT status, count FROM non_queued- UNION ALL- SELECT 'queued' as status, total_queued::bigint as count- FROM totals- WHERE total_queued > 0- UNION ALL- SELECT 'concurrency-limited' as status, total_limited::bigint as count- FROM totals- WHERE total_limited > 0- ORDER BY count DESC;- `;- } else {- query = `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`;- }-- const result = await nuqPool.query(query);-+ const result = await nuqPool.query(+ `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,+ );logger.info("nuqGetMetrics metrics", {module: "nuq/metrics",method: "nuqGetMetrics",@@ -1782,139 +923,6 @@ class NuQ<JobData = any, JobReturnValue = any> {}}-// === Group--type NuQGroupOptions = {- memberQueues: NuQ[];- finishQueue?: NuQ;- groupTTL: number;-};--type NuQGroupConcurrencySettings = {- queue: NuQ;- maxConcurrency?: number;-};--type NuQGroupInstanceOptions = {- concurrency: NuQGroupConcurrencySettings[];- ownerId: string;-};--type NuQGroupInstance = {- id: string;- status: "active" | "completed";- createdAt: Date;- finishedAt?: Date;- expiresAt?: Date;- ownerId?: string;-};--class NuQGroup {- constructor(- public readonly groupName: string,- public readonly options: NuQGroupOptions,- ) {}-- private groupReturning = [- "id",- "status",- "created_at",- "finished_at",- "expires_at",- "owner_id",- ];-- private rowToGroup(row: any): NuQGroupInstance | null {- if (!row) return null;- return {- id: row.id,- status: row.status,- createdAt: new Date(row.created_at),- finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,- expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,- ownerId: row.owner_id ?? undefined,- };- }-- public async addGroup(- id: string,- options: NuQGroupInstanceOptions,- ): Promise<NuQGroupInstance> {- const client = await nuqPool.connect();-- await client.query("BEGIN");-- try {- const insert = await client.query(- `INSERT INTO ${this.groupName} (id, owner_id) VALUES ($1, $2) RETURNING ${this.groupReturning.join(", ")};`,- [id, normalizeOwnerId(options.ownerId) ?? null],- );-- for (const entry of options.concurrency) {- if (entry.queue.options.concurrencyLimit === "per-owner-per-group") {- await client.query(- `INSERT INTO ${entry.queue.queueName}_group_concurrency (id, max_concurrency) VALUES ($1, $2);`,- [id, entry.maxConcurrency ?? null],- );- }- }-- await client.query("COMMIT");- client.release();-- return this.rowToGroup(insert.rows[0])!;- } catch (e) {- await client.query("ROLLBACK");- client.release(e);- throw e;- }- }-- public async getGroup(id: string): Promise<NuQGroupInstance | null> {- return this.rowToGroup(- (- await nuqPool.query(- `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE id = $1 LIMIT 1;`,- [id],- )- ).rows[0],- );- }-- public async cancelGroup(id: string): Promise<boolean> {- const client = await nuqPool.connect();-- await client.query("BEGIN");-- try {- const updateOp = await client.query(- `UPDATE ${this.groupName} SET status = 'cancelled'::nuq.group_status WHERE id = $1 AND status = 'active'::nuq.group_status`,- [id],- );-- if (updateOp.rowCount === 0) {- client.release();- return false;- }-- for (const queue of this.options.memberQueues) {- await client.query(- `UPDATE ${queue.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = 'CANCELLED' WHERE group_id = $1 AND status = 'queued'::nuq.job_status`,- [id],- );- }-- client.release();- return true;- } catch (e) {- client.release(e);- throw e;- }- }-}--// === Utilities-export function nuqGetLocalMetrics(): string {return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n@@ -1936,16 +944,7 @@ export async function nuqHealthCheck(): Promise<boolean> {// === Instances-export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {- concurrencyLimit: "per-owner-per-group",-});-// export const crawlFinishQueue = new NuQ("nuq.queue_crawl_finish");--export const crawlGroup = new NuQGroup("nuq.group_crawl", {- memberQueues: [scrapeQueue],- // finishQueue: crawlFinishQueue,- groupTTL: 24 * 60 * 60,-});+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");// === Cleanupdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex f86dfef41..14b888a40 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -4,6 +4,10 @@ import http from "http";import https from "https";import { logger as _logger } from "../../lib/logger";+import {+ concurrentJobDone,+ pushConcurrencyLimitActiveJob,+} from "../../lib/concurrency-limit";import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";import { v4 as uuidv4 } from "uuid";@@ -23,13 +27,17 @@ import {StoredCrawl,} from "../../lib/crawl-redis";import { redisEvictConnection } from "../redis";-import { addScrapeJob, addScrapeJobs } from "../queue-jobs";+import {+ _addScrapeJobToBullMQ,+ addScrapeJob,+ addScrapeJobs,+} from "../queue-jobs";import psl from "psl";import { getJobPriority } from "../../lib/job-priority";import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";import { hasFormatOfType } from "../../lib/format-utils";import { getACUCTeam } from "../../controllers/auth";-import { createWebhookSender, WebhookEvent } from "../webhook/index";+import { createWebhookSender, WebhookEvent } from "../webhook";import { CustomError } from "../../lib/custom-error";import { startWebScraperPipeline } from "../../main/runWebScraper";import { CostTracking } from "../../lib/cost-tracking";@@ -330,7 +338,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {// Store robots blocked URLs in Redis setfor (const [url, reason] of links.denialReasons) {if (reason === "URL blocked by robots.txt") {- await recordRobotsBlocked(job.data.crawl_id, url);+ await recordRobotsBlocked(+ job.data.crawl_id,+ url+ );}}@@ -541,12 +552,15 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {error instanceof Error &&error.message === "URL blocked by robots.txt") {- await recordRobotsBlocked(job.data.crawl_id, job.data.url);+ await recordRobotsBlocked(+ job.data.crawl_id,+ job.data.url,+ );}} catch (e) {logger.debug("Failed to record top-level robots block", { e });}-+if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;@@ -750,7 +764,7 @@ async function addKickoffSitemapJob(}const jobId = uuidv4();- await addScrapeJob(+ await _addScrapeJobToBullMQ({mode: "kickoff_sitemap" as const,team_id: sourceJob.data.team_id,@@ -1097,55 +1111,75 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {try {- await addJobPriority(job.data.team_id, job.id);try {- if (job.data.mode === "kickoff") {- const result = await processKickoffJob(job as NuQJob<ScrapeJobKickoff>);- if (result.success) {- return null;- } else {- throw (result as any).error;- }- } else if (job.data.mode === "kickoff_sitemap") {- const result = await processKickoffSitemapJob(- job as NuQJob<ScrapeJobKickoffSitemap>,- );- if (result.success) {- return null;+ let extendLockInterval: NodeJS.Timeout | null = null;+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {+ extendLockInterval = setInterval(async () => {+ await pushConcurrencyLimitActiveJob(+ job.data.team_id,+ job.id,+ 60 * 1000,+ ); // 60s lock renew, just like in the queue+ }, jobLockExtendInterval);+ }++ await addJobPriority(job.data.team_id, job.id);+ try {+ if (job.data.mode === "kickoff") {+ const result = await processKickoffJob(+ job as NuQJob<ScrapeJobKickoff>,+ );+ if (result.success) {+ return null;+ } else {+ throw (result as any).error;+ }+ } else if (job.data.mode === "kickoff_sitemap") {+ const result = await processKickoffSitemapJob(+ job as NuQJob<ScrapeJobKickoffSitemap>,+ );+ if (result.success) {+ return null;+ } else {+ throw (result as any).error;+ }} else {- throw (result as any).error;- }- } else {- const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);- if (result.success) {- try {- if (job.data.team_id) {- await redisEvictConnection.set(- "most-recent-success:" + job.data.team_id,- new Date().toISOString(),- "EX",- 60 * 60 * 24,- );+ const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);+ if (result.success) {+ try {+ if (job.data.team_id) {+ await redisEvictConnection.set(+ "most-recent-success:" + job.data.team_id,+ new Date().toISOString(),+ "EX",+ 60 * 60 * 24,+ );+ }+ } catch (e) {+ logger.warn("Failed to set most recent success", { error: e });}- } catch (e) {- logger.warn("Failed to set most recent success", { error: e });- }- try {- if (process.env.GCS_BUCKET_NAME) {- logger.debug("Job succeeded -- putting null in Redis");- return null;- } else {- logger.debug("Job succeeded -- putting result in Redis");- return result.document;- }- } catch (e) {}- } else {- throw (result as any).error;+ try {+ if (process.env.GCS_BUCKET_NAME) {+ logger.debug("Job succeeded -- putting null in Redis");+ return null;+ } else {+ logger.debug("Job succeeded -- putting result in Redis");+ return result.document;+ }+ } catch (e) {}+ } else {+ throw (result as any).error;+ }+ }+ } finally {+ await deleteJobPriority(job.data.team_id, job.id);+ if (extendLockInterval) {+ clearInterval(extendLockInterval);}}} finally {- await deleteJobPriority(job.data.team_id, job.id);+ await concurrentJobDone(job);}} catch (error) {logger.debug("Job failed", { error });diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 1697a1f8c..f22a6b83a 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -9,12 +9,6 @@ EXCEPTIONWHEN duplicate_object THEN null;END $$;-DO $$ BEGIN- CREATE TYPE nuq.group_status AS ENUM ('active', 'completed', 'cancelled');-EXCEPTION- WHEN duplicate_object THEN null;-END $$;-CREATE TABLE IF NOT EXISTS nuq.queue_scrape (id uuid NOT NULL DEFAULT gen_random_uuid(),status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,@@ -28,9 +22,6 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (listen_channel_id text, -- for listenable jobs over rabbitmqreturnvalue jsonb, -- only for selfhostfailedreason text, -- only for selfhost- owner_id uuid,- group_id uuid,- times_out_at timestamp with time zone,CONSTRAINT queue_scrape_pkey PRIMARY KEY (id));@@ -44,146 +35,21 @@ CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrapeCREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_idx ON nuq.queue_scrape (owner_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status);-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_group_idx ON nuq.queue_scrape (owner_id, group_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status);---- Performance indexes for prefetchJobs query optimization--- This index dramatically speeds up the "SELECT DISTINCT owner_id, group_id" query in the queued_combinations CTE--- by allowing an index-only scan instead of a sequential scan of all queued jobs-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_group_distinct_idx ON nuq.queue_scrape (owner_id, group_id) WHERE (status = 'queued'::nuq.job_status);---- This index helps with group-specific queries and provides better coverage for per-group capacity lookups-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_group_priority_idx ON nuq.queue_scrape (group_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status AND group_id IS NOT NULL);---- This index helps with owner-specific queries when group_id is NULL-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_no_group_idx ON nuq.queue_scrape (owner_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status AND group_id IS NULL);--CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_complete_idx ON nuq.queue_scrape (owner_id, group_id, priority ASC, created_at ASC, id ASC) WHERE (status = 'queued'::nuq.job_status);---- Indexes for dynamic concurrency counting (replacing current_concurrency counters)--- This allows fast COUNT(*) queries for active jobs per owner/group without maintaining counters-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_owner_idx ON nuq.queue_scrape (owner_id) WHERE (status = 'active'::nuq.job_status AND owner_id IS NOT NULL);-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_group_idx ON nuq.queue_scrape (group_id) WHERE (status = 'active'::nuq.job_status AND group_id IS NOT NULL);-CREATE INDEX IF NOT EXISTS idx_queue_scrape_prefetch_composite ON queue_scrape (status, owner_id, group_id, priority, created_at, id) WHERE status = 'queued';--CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (- id uuid NOT NULL,- max_concurrency int8 NOT NULL,- CONSTRAINT queue_scrape_owner_concurrency_pkey PRIMARY KEY (id)-);---- fake concurrency limit source for tests-CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency_source (- id uuid NOT NULL,- max_concurrency int8 NOT NULL,- CONSTRAINT queue_scrape_owner_concurrency_source_pkey PRIMARY KEY (id)-);--CREATE OR REPLACE FUNCTION nuq_queue_scrape_owner_resolve_max_concurrency(owner_id uuid)-RETURNS int8-LANGUAGE sql-STABLE-AS $$- SELECT COALESCE((SELECT max_concurrency FROM nuq.queue_scrape_owner_concurrency_source WHERE id = owner_id LIMIT 1), 100)::int8;-$$;--CREATE TABLE IF NOT EXISTS nuq.queue_scrape_group_concurrency (- id uuid NOT NULL,- max_concurrency int8,- CONSTRAINT queue_scrape_group_concurrency_pkey PRIMARY KEY (id)-);SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.group_id IS NULL AND nuq.queue_scrape.created_at < now() - interval '1 hour';+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';$$);SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.group_id IS NULL AND nuq.queue_scrape.created_at < now() - interval '6 hours';+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';$$);SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$- WITH stalled_jobs AS (- SELECT id, owner_id, group_id, stalls- FROM nuq.queue_scrape- WHERE locked_at <= now() - interval '1 minute'- AND status = 'active'::nuq.job_status- ),- requeued AS (- UPDATE nuq.queue_scrape- SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1- WHERE id IN (- SELECT sj.id- FROM stalled_jobs sj- WHERE COALESCE(sj.stalls, 0) < 9- )- RETURNING id- ),- stallfail AS (- UPDATE nuq.queue_scrape- SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1- WHERE id IN (- SELECT sj.id- FROM stalled_jobs sj- WHERE COALESCE(sj.stalls, 0) >= 9- )- RETURNING id- )+ UPDATE nuq.queue_scrape SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) < 9;+ WITH stallfail AS (UPDATE nuq.queue_scrape SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9 RETURNING id)SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;$$);SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$REINDEX TABLE CONCURRENTLY nuq.queue_scrape;$$);--SELECT cron.schedule('nuq_queue_scrape_max_concurrency_sync', '*/5 * * * *', $$- UPDATE nuq.queue_scrape_owner_concurrency- SET max_concurrency = (SELECT nuq_queue_scrape_owner_resolve_max_concurrency(nuq.queue_scrape_owner_concurrency.id));-$$);--SELECT cron.schedule('nuq_queue_scrape_timeout', '* * * * *', $$- UPDATE nuq.queue_scrape- SET status = 'failed', finished_at = now(), failedreason = 'SCRAPE_TIMEOUT|{"stack":"Error: Scrape timed out\n in DB","message":"Scrape timed out"}'- WHERE status = 'queued' AND times_out_at < now();-$$);--CREATE TABLE IF NOT EXISTS nuq.group_crawl (- id uuid NOT NULL DEFAULT gen_random_uuid(),- status nuq.group_status NOT NULL DEFAULT 'active'::nuq.group_status,- created_at timestamp with time zone NOT NULL DEFAULT now(),- finished_at timestamp with time zone,- expires_at timestamp with time zone,- owner_id uuid,- CONSTRAINT group_crawl_pkey PRIMARY KEY (id)-);--CREATE OR REPLACE FUNCTION nuq_queue_scrape_check_group_completion()-RETURNS TRIGGER-LANGUAGE plpgsql-AS $$-BEGIN- IF NEW.group_id IS NOT NULL THEN- UPDATE nuq.group_crawl- SET status = 'completed'::nuq.group_status,- finished_at = now(),- expires_at = now() + interval '24 hours'- WHERE id = NEW.group_id- AND status != 'completed'::nuq.group_status- AND NOT EXISTS (- SELECT 1- FROM nuq.queue_scrape- WHERE group_id = NEW.group_id- AND status NOT IN ('completed'::nuq.job_status, 'failed'::nuq.job_status)- );- END IF;-- RETURN NEW;-END;-$$;---- Trigger to automatically mark groups as completed-CREATE OR REPLACE TRIGGER nuq_queue_scrape_group_completion_trigger-AFTER UPDATE OF status ON nuq.queue_scrape-FOR EACH ROW-WHEN (NEW.status IN ('completed'::nuq.job_status, 'failed'::nuq.job_status)- AND OLD.status NOT IN ('completed'::nuq.job_status, 'failed'::nuq.job_status))-EXECUTE FUNCTION nuq_queue_scrape_check_group_completion();