Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq): revert to pre-concurrency limit tracking

Gergő Móricz186335ebb7d0
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index ecba85ef9..959141572 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -12,7 +12,7 @@
/apps/api/src/controllers/v2/crawl* @mogery
### /batch/scrape
-/apps/api/src/controllers/v2/batch-scrape* @mogery
+/apps/api/src/controllers/v2/batch-scrape* @mogery
### /extract
/apps/api/src/controllers/v2/extract* @nickscamara
@@ -36,7 +36,7 @@
/apps/api/src/controllers/v1/crawl* @mogery
### /batch/scrape
-/apps/api/src/controllers/v1/batch-scrape* @mogery
+/apps/api/src/controllers/v1/batch-scrape* @mogery
### /extract
/apps/api/src/controllers/v1/extract* @nickscamara
@@ -77,6 +77,9 @@
### remnants of WebScraper/WebCrawler
/apps/api/src/scraper/WebScraper/* @mogery @nickscamara
+## concurrency limits
+/apps/api/src/lib/concurrency-limit.ts @mogery @nickscamara
+
## BullMQ-related code
/apps/api/src/services/queue-worker.ts @mogery @nickscamara
/apps/api/src/services/worker/scrape-worker.ts @mogery
diff --git a/apps/api/package.json b/apps/api/package.json
index 404e738e2..173187409 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -71,6 +71,7 @@
"@ai-sdk/fireworks": "^0.2.4",
"@ai-sdk/google": "^1.2.3",
"@ai-sdk/google-vertex": "^2.2.15",
+ "@pinecone-database/pinecone": "^6.1.2",
"@ai-sdk/groq": "^1.2.1",
"@ai-sdk/openai": "^1.3.12",
"@apidevtools/json-schema-ref-parser": "^11.7.3",
@@ -136,7 +137,7 @@
"tough-cookie": "^4.1.4",
"turndown": "^7.1.3",
"undici": "^7.10.0",
- "uuid": "^13.0.0",
+ "uuid": "^10.0.0",
"winston": "^3.14.2",
"ws": "^8.18.0",
"x402-express": "^0.6.5",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 54b9eebfd..43bf374bf 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -76,6 +76,9 @@ importers:
'@opentelemetry/sdk-node':
specifier: ^0.205.0
version: 0.205.0(@opentelemetry/[email protected])
+ '@pinecone-database/pinecone':
+ specifier: ^6.1.2
+ version: 6.1.2
'@sentry/cli':
specifier: ^2.33.1
version: 2.33.1([email protected])
@@ -224,8 +227,8 @@ importers:
specifier: ^7.10.0
version: 7.10.0
uuid:
- specifier: ^13.0.0
- version: 13.0.0
+ specifier: ^10.0.0
+ version: 10.0.0
winston:
specifier: ^3.14.2
version: 3.14.2
@@ -2623,6 +2626,10 @@ packages:
resolution: {integrity: sha512-IHnV6A+zxU7XwmKFinmYjUcwlyK9+xkG3/s9KcQhI9BjQKycrJ1JRO+FbNYPwZiPKW3je/DR0k7w8/gLa5eaxQ==}
deprecated: 'The package is now available as "qr": npm install qr'
+ '@pinecone-database/[email protected]':
+ resolution: {integrity: sha512-ydIlbtgIIHFgBL08sPzua5ckmOgtjgDz8xg21CnP1fqnnEgDmOlnfd10MRKU+fvFRhDlh4Md37SwZDr0d4cBqg==}
+ engines: {node: '>=18.0.0'}
+
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -6744,8 +6751,8 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
- resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==}
+ resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}
hasBin: true
@@ -7530,7 +7537,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7604,7 +7611,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7807,11 +7814,11 @@ snapshots:
ethereum-cryptography: 2.2.1
micro-ftch: 0.3.1
dependencies:
'@metamask/rpc-errors': 7.0.2
eventemitter3: 5.0.1
transitivePeerDependencies:
- supports-color
@@ -9795,6 +9802,8 @@ snapshots:
'@paulmillr/[email protected]': {}
+ '@pinecone-database/[email protected]': {}
+
optional: true
@@ -9851,7 +9860,7 @@ snapshots:
dependencies:
big.js: 6.2.2
dayjs: 1.11.13
transitivePeerDependencies:
- bufferutil
- typescript
@@ -9864,7 +9873,7 @@ snapshots:
'@reown/appkit-wallet': 1.7.8([email protected])([email protected])([email protected])
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10010,7 +10019,7 @@ snapshots:
'@walletconnect/logger': 2.1.2
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10063,7 +10072,7 @@ snapshots:
bs58: 6.0.0
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10104,7 +10113,7 @@ snapshots:
dependencies:
'@safe-global/safe-gateway-typescript-sdk': 3.23.1
transitivePeerDependencies:
- bufferutil
- typescript
@@ -11063,19 +11072,19 @@ snapshots:
dependencies:
'@types/yargs-parser': 21.0.3
dependencies:
'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])
cbw-sdk: '@coinbase/[email protected]'
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -11109,11 +11118,11 @@ snapshots:
- wagmi
- zod
dependencies:
eventemitter3: 5.0.1
mipd: 0.0.7([email protected])
optionalDependencies:
'@tanstack/query-core': 5.83.1
@@ -11661,6 +11670,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -11671,6 +11685,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -14142,6 +14161,21 @@ snapshots:
transitivePeerDependencies:
- zod
+ dependencies:
+ '@adraffy/ens-normalize': 1.11.1
+ '@noble/ciphers': 1.3.0
+ '@noble/curves': 1.9.6
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ eventemitter3: 5.0.1
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - zod
+
dependencies:
'@adraffy/ens-normalize': 1.11.1
@@ -14381,21 +14415,21 @@ snapshots:
dependencies:
hono: 4.9.10
idb-keyval: 6.2.2
mipd: 0.0.7([email protected])
zod: 4.1.11
optionalDependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
typescript: 5.8.3
transitivePeerDependencies:
- '@types/react'
- immer
@@ -15267,7 +15301,7 @@ snapshots:
@@ -15325,6 +15359,23 @@ snapshots:
- utf-8-validate
- zod
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+ - zod
+
dependencies:
'@noble/curves': 1.9.2
@@ -15346,14 +15397,14 @@ snapshots:
dependencies:
xml-name-validator: 5.0.0
dependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
use-sync-external-store: 1.4.0([email protected])
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -15560,8 +15611,8 @@ snapshots:
zod: 3.25.76
transitivePeerDependencies:
- '@azure/app-configuration'
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 19d096fed..ba9fec277 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -2,7 +2,6 @@ import { configDotenv } from "dotenv";
configDotenv();
import { TeamFlags } from "../../controllers/v1/types";
-import { Client as PgClient } from "pg";
// =========================================
// Configuration
@@ -44,14 +43,12 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {
runNumber = 0;
}
- const concurrency = req.concurrency ?? 100;
-
const res = await fetch(process.env.IDMUX_URL + "/", {
method: "POST",
body: JSON.stringify({
refName: process.env.GITHUB_REF_NAME!,
runNumber,
- concurrency,
+ concurrency: req.concurrency ?? 100,
...req,
}),
headers: {
@@ -64,24 +61,7 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {
}
expect(res.ok).toBe(true);
- const result: Identity = await res.json();
-
- const nuqClient = new PgClient({
- connectionString: process.env.NUQ_DATABASE_URL,
- });
-
- // update the concurrency limit for the team in the nuq spoof table
- await nuqClient.query(
- `
- INSERT INTO nuq.queue_scrape_owner_concurrency_source (id, max_concurrency) VALUES ($1, $2)
- ON CONFLICT (id) DO UPDATE SET max_concurrency = EXCLUDED.max_concurrency
- `,
- [result.teamId, concurrency],
- );
-
- await nuqClient.end();
-
- return result;
+ return await res.json();
}
export type Identity = {
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 26172d214..598a85682 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -112,7 +112,7 @@ const mockPreviewACUC: (
const mockACUC: () => AuthCreditUsageChunk = () => ({
api_key: "bypass",
api_key_id: 0,
- team_id: "00000000-0000-0000-0000-000000000000",
+ team_id: "bypass",
sub_id: "bypass",
sub_current_period_start: new Date().toISOString(),
sub_current_period_end: new Date(
@@ -396,7 +396,7 @@ export async function authenticateUser(
return withAuth(supaAuthenticateUser, {
success: true,
chunk: null,
- team_id: "00000000-0000-0000-0000-000000000000",
+ team_id: "bypass",
})(req, res, mode);
}
diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.ts
new file mode 100644
index 000000000..76d2c415d
--- /dev/null
+++ b/apps/api/src/controllers/v0/admin/cclog.ts
@@ -0,0 +1,67 @@
+import { redisEvictConnection } from "../../../services/redis";
+import { supabase_service } from "../../../services/supabase";
+import { logger as _logger } from "../../../lib/logger";
+import { Request, Response } from "express";
+
+async function cclog() {
+ const logger = _logger.child({
+ module: "cclog",
+ });
+
+ let cursor = 0;
+ do {
+ const result = await redisEvictConnection.scan(
+ cursor,
+ "MATCH",
+ "concurrency-limiter:*",
+ "COUNT",
+ 100000,
+ );
+ cursor = parseInt(result[0], 10);
+ const usable = result[1].filter(x => !x.includes("preview_"));
+
+ logger.info("Stepped", { cursor, usable: usable.length });
+
+ if (usable.length > 0) {
+ const entries: {
+ team_id: string;
+ concurrency: number;
+ created_at: Date;
+ }[] = [];
+
+ for (const x of usable) {
+ const at = new Date();
+ const concurrency = await redisEvictConnection.zrangebyscore(
+ x,
+ Date.now(),
+ Infinity,
+ );
+ if (concurrency) {
+ entries.push({
+ team_id: x.split(":")[1],
+ concurrency: concurrency.length,
+ created_at: at,
+ });
+ }
+ }
+
+ try {
+ await supabase_service.from("concurrency_log").insert(entries);
+ } catch (e) {
+ logger.error("Error inserting", { error: e });
+ }
+ }
+ } while (cursor != 0);
+}
+
+export async function cclogController(req: Request, res: Response) {
+ try {
+ await cclog();
+ res.status(200).json({ ok: true });
+ } catch (e) {
+ _logger.error("Error", { module: "cclog", error: e });
+ res.status(500).json({
+ message: "Error",
+ });
+ }
+}
diff --git a/apps/api/src/controllers/v0/admin/crawl-check.ts b/apps/api/src/controllers/v0/admin/crawl-check.ts
index b462a1e7d..75b42c5e4 100644
--- a/apps/api/src/controllers/v0/admin/crawl-check.ts
+++ b/apps/api/src/controllers/v0/admin/crawl-check.ts
@@ -6,6 +6,7 @@ import {
StoredCrawl,
} from "../../../lib/crawl-redis";
import { supabase_service } from "../../../services/supabase";
+import { getConcurrencyLimitedJobs } from "../../../lib/concurrency-limit";
import { scrapeQueue } from "../../../services/worker/nuq";
type AnalyzedCrawlPass1 = {
@@ -216,8 +217,23 @@ export async function crawlCheckController(req: Request, res: Response) {
await Promise.all(activeCrawls.map(analyzeCrawlPass1))
).filter(result => result.success);
+ const teamIds = [...new Set(firstPass.map(result => result.teamId))];
+ const teamConcurrencyQueuedJobs = Object.fromEntries(
+ await Promise.all(
+ teamIds.map(async teamId => [
+ teamId,
+ await getConcurrencyLimitedJobs(teamId),
+ ]),
+ ),
+ );
+
const results = await Promise.all(
- firstPass.map(result => analyzeCrawlPass2({ ...result }, new Set())),
+ firstPass.map(result =>
+ analyzeCrawlPass2(
+ { ...result },
+ teamConcurrencyQueuedJobs[result.teamId],
+ ),
+ ),
);
// Clean results which should be "solved"
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index e9e1e80ec..59c38eb89 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -33,7 +33,6 @@ import { ZodError } from "zod";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { fromV0ScrapeOptions } from "../v2/types";
import { isSelfHosted } from "../../lib/deployment";
-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";
export async function crawlController(req: Request, res: Response) {
try {
@@ -147,6 +146,35 @@ export async function crawlController(req: Request, res: Response) {
});
}
+ // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
+ // try {
+ // const a = new WebScraperDataProvider();
+ // await a.setOptions({
+ // jobId: uuidv4(),
+ // mode: "single_urls",
+ // urls: [url],
+ // crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },
+ // pageOptions: pageOptions,
+ // });
+
+ // const docs = await a.getDocuments(false, (progress) => {
+ // job.updateProgress({
+ // current: progress.current,
+ // total: progress.total,
+ // current_step: "SCRAPING",
+ // current_url: progress.currentDocumentUrl,
+ // });
+ // });
+ // return res.json({
+ // success: true,
+ // documents: docs,
+ // });
+ // } catch (error) {
+ // logger.error(error);
+ // return res.status(500).json({ error: error.message });
+ // }
+ // }
+
const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(
pageOptions,
undefined,
@@ -175,16 +203,6 @@ export async function crawlController(req: Request, res: Response) {
sc.robots = await crawler.getRobotsTxt();
} catch (_) {}
- await crawlGroup.addGroup(id, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: undefined,
- },
- ],
- ownerId: sc.team_id,
- });
-
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index ed56d5397..c12ff7a9f 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -25,7 +25,6 @@ import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -137,15 +136,6 @@ export async function batchScrapeController(
};
if (!req.body.appendToId) {
- await crawlGroup.addGroup(id, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: sc.maxConcurrency,
- },
- ],
- ownerId: sc.team_id,
- });
await saveCrawl(id, sc);
await markCrawlActive(id);
}
diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.ts
index 26b11f74e..517aed515 100644
--- a/apps/api/src/controllers/v1/concurrency-check.ts
+++ b/apps/api/src/controllers/v1/concurrency-check.ts
@@ -1,59 +1,27 @@
import {
- AuthCreditUsageChunkFromTeam,
ConcurrencyCheckParams,
ConcurrencyCheckResponse,
RequestWithAuth,
} from "./types";
import { Response } from "express";
-import { scrapeQueue } from "../../services/worker/nuq";
-import { getACUCTeam } from "../auth";
-import { RateLimiterMode } from "../../types";
+import { redisEvictConnection } from "../../../src/services/redis";
// Basically just middleware and error wrapping
export async function concurrencyCheckController(
req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,
res: Response<ConcurrencyCheckResponse>,
) {
- if (!req.acuc) {
- return res.status(401).json({
- success: false,
- error: "Unauthorized",
- });
- }
-
- const ownerConcurrency = await scrapeQueue.getOwnerConcurrency(
- req.auth.team_id,
+ const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
+ const now = Date.now();
+ const activeJobsOfTeam = await redisEvictConnection.zrangebyscore(
+ concurrencyLimiterKey,
+ now,
+ Infinity,
);
- let maxConcurrency: number | null = ownerConcurrency?.maxConcurrency ?? null;
-
- if (maxConcurrency === null) {
- let otherACUC: AuthCreditUsageChunkFromTeam | null = null;
- if (!req.acuc.is_extract) {
- otherACUC = await getACUCTeam(
- req.auth.team_id,
- false,
- true,
- RateLimiterMode.Extract,
- );
- } else {
- otherACUC = await getACUCTeam(
- req.auth.team_id,
- false,
- true,
- RateLimiterMode.Crawl,
- );
- }
-
- maxConcurrency = Math.max(
- req.acuc.concurrency,
- otherACUC?.concurrency ?? 0,
- );
- }
-
return res.status(200).json({
success: true,
- concurrency: ownerConcurrency?.currentConcurrency ?? 0,
- maxConcurrency,
+ concurrency: activeJobsOfTeam.length,
+ maxConcurrency: req.acuc?.concurrency ?? 0,
});
}
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index d96d5c1b9..6c4c6155b 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -18,6 +18,7 @@ import {
} from "../../lib/crawl-redis";
import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
+import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
import { getErrorContactMessage } from "../../lib/deployment";
@@ -110,9 +111,10 @@ async function crawlStatusWS(
setTimeout(loop, 1000);
- let [_doneJobIDs, jobIDs] = await Promise.all([
+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([
getDoneJobsOrdered(req.params.jobId),
getCrawlJobs(req.params.jobId),
+ getConcurrencyLimitedJobs(req.auth.team_id),
]);
doneJobIDs = _doneJobIDs;
@@ -122,10 +124,15 @@ async function crawlStatusWS(
const validJobIDs: string[] = [];
for (const id of jobIDs) {
- const job = jobs.get(id);
- if (job && job.status !== "failed") {
- validJobStatuses.push([id, job.status]);
+ if (throttledJobsSet.has(id)) {
+ validJobStatuses.push([id, "queued"]);
validJobIDs.push(id);
+ } else {
+ const job = jobs.get(id);
+ if (job && job.status !== "failed") {
+ validJobStatuses.push([id, job.status]);
+ validJobIDs.push(id);
+ }
}
}
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index cba3c6342..673d9d315 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -387,7 +387,7 @@ export async function crawlStatusController(
data: scrapes,
next:
(outputBulkA.total ?? 0) > start + iteratedOver
- ? `${req.protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
} else {
@@ -445,7 +445,7 @@ export async function crawlStatusController(
data: scrapes,
next:
(outputBulkA.total ?? 0) > start + iteratedOver
- ? `${req.protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
}
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 669c0908f..e83489826 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -13,11 +13,10 @@ import {
StoredCrawl,
markCrawlActive,
} from "../../lib/crawl-redis";
-import { addScrapeJob } from "../../services/queue-jobs";
+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -136,21 +135,11 @@ export async function crawlController(
});
}
- await crawlGroup.addGroup(id, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: sc.maxConcurrency,
- },
- ],
- ownerId: sc.team_id,
- });
-
await saveCrawl(id, sc);
await markCrawlActive(id);
- await addScrapeJob(
+ await _addScrapeJobToBullMQ(
{
mode: "kickoff" as const,
url: req.body.url,
diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.ts
index 46c46107d..63e0157fc 100644
--- a/apps/api/src/controllers/v1/queue-status.ts
+++ b/apps/api/src/controllers/v1/queue-status.ts
@@ -3,7 +3,12 @@ import { getACUCTeam } from "../auth";
import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";
import { Response } from "express";
import { redisEvictConnection } from "../../services/redis";
-import { scrapeQueue } from "../../services/worker/nuq";
+import {
+ cleanOldConcurrencyLimitedJobs,
+ cleanOldConcurrencyLimitEntries,
+ getConcurrencyLimitActiveJobsCount,
+ getConcurrencyQueueJobsCount,
+} from "../../lib/concurrency-limit";
type QueueStatusResponse = {
success: boolean;
@@ -35,7 +40,12 @@ export async function queueStatusController(
);
}
- const jobCounts = await scrapeQueue.getOwnerJobCounts(req.auth.team_id);
+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);
+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
+ req.auth.team_id,
+ );
+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
const mostRecentSuccess = await redisEvictConnection.get(
"most-recent-success:" + req.auth.team_id,
@@ -44,9 +54,9 @@ export async function queueStatusController(
return res.status(200).json({
success: true,
- jobsInQueue: jobCounts.active + jobCounts.queued,
- activeJobsInQueue: jobCounts.active,
- waitingJobsInQueue: jobCounts.queued,
+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,
+ activeJobsInQueue: activeJobsOfTeam,
+ waitingJobsInQueue: queuedJobsOfTeam,
maxConcurrency: Math.max(
req.acuc?.concurrency ?? 1,
otherACUC?.concurrency ?? 1,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index d0b362c21..04a478bb1 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -25,7 +25,6 @@ import { logger as _logger } from "../../lib/logger";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -130,15 +129,6 @@ export async function batchScrapeController(
};
if (!req.body.appendToId) {
- await crawlGroup.addGroup(id, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: sc.maxConcurrency,
- },
- ],
- ownerId: sc.team_id,
- });
await saveCrawl(id, sc);
await markCrawlActive(id);
}
diff --git a/apps/api/src/controllers/v2/concurrency-check.ts b/apps/api/src/controllers/v2/concurrency-check.ts
index 04b6980ec..17edca1c4 100644
--- a/apps/api/src/controllers/v2/concurrency-check.ts
+++ b/apps/api/src/controllers/v2/concurrency-check.ts
@@ -5,9 +5,9 @@ import {
} from "./types";
import { AuthCreditUsageChunkFromTeam } from "../v1/types";
import { Response } from "express";
+import { redisEvictConnection } from "../../../src/services/redis";
import { getACUCTeam } from "../auth";
import { RateLimiterMode } from "../../types";
-import { scrapeQueue } from "../../services/worker/nuq";
// Basically just middleware and error wrapping
export async function concurrencyCheckController(
@@ -21,39 +21,34 @@ export async function concurrencyCheckController(
});
}
- const ownerConcurrency = await scrapeQueue.getOwnerConcurrency(
- req.auth.team_id,
- );
-
- let maxConcurrency: number | null = ownerConcurrency?.maxConcurrency ?? null;
-
- if (maxConcurrency === null) {
- let otherACUC: AuthCreditUsageChunkFromTeam | null = null;
- if (!req.acuc.is_extract) {
- otherACUC = await getACUCTeam(
- req.auth.team_id,
- false,
- true,
- RateLimiterMode.Extract,
- );
- } else {
- otherACUC = await getACUCTeam(
- req.auth.team_id,
- false,
- true,
- RateLimiterMode.Crawl,
- );
- }
-
- maxConcurrency = Math.max(
- req.acuc.concurrency,
- otherACUC?.concurrency ?? 0,
+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;
+ if (!req.acuc.is_extract) {
+ otherACUC = await getACUCTeam(
+ req.auth.team_id,
+ false,
+ true,
+ RateLimiterMode.Extract,
+ );
+ } else {
+ otherACUC = await getACUCTeam(
+ req.auth.team_id,
+ false,
+ true,
+ RateLimiterMode.Crawl,
);
}
+ const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
+ const now = Date.now();
+ const activeJobsOfTeam = await redisEvictConnection.zrangebyscore(
+ concurrencyLimiterKey,
+ now,
+ Infinity,
+ );
+
return res.status(200).json({
success: true,
- concurrency: ownerConcurrency?.currentConcurrency ?? 0,
- maxConcurrency,
+ concurrency: activeJobsOfTeam.length,
+ maxConcurrency: Math.max(req.acuc.concurrency, otherACUC?.concurrency ?? 0),
});
}
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index df5bfe5df..91c1dd809 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -18,6 +18,7 @@ import {
} from "../../lib/crawl-redis";
import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
+import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
import { getErrorContactMessage } from "../../lib/deployment";
@@ -113,9 +114,10 @@ async function crawlStatusWS(
setTimeout(loop, 1000);
- let [_doneJobIDs, jobIDs] = await Promise.all([
+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([
getDoneJobsOrdered(req.params.jobId),
getCrawlJobs(req.params.jobId),
+ getConcurrencyLimitedJobs(req.auth.team_id),
]);
doneJobIDs = _doneJobIDs;
@@ -125,10 +127,15 @@ async function crawlStatusWS(
const validJobIDs: string[] = [];
for (const id of jobIDs) {
- const job = jobs.get(id);
- if (job && job.status !== "failed") {
- validJobStatuses.push([id, job.status]);
+ if (throttledJobsSet.has(id)) {
+ validJobStatuses.push([id, "queued"]);
validJobIDs.push(id);
+ } else {
+ const job = jobs.get(id);
+ if (job && job.status !== "failed") {
+ validJobStatuses.push([id, job.status]);
+ validJobIDs.push(id);
+ }
}
}
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index e8ccd6f83..16e6d845c 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -406,7 +406,7 @@ export async function crawlStatusController(
data: scrapes,
next:
(outputBulkA.total ?? 0) > start + iteratedOver
- ? `${req.protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
} else {
@@ -464,7 +464,7 @@ export async function crawlStatusController(
data: scrapes,
next:
(outputBulkA.total ?? 0) > start + iteratedOver
- ? `${req.protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
}
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 3825cc706..2a74b00fa 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -13,14 +13,13 @@ import {
StoredCrawl,
markCrawlActive,
} from "../../lib/crawl-redis";
-import { addScrapeJob } from "../../services/queue-jobs";
+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
import { modifyCrawlUrl } from "../../utils/url-utils";
-import { crawlGroup, scrapeQueue } from "../../services/worker/nuq";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -203,20 +202,11 @@ export async function crawlController(
});
}
- await crawlGroup.addGroup(id, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: sc.maxConcurrency,
- },
- ],
- ownerId: sc.team_id,
- });
await saveCrawl(id, sc);
await markCrawlActive(id);
- await addScrapeJob(
+ await _addScrapeJobToBullMQ(
{
url: req.body.url,
mode: "kickoff" as const,
diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.ts
index 1c11851f4..e52542dce 100644
--- a/apps/api/src/controllers/v2/queue-status.ts
+++ b/apps/api/src/controllers/v2/queue-status.ts
@@ -4,7 +4,12 @@ import { RequestWithAuth } from "./types";
import { AuthCreditUsageChunkFromTeam } from "../v1/types";
import { Response } from "express";
import { redisEvictConnection } from "../../services/redis";
-import { scrapeQueue } from "../../services/worker/nuq";
+import {
+ cleanOldConcurrencyLimitedJobs,
+ cleanOldConcurrencyLimitEntries,
+ getConcurrencyLimitActiveJobsCount,
+ getConcurrencyQueueJobsCount,
+} from "../../lib/concurrency-limit";
type QueueStatusResponse = {
success: boolean;
@@ -36,7 +41,12 @@ export async function queueStatusController(
);
}
- const jobCounts = await scrapeQueue.getOwnerJobCounts(req.auth.team_id);
+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);
+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
+ req.auth.team_id,
+ );
+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
const mostRecentSuccess = await redisEvictConnection.get(
"most-recent-success:" + req.auth.team_id,
@@ -45,9 +55,9 @@ export async function queueStatusController(
return res.status(200).json({
success: true,
- jobsInQueue: jobCounts.active + jobCounts.queued,
- activeJobsInQueue: jobCounts.active,
- waitingJobsInQueue: jobCounts.queued,
+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,
+ activeJobsInQueue: activeJobsOfTeam,
+ waitingJobsInQueue: queuedJobsOfTeam,
maxConcurrency: Math.max(
req.acuc?.concurrency ?? 1,
otherACUC?.concurrency ?? 1,
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
new file mode 100644
index 000000000..f09b34c1a
--- /dev/null
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -0,0 +1,355 @@
+import { RateLimiterMode } from "../types";
+import { redisEvictConnection } from "../services/redis";
+import { getACUCTeam } from "../controllers/auth";
+import { getCrawl, StoredCrawl } from "./crawl-redis";
+import { logger } from "./logger";
+import { abTestJob } from "../services/ab-test";
+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
+
+const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
+const constructQueueKey = (team_id: string) =>
+ "concurrency-limit-queue:" + team_id;
+
+const constructCrawlKey = (crawl_id: string) =>
+ "crawl-concurrency-limiter:" + crawl_id;
+
+export async function cleanOldConcurrencyLimitEntries(
+ team_id: string,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zremrangebyscore(
+ constructKey(team_id),
+ -Infinity,
+ now,
+ );
+}
+
+export async function getConcurrencyLimitActiveJobsCount(
+ team_id: string,
+): Promise<number> {
+ return await redisEvictConnection.zcount(
+ constructKey(team_id),
+ Date.now(),
+ Infinity,
+ );
+}
+
+export async function getConcurrencyLimitActiveJobs(
+ team_id: string,
+ now: number = Date.now(),
+): Promise<string[]> {
+ return await redisEvictConnection.zrangebyscore(
+ constructKey(team_id),
+ now,
+ Infinity,
+ );
+}
+
+export async function pushConcurrencyLimitActiveJob(
+ team_id: string,
+ id: string,
+ timeout: number,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zadd(constructKey(team_id), now + timeout, id);
+}
+
+async function removeConcurrencyLimitActiveJob(team_id: string, id: string) {
+ await redisEvictConnection.zrem(constructKey(team_id), id);
+}
+
+type ConcurrencyLimitedJob = {
+ id: string;
+ data: any;
+ priority: number;
+ listenable: boolean;
+};
+
+export async function cleanOldConcurrencyLimitedJobs(
+ team_id: string,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zremrangebyscore(
+ constructQueueKey(team_id),
+ -Infinity,
+ now,
+ );
+}
+
+export async function pushConcurrencyLimitedJob(
+ team_id: string,
+ job: ConcurrencyLimitedJob,
+ timeout: number,
+ now: number = Date.now(),
+) {
+ const queueKey = constructQueueKey(team_id);
+ await redisEvictConnection.zadd(queueKey, now + timeout, JSON.stringify(job));
+ await redisEvictConnection.sadd("concurrency-limit-queues", queueKey);
+}
+
+export async function getConcurrencyLimitedJobs(team_id: string) {
+ return new Set(
+ (await redisEvictConnection.zrange(constructQueueKey(team_id), 0, -1)).map(
+ x => JSON.parse(x).id,
+ ),
+ );
+}
+
+export async function getConcurrencyQueueJobsCount(
+ team_id: string,
+): Promise<number> {
+ return await redisEvictConnection.zcount(
+ constructQueueKey(team_id),
+ Date.now(),
+ Infinity,
+ );
+}
+
+async function cleanOldCrawlConcurrencyLimitEntries(
+ crawl_id: string,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zremrangebyscore(
+ constructCrawlKey(crawl_id),
+ -Infinity,
+ now,
+ );
+}
+
+export async function getCrawlConcurrencyLimitActiveJobs(
+ crawl_id: string,
+ now: number = Date.now(),
+): Promise<string[]> {
+ return await redisEvictConnection.zrangebyscore(
+ constructCrawlKey(crawl_id),
+ now,
+ Infinity,
+ );
+}
+
+export async function pushCrawlConcurrencyLimitActiveJob(
+ crawl_id: string,
+ id: string,
+ timeout: number,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zadd(
+ constructCrawlKey(crawl_id),
+ now + timeout,
+ id,
+ );
+}
+
+async function removeCrawlConcurrencyLimitActiveJob(
+ crawl_id: string,
+ id: string,
+) {
+ await redisEvictConnection.zrem(constructCrawlKey(crawl_id), id);
+}
+
+/**
+ * Grabs the next job from the team's concurrency limit queue. Handles crawl concurrency limits.
+ *
+ * This function may only be called once the outer code has verified that the team has not reached its concurrency limit.
+ *
+ * @param teamId
+ * @returns A job that can be run, or null if there are no more jobs to run.
+ */
+async function getNextConcurrentJob(
+ teamId: string,
+ i = 0,
+): Promise<{
+ job: ConcurrencyLimitedJob;
+ timeout: number;
+} | null> {
+ let finalJobs: {
+ job: ConcurrencyLimitedJob;
+ _member: string;
+ timeout: number;
+ }[] = [];
+
+ const crawlCache = new Map<string, StoredCrawl>();
+ let cursor: string = "0";
+
+ do {
+ const scanResult = await redisEvictConnection.zscan(
+ constructQueueKey(teamId),
+ cursor,
+ "COUNT",
+ 20,
+ );
+ cursor = scanResult[0];
+ const results = scanResult[1];
+
+ for (let i = 0; i < results.length; i += 2) {
+ const res = {
+ job: JSON.parse(results[i]),
+ _member: results[i],
+ timeout:
+ results[i + 1] === "inf" ? Infinity : parseFloat(results[i + 1]),
+ };
+
+ // If the job is associated with a crawl ID, we need to check if the crawl has a max concurrency limit
+ if (res.job.data.crawl_id) {
+ const sc =
+ crawlCache.get(res.job.data.crawl_id) ??
+ (await getCrawl(res.job.data.crawl_id));
+ if (sc !== null) {
+ crawlCache.set(res.job.data.crawl_id, sc);
+ }
+
+ const maxCrawlConcurrency =
+ sc === null
+ ? null
+ : typeof sc.crawlerOptions?.delay === "number" &&
+ sc.crawlerOptions.delay > 0
+ ? 1
+ : (sc.maxConcurrency ?? null);
+
+ if (maxCrawlConcurrency !== null) {
+ // If the crawl has a max concurrency limit, we need to check if the crawl has reached the limit
+ const currentActiveConcurrency = (
+ await getCrawlConcurrencyLimitActiveJobs(res.job.data.crawl_id)
+ ).length;
+ if (currentActiveConcurrency < maxCrawlConcurrency) {
+ // If we're under the max concurrency limit, we can run the job
+ finalJobs.push(res);
+ }
+ } else {
+ // If the crawl has no max concurrency limit, we can run the job
+ finalJobs.push(res);
+ }
+ } else {
+ // If the job is not associated with a crawl ID, we can run the job
+ finalJobs.push(res);
+ }
+ }
+ } while (finalJobs.length === 0 && cursor !== "0");
+
+ let finalJob: (typeof finalJobs)[number] | null = null;
+ if (finalJobs.length > 0) {
+ for (const job of finalJobs) {
+ const res = await redisEvictConnection.zrem(
+ constructQueueKey(teamId),
+ job._member,
+ );
+ if (res !== 0) {
+ finalJob = job;
+ break;
+ }
+ }
+
+ if (finalJob === null) {
+ // It's normal for this to happen, but if it happens too many times, we should log a warning
+ if (i > 100) {
+ logger.error(
+ "Failed to remove job from concurrency limit queue, hard bailing",
+ {
+ teamId,
+ jobIds: finalJobs.map(x => x.job.id),
+ zeroDataRetention: finalJobs.some(
+ x => x.job.data?.zeroDataRetention,
+ ),
+ i,
+ },
+ );
+ return null;
+ } else if (i > 15) {
+ logger.warn("Failed to remove job from concurrency limit queue", {
+ teamId,
+ jobIds: finalJobs.map(x => x.job.id),
+ zeroDataRetention: finalJobs.some(x => x.job.data?.zeroDataRetention),
+ i,
+ });
+ }
+
+ return await new Promise((resolve, reject) =>
+ setTimeout(
+ () => {
+ getNextConcurrentJob(teamId, i + 1)
+ .then(resolve)
+ .catch(reject);
+ },
+ Math.floor(Math.random() * 300),
+ ),
+ ); // Stagger the workers off to break up the clump that causes the race condition
+ } else {
+ logger.debug("Removed job from concurrency limit queue", {
+ teamId,
+ jobId: finalJob.job.id,
+ zeroDataRetention: finalJob.job.data?.zeroDataRetention,
+ i,
+ });
+ }
+ }
+
+ return finalJob;
+}
+
+/**
+ * Called when a job associated with a concurrency queue is done.
+ *
+ * @param job The BullMQ job that is done.
+ */
+export async function concurrentJobDone(job: NuQJob<any>) {
+ if (job.id && job.data && job.data.team_id) {
+ await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
+ await cleanOldConcurrencyLimitEntries(job.data.team_id);
+
+ if (job.data.crawl_id) {
+ await removeCrawlConcurrencyLimitActiveJob(job.data.crawl_id, job.id);
+ await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);
+ }
+
+ const maxTeamConcurrency =
+ (
+ await getACUCTeam(
+ job.data.team_id,
+ false,
+ true,
+ job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+ const currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(job.data.team_id)
+ ).length;
+
+ if (currentActiveConcurrency < maxTeamConcurrency) {
+ const nextJob = await getNextConcurrentJob(job.data.team_id);
+ if (nextJob !== null) {
+ await pushConcurrencyLimitActiveJob(
+ job.data.team_id,
+ nextJob.job.id,
+ 60 * 1000,
+ );
+
+ if (nextJob.job.data.crawl_id) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ 60 * 1000,
+ );
+
+ const sc = await getCrawl(nextJob.job.data.crawl_id);
+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {
+ await new Promise(resolve =>
+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),
+ );
+ }
+ }
+
+ abTestJob(nextJob.job.data);
+
+ await scrapeQueue.addJob(
+ nextJob.job.id,
+ {
+ ...nextJob.job.data,
+ concurrencyLimitHit: true,
+ },
+ nextJob.job.priority,
+ nextJob.job.listenable,
+ );
+ }
+ }
+ }
+}
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 1b256ea67..ff4ee837f 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -49,8 +49,14 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
}
-export async function recordRobotsBlocked(crawlId: string, url: string) {
- await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);
+export async function recordRobotsBlocked(
+ crawlId: string,
+ url: string,
+) {
+ await redisEvictConnection.sadd(
+ "crawl:" + crawlId + ":robots_blocked",
+ url,
+ );
await redisEvictConnection.expire(
"crawl:" + crawlId + ":robots_blocked",
24 * 60 * 60,
diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.ts
index dd0f1da67..bfee651cf 100644
--- a/apps/api/src/lib/search-index-client.ts
+++ b/apps/api/src/lib/search-index-client.ts
@@ -1,6 +1,6 @@
/**
* HTTP Client for Search Index Service
- *
+ *
* This client communicates with the standalone search index service
* (firecrawl-search backend) via HTTP.
*/
@@ -8,13 +8,13 @@
import { logger as _logger } from "./logger";
import type { Logger } from "winston";
-interface SearchIndexClientConfig {
+export interface SearchIndexClientConfig {
baseUrl: string;
apiSecret?: string;
timeout?: number;
}
-interface IndexDocumentRequest {
+export interface IndexDocumentRequest {
url: string;
resolvedUrl: string;
title?: string;
@@ -43,7 +43,7 @@ export interface SearchRequest {
};
}
-interface SearchResult {
+export interface SearchResult {
documentId: string;
url: string;
title: string | null;
@@ -58,7 +58,7 @@ interface SearchResult {
lastCrawledAt: string;
}
-interface SearchResponse {
+export interface SearchResponse {
results: SearchResult[];
total: number;
query: string;
@@ -75,13 +75,13 @@ export class SearchIndexClient {
private baseUrl: string;
private apiSecret?: string;
private timeout: number;
-
+
constructor(config: SearchIndexClientConfig) {
this.baseUrl = config.baseUrl.replace(/\/$/, ""); // Remove trailing slash
this.apiSecret = config.apiSecret;
this.timeout = config.timeout || 30000;
}
-
+
/**
* Check if search index service is enabled
*/
@@ -91,7 +91,7 @@ export class SearchIndexClient {
process.env.ENABLE_SEARCH_INDEX === "true"
);
}
-
+
/**
* Make HTTP request to search service
*/
@@ -103,36 +103,36 @@ export class SearchIndexClient {
): Promise<T> {
const log = logger ?? _logger.child({ module: "search-index-client" });
const url = `${this.baseUrl}${path}`;
-
+
const headers: Record<string, string> = {
"Content-Type": "application/json",
};
-
+
if (this.apiSecret) {
headers["X-API-Secret"] = this.apiSecret;
}
-
+
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
-
+
try {
log.debug("Making request to search service", {
method,
url,
hasBody: !!body,
});
-
+
const response = await fetch(url, {
method,
headers,
body: body ? JSON.stringify(body) : undefined,
signal: controller.signal,
});
-
+
clearTimeout(timeoutId);
-
+
const data = await response.json();
-
+
if (!response.ok) {
log.error("Search service request failed", {
status: response.status,
@@ -142,19 +142,16 @@ export class SearchIndexClient {
data.error || `Search service returned ${response.status}`,
);
}
-
+
return data;
} catch (error) {
clearTimeout(timeoutId);
-
+
if (error.name === "AbortError") {
- log.error("Search service request timed out", {
- url,
- timeout: this.timeout,
- });
+ log.error("Search service request timed out", { url, timeout: this.timeout });
throw new Error("Search service request timed out");
}
-
+
log.error("Search service request failed", {
error: (error as Error).message,
url,
@@ -162,7 +159,7 @@ export class SearchIndexClient {
throw error;
}
}
-
+
/**
* Index a document (async - queues for processing)
*/
@@ -171,7 +168,7 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<{ success: boolean; message: string }> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<any>(
"POST",
@@ -179,12 +176,12 @@ export class SearchIndexClient {
request,
log,
);
-
+
log.info("Document queued for indexing", {
url: request.url,
success: response.success,
});
-
+
return {
success: response.success,
message: response.message || "Document queued for indexing",
@@ -194,7 +191,7 @@ export class SearchIndexClient {
error: (error as Error).message,
url: request.url,
});
-
+
// Don't throw - indexing failures shouldn't break scraping
return {
success: false,
@@ -202,7 +199,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Search indexed documents
*/
@@ -211,30 +208,32 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<SearchResponse> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
- const response = await this.request<{
- success: boolean;
- data: SearchResponse;
- }>("POST", "/api/search", request, log);
-
+ const response = await this.request<{ success: boolean; data: SearchResponse }>(
+ "POST",
+ "/api/search",
+ request,
+ log,
+ );
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
log.info("Search completed", {
query: request.query,
results: response.data.results.length,
took: response.data.took,
});
-
+
return response.data;
} catch (error) {
log.error("Search request failed", {
error: (error as Error).message,
query: request.query,
});
-
+
// Return empty results on error
return {
results: [],
@@ -250,7 +249,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Get search index statistics
*/
@@ -259,7 +258,7 @@ export class SearchIndexClient {
queue: any;
}> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean; data: any }>(
"GET",
@@ -267,30 +266,30 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
return response.data;
} catch (error) {
log.error("Failed to get search stats", {
error: (error as Error).message,
});
-
+
return {
index: {},
queue: {},
};
}
}
-
+
/**
* Health check
*/
async health(logger?: Logger): Promise<boolean> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean }>(
"GET",
@@ -298,7 +297,7 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
return response.success;
} catch (error) {
log.warn("Search service health check failed", {
@@ -319,22 +318,22 @@ export function getSearchIndexClient(): SearchIndexClient | null {
if (!SearchIndexClient.isEnabled()) {
return null;
}
-
+
if (!searchIndexClient) {
const baseUrl = process.env.SEARCH_SERVICE_URL!;
const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;
-
+
searchIndexClient = new SearchIndexClient({
baseUrl,
apiSecret,
timeout: 30000,
});
-
+
_logger.info("Search index client initialized", {
baseUrl: baseUrl.substring(0, 30) + "...",
});
}
-
+
return searchIndexClient;
}
@@ -346,11 +345,11 @@ export async function indexDocumentIfEnabled(
logger?: Logger,
): Promise<void> {
const client = getSearchIndexClient();
-
+
if (!client) {
return;
}
-
+
try {
await client.indexDocument(request, logger);
} catch (error) {
@@ -361,3 +360,4 @@ export async function indexDocumentIfEnabled(
});
}
}
+
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 205b3aae3..370e98291 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -3,6 +3,7 @@ import { redisHealthController } from "../controllers/v0/admin/redis-health";
import { authMiddleware, blocklistMiddleware, wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
+import { cclogController } from "../controllers/v0/admin/cclog";
import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";
import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
@@ -30,6 +31,11 @@ adminRouter.get(
wrap(checkFireEngine),
);
+adminRouter.get(
+ `/admin/${process.env.BULL_AUTH_KEY}/cclog`,
+ wrap(cclogController),
+);
+
adminRouter.get(
`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`,
wrap(zdrcleanerController),
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 22b21f62c..e662e999b 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -34,10 +34,9 @@ import {
toV0CrawlerOptions,
} from "../../controllers/v2/types";
import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
-import { addScrapeJob } from "../queue-jobs";
+import { _addScrapeJobToBullMQ } from "../queue-jobs";
import { BullMQOtel } from "bullmq-otel";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
-import { crawlGroup, scrapeQueue } from "../worker/nuq";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
@@ -491,19 +490,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {
// });
// }
- await crawlGroup.addGroup(crawlId, {
- concurrency: [
- {
- queue: scrapeQueue,
- maxConcurrency: sc.maxConcurrency ?? undefined,
- },
- ],
- ownerId: sc.team_id,
- });
-
await saveCrawl(crawlId, sc);
- await addScrapeJob(
+ await _addScrapeJobToBullMQ(
{
url: url,
mode: "kickoff" as const,
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 6648c3f89..885be5005 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,8 +1,21 @@
import { v4 as uuidv4 } from "uuid";
-import { ScrapeJobData } from "../types";
+import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";
+import {
+ cleanOldConcurrencyLimitEntries,
+ getConcurrencyLimitActiveJobs,
+ getConcurrencyQueueJobsCount,
+ getCrawlConcurrencyLimitActiveJobs,
+ pushConcurrencyLimitActiveJob,
+ pushConcurrencyLimitedJob,
+ pushCrawlConcurrencyLimitActiveJob,
+} from "../lib/concurrency-limit";
import { logger as _logger } from "../lib/logger";
+import { sendNotificationWithCustomDays } from "./notification/email_notification";
+import { shouldSendConcurrencyLimitNotification } from "./notification/notification-check";
+import { getACUCTeam } from "../controllers/auth";
import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";
import { Document } from "../controllers/v1/types";
+import { getCrawl } from "../lib/crawl-redis";
import { Logger } from "winston";
import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
@@ -10,6 +23,171 @@ import { abTestJob } from "./ab-test";
import { NuQJob, scrapeQueue } from "./worker/nuq";
import { serializeTraceContext } from "../lib/otel-tracer";
+/**
+ * Checks if a job is a crawl or batch scrape based on its options
+ * @param options The job options containing crawlerOptions and crawl_id
+ * @returns true if the job is either a crawl or batch scrape
+ */
+function isCrawlOrBatchScrape(options: {
+ crawlerOptions?: any;
+ crawl_id?: string;
+}): boolean {
+ // If crawlerOptions exists, it's a crawl
+ // If crawl_id exists but no crawlerOptions, it's a batch scrape
+ return !!options.crawlerOptions || !!options.crawl_id;
+}
+
+async function _addScrapeJobToConcurrencyQueue(
+ webScraperOptions: any,
+ jobId: string,
+ priority: number = 0,
+ listenable: boolean = false,
+) {
+ await pushConcurrencyLimitedJob(
+ webScraperOptions.team_id,
+ {
+ id: jobId,
+ data: webScraperOptions,
+ priority,
+ listenable,
+ },
+ webScraperOptions.crawl_id
+ ? Infinity
+ : (webScraperOptions.scrapeOptions?.timeout ?? 60 * 1000),
+ );
+}
+
+export async function _addScrapeJobToBullMQ(
+ webScraperOptions: ScrapeJobData,
+ jobId: string,
+ priority: number = 0,
+ listenable: boolean = false,
+): Promise<NuQJob<ScrapeJobData>> {
+ if (webScraperOptions.mode === "single_urls") {
+ abTestJob(webScraperOptions);
+ }
+
+ if (webScraperOptions && webScraperOptions.team_id) {
+ await pushConcurrencyLimitActiveJob(
+ webScraperOptions.team_id,
+ jobId,
+ 60 * 1000,
+ ); // 60s default timeout
+
+ if (webScraperOptions.crawl_id) {
+ const sc = await getCrawl(webScraperOptions.crawl_id);
+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ webScraperOptions.crawl_id,
+ jobId,
+ 60 * 1000,
+ );
+ }
+ }
+ }
+
+ return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);
+}
+
+async function addScrapeJobRaw(
+ webScraperOptions: ScrapeJobData,
+ jobId: string,
+ priority: number = 0,
+ directToBullMQ: boolean = false,
+ listenable: boolean = false,
+): Promise<NuQJob<ScrapeJobData> | null> {
+ let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
+ let currentActiveConcurrency = 0;
+ let maxConcurrency = 0;
+
+ if (directToBullMQ) {
+ concurrencyLimited = "no";
+ } else {
+ if (webScraperOptions.crawl_id) {
+ const crawl = await getCrawl(webScraperOptions.crawl_id);
+ const concurrencyLimit = !crawl
+ ? null
+ : crawl.crawlerOptions?.delay === undefined &&
+ crawl.maxConcurrency === undefined
+ ? null
+ : (crawl.maxConcurrency ?? 1);
+
+ if (concurrencyLimit !== null) {
+ const crawlConcurrency = (
+ await getCrawlConcurrencyLimitActiveJobs(webScraperOptions.crawl_id)
+ ).length;
+ const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);
+ if (freeSlots === 0) {
+ concurrencyLimited = "yes-crawl";
+ }
+ }
+ }
+
+ if (concurrencyLimited === null) {
+ const now = Date.now();
+ const maxConcurrency =
+ (
+ await getACUCTeam(
+ webScraperOptions.team_id,
+ false,
+ true,
+ webScraperOptions.mode === "single_urls" &&
+ webScraperOptions.from_extract
+ ? RateLimiterMode.Extract
+ : RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+ await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
+ const currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)
+ ).length;
+ concurrencyLimited =
+ currentActiveConcurrency >= maxConcurrency ? "yes" : "no";
+ }
+ }
+
+ if (concurrencyLimited === "yes" || concurrencyLimited === "yes-crawl") {
+ if (concurrencyLimited === "yes") {
+ // Detect if they hit their concurrent limit
+ // If above by 2x, send them an email
+ // No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x
+ const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(
+ webScraperOptions.team_id,
+ );
+ if (concurrencyQueueJobs > maxConcurrency) {
+ // logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);
+
+ // Only send notification if it's not a crawl or batch scrape
+ const shouldSendNotification =
+ await shouldSendConcurrencyLimitNotification(
+ webScraperOptions.team_id,
+ );
+ if (shouldSendNotification) {
+ sendNotificationWithCustomDays(
+ webScraperOptions.team_id,
+ NotificationType.CONCURRENCY_LIMIT_REACHED,
+ 15,
+ false,
+ true,
+ ).catch(error => {
+ _logger.error(
+ "Error sending notification (concurrency limit reached)",
+ { error },
+ );
+ });
+ }
+ }
+ }
+
+ webScraperOptions.concurrencyLimited = true;
+
+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId, priority, listenable);
+ return null;
+ } else {
+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority, listenable);
+ }
+}
+
export async function addScrapeJob(
webScraperOptions: ScrapeJobData,
jobId: string = uuidv4(),
@@ -24,22 +202,13 @@ export async function addScrapeJob(
traceContext,
};
- if (webScraperOptions.mode === "single_urls") {
- abTestJob(webScraperOptions);
- }
-
- return await scrapeQueue.addJob(jobId, webScraperOptions, {
+ return await addScrapeJobRaw(
+ optionsWithTrace,
+ jobId,
priority,
+ directToBullMQ,
listenable,
- ownerId: webScraperOptions.team_id,
- groupId: webScraperOptions.crawl_id ?? undefined,
- timesOutAt: webScraperOptions.crawl_id
- ? undefined
- : new Date(
- Date.now() +
- ((webScraperOptions as any).scrapeOptions?.timeout ?? 300) * 1000,
- ),
- });
+ );
}
export async function addScrapeJobs(
@@ -55,27 +224,173 @@ export async function addScrapeJobs(
// Capture trace context for all jobs
const traceContext = serializeTraceContext();
+ const jobsByTeam = new Map<
+ string,
+ {
+ jobId: string;
+ data: ScrapeJobData;
+ priority: number;
+ listenable?: boolean;
+ }[]
+ >();
+
for (const job of jobs) {
- if (job.data.mode === "single_urls") {
- abTestJob(job.data);
+ if (!jobsByTeam.has(job.data.team_id)) {
+ jobsByTeam.set(job.data.team_id, []);
}
+ jobsByTeam.get(job.data.team_id)!.push(job);
}
- await scrapeQueue.addJobs(
- jobs.map(job => ({
- data: {
- ...job.data,
- traceContext,
- },
- id: job.jobId,
- options: {
- priority: job.priority,
- listenable: job.listenable,
- ownerId: job.data.team_id,
- groupId: job.data.crawl_id ?? undefined,
- },
- })),
- );
+ for (const [teamId, teamJobs] of jobsByTeam) {
+ // == Buckets for jobs ==
+ let jobsForcedToCQ: {
+ data: ScrapeJobData;
+ jobId: string;
+ priority: number;
+ listenable?: boolean;
+ }[] = [];
+
+ let jobsPotentiallyInCQ: {
+ data: ScrapeJobData;
+ jobId: string;
+ priority: number;
+ listenable?: boolean;
+ }[] = [];
+
+ // == Select jobs by crawl ID ==
+ const jobsByCrawlID = new Map<
+ string,
+ {
+ data: ScrapeJobData;
+ jobId: string;
+ priority: number;
+ listenable?: boolean;
+ }[]
+ >();
+
+ const jobsWithoutCrawlID: {
+ data: ScrapeJobData;
+ jobId: string;
+ priority: number;
+ listenable?: boolean;
+ }[] = [];
+
+ for (const job of teamJobs) {
+ if (job.data.crawl_id) {
+ if (!jobsByCrawlID.has(job.data.crawl_id)) {
+ jobsByCrawlID.set(job.data.crawl_id, []);
+ }
+ jobsByCrawlID.get(job.data.crawl_id)!.push(job);
+ } else {
+ jobsWithoutCrawlID.push(job);
+ }
+ }
+
+ // == Select jobs by crawl ID ==
+ for (const [crawlID, crawlJobs] of jobsByCrawlID) {
+ const crawl = await getCrawl(crawlID);
+ const concurrencyLimit = !crawl
+ ? null
+ : crawl.crawlerOptions?.delay === undefined &&
+ crawl.maxConcurrency === undefined
+ ? null
+ : (crawl.maxConcurrency ?? 1);
+
+ if (concurrencyLimit === null) {
+ // All jobs may be in the CQ depending on the global team concurrency limit
+ jobsPotentiallyInCQ.push(...crawlJobs);
+ } else {
+ const crawlConcurrency = (
+ await getCrawlConcurrencyLimitActiveJobs(crawlID)
+ ).length;
+ const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);
+
+ // The first n jobs may be in the CQ depending on the global team concurrency limit
+ jobsPotentiallyInCQ.push(...crawlJobs.slice(0, freeSlots));
+
+ // Every job after that must be in the CQ, as the crawl concurrency limit has been reached
+ jobsForcedToCQ.push(...crawlJobs.slice(freeSlots));
+ }
+ }
+
+ // All jobs without a crawl ID may be in the CQ depending on the global team concurrency limit
+ jobsPotentiallyInCQ.push(...jobsWithoutCrawlID);
+
+ const now = Date.now();
+ const maxConcurrency =
+ (
+ await getACUCTeam(
+ teamId,
+ false,
+ true,
+ jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract
+ ? RateLimiterMode.Extract
+ : RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+ await cleanOldConcurrencyLimitEntries(teamId, now);
+
+ const currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(teamId, now)
+ ).length;
+
+ const countCanBeDirectlyAdded = Math.max(
+ maxConcurrency - currentActiveConcurrency,
+ 0,
+ );
+
+ const addToBull = jobsPotentiallyInCQ.slice(0, countCanBeDirectlyAdded);
+ const addToCQ = jobsPotentiallyInCQ
+ .slice(countCanBeDirectlyAdded)
+ .concat(jobsForcedToCQ);
+
+ // equals 2x the max concurrency
+ if (jobsPotentiallyInCQ.length - countCanBeDirectlyAdded > maxConcurrency) {
+ // logger.info(`Concurrency limited 2x (multiple) - Concurrency queue jobs: ${addToCQ.length} Max concurrency: ${maxConcurrency} Team ID: ${jobs[0].data.team_id}`);
+ // Only send notification if it's not a crawl or batch scrape
+ if (!isCrawlOrBatchScrape(jobs[0].data)) {
+ const shouldSendNotification =
+ await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);
+ if (shouldSendNotification) {
+ sendNotificationWithCustomDays(
+ jobs[0].data.team_id,
+ NotificationType.CONCURRENCY_LIMIT_REACHED,
+ 15,
+ false,
+ true,
+ ).catch(error => {
+ _logger.error(
+ "Error sending notification (concurrency limit reached)",
+ { error },
+ );
+ });
+ }
+ }
+ }
+
+ await Promise.all(
+ addToCQ.map(async job => {
+ const size = JSON.stringify(job.data).length;
+ await _addScrapeJobToConcurrencyQueue(
+ { ...job.data, traceContext },
+ job.jobId,
+ job.priority,
+ job.listenable,
+ );
+ }),
+ );
+
+ await Promise.all(
+ addToBull.map(async job => {
+ await _addScrapeJobToBullMQ(
+ { ...job.data, traceContext },
+ job.jobId,
+ job.priority,
+ job.listenable,
+ );
+ }),
+ );
+ }
}
export async function waitForJob(
diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.ts
new file mode 100644
index 000000000..8b199f323
--- /dev/null
+++ b/apps/api/src/services/search-index-db.ts
@@ -0,0 +1,104 @@
+/**
+ * Search Index Database Service
+ *
+ * Separate Supabase client for the search index database.
+ * This allows independent scaling and isolation from the main index DB.
+ */
+
+import { createClient, SupabaseClient } from "@supabase/supabase-js";
+import { logger as _logger } from "../lib/logger";
+import { configDotenv } from "dotenv";
+
+configDotenv();
+
+/**
+ * SearchIndexSupabaseService - Manages connection to search index database
+ */
+class SearchIndexSupabaseService {
+ private client: SupabaseClient | null = null;
+
+ constructor() {
+ const supabaseUrl = process.env.SEARCH_INDEX_SUPABASE_URL;
+ const supabaseServiceToken = process.env.SEARCH_INDEX_SUPABASE_SERVICE_TOKEN;
+
+ // Only initialize if both URL and token are provided
+ if (!supabaseUrl || !supabaseServiceToken) {
+ _logger.warn("Search index database not configured. Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN to enable.");
+ this.client = null;
+ } else {
+ this.client = createClient(supabaseUrl, supabaseServiceToken);
+ _logger.info("Search index database client initialized", {
+ url: supabaseUrl.substring(0, 30) + "...",
+ });
+ }
+ }
+
+ /**
+ * Get the Supabase client for search index
+ */
+ getClient(): SupabaseClient | null {
+ return this.client;
+ }
+
+ /**
+ * Check if search index is enabled
+ */
+ isEnabled(): boolean {
+ return this.client !== null;
+ }
+}
+
+const searchIndexService = new SearchIndexSupabaseService();
+
+/**
+ * Proxy to provide clean error messages when search index is not configured
+ */
+export const search_index_supabase_service: SupabaseClient = new Proxy(
+ searchIndexService,
+ {
+ get: function (target, prop, receiver) {
+ const client = target.getClient();
+
+ // If client is not initialized, throw immediately to prevent nested access errors
+ if (client === null) {
+ throw new Error(
+ "Search index database is not configured. " +
+ "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."
+ );
+ }
+
+ // Direct access to service properties
+ if (prop in target) {
+ return Reflect.get(target, prop, receiver);
+ }
+
+ // Delegate to Supabase client
+ return Reflect.get(client, prop, receiver);
+ },
+ }
+) as unknown as SupabaseClient;
+
+/**
+ * Check if search index database is enabled
+ */
+export function isSearchIndexEnabled(): boolean {
+ return (
+ process.env.ENABLE_SEARCH_INDEX === "true" &&
+ searchIndexService.isEnabled()
+ );
+}
+
+/**
+ * Get search index database client (throws if not configured)
+ */
+export function getSearchIndexClient(): SupabaseClient {
+ const client = searchIndexService.getClient();
+ if (!client) {
+ throw new Error(
+ "Search index database is not configured. " +
+ "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN."
+ );
+ }
+ return client;
+}
+
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 4131f4388..af3489d44 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -18,7 +18,7 @@ import { v4 as uuidv4 } from "uuid";
import { addScrapeJobs } from "../queue-jobs";
import { getJobs } from "../../controllers/v1/crawl-status";
import { logJob } from "../logging/log_job";
-import { createWebhookSender, WebhookEvent } from "../webhook/index";
+import { createWebhookSender, WebhookEvent } from "../webhook";
import { hasFormatOfType } from "../../lib/format-utils";
import type { NuQJob } from "./nuq";
import { ScrapeJobData } from "../../types";
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.ts
index ead9341e1..286265a6b 100644
--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts
+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts
@@ -1,45 +1,35 @@
import "dotenv/config";
-import {
- scrapeQueue,
- nuqGetLocalMetrics,
- nuqHealthCheck,
- nuqShutdown,
-} from "./nuq";
+import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck, nuqShutdown } from "./nuq";
import Express from "express";
import { logger } from "../../lib/logger";
(async () => {
- const app = Express();
+ const app = Express();
- app.get("/metrics", (_, res) =>
- res.contentType("text/plain").send(nuqGetLocalMetrics()),
- );
- app.get("/health", async (_, res) => {
- if (await nuqHealthCheck()) {
- res.status(200).send("OK");
- } else {
- res.status(500).send("Not OK");
- }
- });
+ app.get("/metrics", (_, res) => res.contentType("text/plain").send(nuqGetLocalMetrics()));
+ app.get("/health", async (_, res) => {
+ if (await nuqHealthCheck()) {
+ res.status(200).send("OK");
+ } else {
+ res.status(500).send("Not OK");
+ }
+ });
- const server = app.listen(
- Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),
- () => {
- logger.info("NuQ prefetch worker metrics server started");
- },
- );
+ const server = app.listen(Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011), () => {
+ logger.info("NuQ prefetch worker metrics server started");
+ });
- async function shutdown() {
- server.close();
- await nuqShutdown();
- process.exit(0);
- }
+ async function shutdown() {
+ server.close();
+ await nuqShutdown();
+ process.exit(0);
+ }
- process.on("SIGINT", shutdown);
- process.on("SIGTERM", shutdown);
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
- while (true) {
- await scrapeQueue.prefetchJobs();
- await new Promise(resolve => setTimeout(resolve, 750));
- }
+ while (true) {
+ await scrapeQueue.prefetchJobs();
+ await new Promise(resolve => setTimeout(resolve, 250));
+ }
})();
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index cd7b6f781..13a49a555 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -4,7 +4,6 @@ import { Client, Pool } from "pg";
import { type ScrapeJobData } from "../../types";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import amqp from "amqplib";
-import { v5 as uuidv5, validate as uuidValidate } from "uuid";
// === Basics
@@ -29,42 +28,14 @@ export type NuQJob<Data = any, ReturnValue = any> = {
returnvalue?: ReturnValue;
failedReason?: string;
lock?: string;
- ownerId?: string;
- groupId?: string;
- timesOutAt?: Date;
};
const listenChannelId = process.env.NUQ_POD_NAME ?? "main";
// === Queue
-type NuQOptions = {
- concurrencyLimit?: false | "per-owner" | "per-owner-per-group";
-};
-
-type NuQJobOptions = {
- listenable?: boolean;
- priority?: number;
- ownerId?: string;
- groupId?: string;
- timesOutAt?: Date;
-};
-
-function normalizeOwnerId(ownerId: string | undefined | null) {
- const bareOwnerId = ownerId ?? undefined;
- const normalizedOwnerId = bareOwnerId
- ? uuidValidate(bareOwnerId)
- ? bareOwnerId
- : uuidv5(bareOwnerId, "b208cbac-8bdf-4599-bf17-da78426e3f7c") // preview namespace
- : null;
- return normalizedOwnerId;
-}
-
class NuQ<JobData = any, JobReturnValue = any> {
- constructor(
- public readonly queueName: string,
- public readonly options: NuQOptions = { concurrencyLimit: false },
- ) {}
+ constructor(public readonly queueName: string) {}
// === Listener
@@ -114,7 +85,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
let reconnectTimeout: NodeJS.Timeout | null = null;
- const onClose = function onClose() {
+ const onClose = (function onClose() {
logger.info("NuQ listener channel closed", {
module: "nuq/rabbitmq",
});
@@ -133,7 +104,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
250,
);
return;
- }.bind(this);
+ }).bind(this);
connection.on("close", onClose);
channel.on("close", onClose);
@@ -143,7 +114,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
(msg => {
if (msg === null) {
onClose();
- return;
+ return;
}
logger.info("NuQ job received", {
@@ -295,7 +266,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
await this.startSender();
if (this.sender) {
- this.sender.channel.sendToQueue(
+ await this.sender.channel.sendToQueue(
this.queueName + ".listen." + listenChannelId,
Buffer.from(status, "utf8"),
{
@@ -315,7 +286,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
await this.startSender();
if (this.sender) {
- this.sender.channel.sendToQueue(
+ await this.sender.channel.sendToQueue(
this.queueName + ".prefetch",
Buffer.from(JSON.stringify(job), "utf8"),
{
@@ -343,9 +314,6 @@ class NuQ<JobData = any, JobReturnValue = any> {
"returnvalue",
"failedreason",
"lock",
- "owner_id",
- "group_id",
- "times_out_at",
];
private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {
@@ -361,9 +329,6 @@ class NuQ<JobData = any, JobReturnValue = any> {
returnvalue: row.returnvalue ?? undefined,
failedReason: row.failedreason ?? undefined,
lock: row.lock ?? undefined,
- ownerId: row.owner_id ?? undefined,
- groupId: row.group_id ?? undefined,
- timesOutAt: row.times_out_at ? new Date(row.times_out_at) : undefined,
};
}
@@ -485,60 +450,6 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
- public async getJobCountsOfGroup(
- groupId: string,
- _logger: Logger = logger,
- ): Promise<Record<NuQJobStatus, number>> {
- const start = Date.now();
- try {
- const stats = await nuqPool.query(
- `
- SELECT status, COUNT(*) as count FROM nuq.queue_scrape WHERE group_id = $1 GROUP BY status;
- `,
- [groupId],
- );
-
- return Object.fromEntries(stats.rows.map(x => [x.status, x.count]));
- } finally {
- _logger.info("nuqGetJobCountsOfGroup metrics", {
- module: "nuq/metrics",
- method: "nuqGetJobCountsOfGroup",
- duration: Date.now() - start,
- groupId,
- });
- }
- }
-
- public async getJobsOfGroupWithStatus(
- groupId: string,
- status: NuQJobStatus,
- limit: number,
- offset: number,
- _logger: Logger = logger,
- ): Promise<NuQJob<JobData, JobReturnValue>[]> {
- const start = Date.now();
- try {
- return (
- await nuqPool.query(
- `
- SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName}
- WHERE group_id = $1 AND status = $2
- ORDER BY finished_at ASC
- LIMIT $3 OFFSET $4
- `,
- [groupId, status, limit, offset],
- )
- ).rows.map(x => this.rowToJob(x)!);
- } finally {
- _logger.info("nuqGetJobsOfGroupWithStatus metrics", {
- module: "nuq/metrics",
- method: "nuqGetJobsOfGroupWithStatus",
- duration: Date.now() - start,
- groupId,
- });
- }
- }
-
public async removeJob(
id: string,
_logger: Logger = logger,
@@ -589,72 +500,19 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
// === Producer
- public async tryAddJob(
- id: string,
- data: JobData,
- options: NuQJobOptions = {},
- ): Promise<NuQJob<JobData, JobReturnValue> | null> {
- return withSpan("nuq.tryAddJob", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.job_id": id,
- "nuq.priority": options.priority ?? 0,
- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
- "nuq.listenable": options.listenable ?? false,
- });
-
- const start = Date.now();
- try {
- const result = this.rowToJob(
- (
- await nuqPool.query(
- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING RETURNING ${this.jobReturning.join(", ")};`,
- [
- id,
- data,
- options.priority ?? 0,
- options.listenable ? listenChannelId : null,
- normalizeOwnerId(options.ownerId) ?? null,
- options.groupId ?? null,
- options.timesOutAt ? options.timesOutAt.toISOString() : null,
- ],
- )
- ).rows[0],
- )!;
-
- setSpanAttributes(span, {
- "nuq.job_created": result !== null,
- });
-
- return result;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- logger.info("nuqAddJob metrics", {
- module: "nuq/metrics",
- method: "nuqAddJob",
- duration,
- scrapeId: id,
- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
- });
- }
- });
- }
-
public async addJob(
id: string,
data: JobData,
- options: NuQJobOptions = {},
+ priority: number = 0,
+ listenable: boolean = false,
): Promise<NuQJob<JobData, JobReturnValue>> {
return withSpan("nuq.addJob", async span => {
setSpanAttributes(span, {
"nuq.queue_name": this.queueName,
"nuq.job_id": id,
- "nuq.priority": options.priority ?? 0,
+ "nuq.priority": priority,
"nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
- "nuq.listenable": options.listenable ?? false,
+ "nuq.listenable": listenable,
});
const start = Date.now();
@@ -662,16 +520,8 @@ class NuQ<JobData = any, JobReturnValue = any> {
const result = this.rowToJob(
(
await nuqPool.query(
- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING ${this.jobReturning.join(", ")};`,
- [
- id,
- data,
- options.priority ?? 0,
- options.listenable ? listenChannelId : null,
- normalizeOwnerId(options.ownerId) ?? null,
- options.groupId ?? null,
- options.timesOutAt ? options.timesOutAt.toISOString() : null,
- ],
+ `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id) VALUES ($1, $2, $3, $4) RETURNING ${this.jobReturning.join(", ")};`,
+ [id, data, priority, listenable ? listenChannelId : null],
)
).rows[0],
)!;
@@ -697,87 +547,6 @@ class NuQ<JobData = any, JobReturnValue = any> {
});
}
- public async addJobs(
- jobs: Array<{
- id: string;
- data: JobData;
- options?: NuQJobOptions;
- }>,
- _logger: Logger = logger,
- ): Promise<NuQJob<JobData, JobReturnValue>[]> {
- if (jobs.length === 0) return [];
-
- return withSpan("nuq.addJobs", async span => {
- setSpanAttributes(span, {
- "nuq.queue_name": this.queueName,
- "nuq.job_count": jobs.length,
- });
-
- const start = Date.now();
- try {
- // Prepare arrays for bulk insert
- const ids: string[] = [];
- const dataArray: JobData[] = [];
- const priorities: number[] = [];
- const listenChannelIds: (string | null)[] = [];
- const ownerIds: (string | null)[] = [];
- const groupIds: (string | null)[] = [];
- const timesOutAts: (string | null)[] = [];
-
- for (const job of jobs) {
- ids.push(job.id);
- dataArray.push(job.data);
- priorities.push(job.options?.priority ?? 0);
- listenChannelIds.push(
- job.options?.listenable ? listenChannelId : null,
- );
- ownerIds.push(normalizeOwnerId(job.options?.ownerId));
- groupIds.push(job.options?.groupId ?? null);
- timesOutAts.push(
- job.options?.timesOutAt
- ? job.options.timesOutAt.toISOString()
- : null,
- );
- }
-
- // Bulk insert using UNNEST
- const result = await nuqPool.query(
- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id, group_id, times_out_at)
- SELECT * FROM UNNEST($1::uuid[], $2::jsonb[], $3::int[], $4::text[], $5::uuid[], $6::uuid[], $7::timestamptz[])
- RETURNING ${this.jobReturning.join(", ")};`,
- [
- ids,
- dataArray,
- priorities,
- listenChannelIds,
- ownerIds,
- groupIds,
- timesOutAts,
- ],
- );
-
- const createdJobs = result.rows.map(row => this.rowToJob(row)!);
-
- setSpanAttributes(span, {
- "nuq.jobs_created": createdJobs.length,
- });
-
- return createdJobs;
- } finally {
- const duration = Date.now() - start;
- setSpanAttributes(span, {
- "nuq.duration_ms": duration,
- });
- _logger.info("nuqAddJobs metrics", {
- module: "nuq/metrics",
- method: "nuqAddJobs",
- duration,
- jobCount: jobs.length,
- });
- }
- });
- }
-
private readonly nuqWaitMode =
process.env.NUQ_WAIT_MODE === "listen" || process.env.NUQ_RABBITMQ_URL
? ("listen" as const)
@@ -896,229 +665,14 @@ class NuQ<JobData = any, JobReturnValue = any> {
public async prefetchJobs(_logger: Logger = logger): Promise<number> {
const start = Date.now();
try {
- let shardingIndex = parseInt(
- (process.env.NUQ_POD_NAME ?? "0").split("-").slice(-1)[0],
- 10,
- );
- let shardingCount = parseInt(
- process.env.NUQ_PREFETCH_REPLICAS ?? "1",
- 10,
- );
-
- if (
- isNaN(shardingIndex) ||
- isNaN(shardingCount) ||
- shardingIndex >= shardingCount ||
- shardingIndex < 0 ||
- shardingCount < 1
- ) {
- _logger.warn(
- "Sharding config invalid, falling back to guaranteed working defaults.",
- {
- shardingCount,
- shardingIndex,
- module: "nuq",
- method: "nuqPrefetchJobs",
- },
- );
- shardingIndex = 0;
- shardingCount = 1;
- }
-
- let updateQuery: string;
- if (this.options.concurrencyLimit === "per-owner") {
- updateQuery = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- owner_limited_jobs AS (
- SELECT
- q.id,
- q.owner_id,
- ROW_NUMBER() OVER (
- PARTITION BY q.owner_id
- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC
- ) as owner_rank,
- GREATEST(
- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id))
- - COALESCE(oac.active_count, 0),
- 0
- ) as owner_limit
- FROM ${this.queueName} q
- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_id
- LEFT JOIN owner_active_counts oac ON oac.owner_id = q.owner_id
- WHERE q.status = 'queued'::nuq.job_status
- AND (abs(hashtext(q.id::text)) % ${shardingCount}) = ${shardingIndex}
- ),
- selected_jobs_with_metadata AS (
- SELECT id, owner_id
- FROM owner_limited_jobs
- WHERE owner_rank <= owner_limit
- ),
- locked_jobs AS (
- SELECT q.id, q.owner_id
- FROM ${this.queueName} q
- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)
- AND q.status = 'queued'::nuq.job_status
- FOR UPDATE SKIP LOCKED
- ),
- missing_owners AS (
- SELECT DISTINCT owner_id
- FROM locked_jobs
- WHERE owner_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_owner_concurrency oc
- WHERE oc.id = owner_id
- )
- ),
- ensure_owner_rows AS (
- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)
- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)
- FROM missing_owners
- ON CONFLICT (id) DO NOTHING
- ),
- updated AS (
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM locked_jobs)
- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}
- )
- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;
- `;
- } else if (this.options.concurrencyLimit === "per-owner-per-group") {
- updateQuery = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- group_active_counts AS (
- SELECT
- group_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL
- GROUP BY group_id
- ),
- group_limited_jobs AS (
- SELECT
- q.id,
- q.owner_id,
- q.group_id,
- q.priority,
- q.created_at,
- ROW_NUMBER() OVER (
- PARTITION BY q.owner_id, q.group_id
- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC
- ) as group_rank,
- GREATEST(
- COALESCE(gc.max_concurrency, 999999)
- - COALESCE(gac.active_count, 0),
- 0
- ) as group_limit
- FROM ${this.queueName} q
- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_id
- LEFT JOIN group_active_counts gac ON gac.group_id = q.group_id
- WHERE q.status = 'queued'::nuq.job_status
- AND (abs(hashtext(q.id::text)) % ${shardingCount}) = ${shardingIndex}
- ),
- owner_limited_jobs AS (
- SELECT
- glj.id,
- glj.owner_id,
- glj.group_id,
- ROW_NUMBER() OVER (
- PARTITION BY glj.owner_id
- ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC
- ) as owner_rank,
- GREATEST(
- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id))
- - COALESCE(oac.active_count, 0),
- 0
- ) as owner_limit
- FROM group_limited_jobs glj
- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id
- LEFT JOIN owner_active_counts oac ON oac.owner_id = glj.owner_id
- WHERE glj.group_rank <= glj.group_limit
- ),
- selected_jobs_with_metadata AS (
- SELECT id, owner_id, group_id
- FROM owner_limited_jobs
- WHERE owner_rank <= owner_limit
- ),
- locked_jobs AS (
- SELECT q.id, q.owner_id, q.group_id
- FROM ${this.queueName} q
- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)
- AND q.status = 'queued'::nuq.job_status
- FOR UPDATE SKIP LOCKED
- ),
- missing_owners AS (
- SELECT DISTINCT owner_id
- FROM locked_jobs
- WHERE owner_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_owner_concurrency oc
- WHERE oc.id = owner_id
- )
- ),
- ensure_owner_rows AS (
- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)
- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)
- FROM missing_owners
- ON CONFLICT (id) DO NOTHING
- ),
- missing_groups AS (
- SELECT DISTINCT group_id
- FROM locked_jobs
- WHERE group_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_group_concurrency gc
- WHERE gc.id = group_id
- )
- ),
- ensure_group_rows AS (
- INSERT INTO ${this.queueName}_group_concurrency (id, max_concurrency)
- SELECT group_id, NULL
- FROM missing_groups
- ON CONFLICT (id) DO NOTHING
- ),
- updated AS (
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM locked_jobs)
- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}
- )
- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;
- `;
- } else {
- updateQuery = `
- WITH selected_jobs AS (
- SELECT j.id
- FROM ${this.queueName} j
- WHERE j.status = 'queued'::nuq.job_status
- ORDER BY j.priority ASC, j.created_at ASC
- FOR UPDATE SKIP LOCKED
- LIMIT 100
- )
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM selected_jobs)
- RETURNING ${this.jobReturning.join(", ")};
- `;
- }
-
- const result = await nuqPool.query(updateQuery);
-
- const jobs = result.rows.map(row => this.rowToJob(row)!);
+ const jobs = (
+ await nuqPool.query(
+ `
+ WITH next AS (SELECT id FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 500)
+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};
+ `,
+ )
+ ).rows.map(row => this.rowToJob(row)!);
for (const job of jobs) {
await this.sendJobPrefetch(
@@ -1170,198 +724,16 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
- let updateQuery: string;
- if (this.options.concurrencyLimit === "per-owner") {
- updateQuery = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- owner_limited_jobs AS (
- SELECT
- q.id,
- q.owner_id,
- ROW_NUMBER() OVER (
- PARTITION BY q.owner_id
- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC
- ) as owner_rank,
- GREATEST(
- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(q.owner_id))
- - COALESCE(oac.active_count, 0),
- 0
- ) as owner_limit
- FROM ${this.queueName} q
- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = q.owner_id
- LEFT JOIN owner_active_counts oac ON oac.owner_id = q.owner_id
- WHERE q.status = 'queued'::nuq.job_status
- ),
- selected_jobs_with_metadata AS (
- SELECT id, owner_id
- FROM owner_limited_jobs
- WHERE owner_rank <= owner_limit
- LIMIT 1
- ),
- locked_jobs AS (
- SELECT q.id, q.owner_id
- FROM ${this.queueName} q
- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)
- AND q.status = 'queued'::nuq.job_status
- FOR UPDATE SKIP LOCKED
- ),
- missing_owners AS (
- SELECT DISTINCT owner_id
- FROM locked_jobs
- WHERE owner_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_owner_concurrency oc
- WHERE oc.id = owner_id
- )
- ),
- ensure_owner_rows AS (
- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)
- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)
- FROM missing_owners
- ON CONFLICT (id) DO NOTHING
- ),
- updated AS (
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM locked_jobs)
- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}
- )
- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;
- `;
- } else if (this.options.concurrencyLimit === "per-owner-per-group") {
- updateQuery = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- group_active_counts AS (
- SELECT
- group_id,
- COUNT(*)::int8 as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL
- GROUP BY group_id
- ),
- group_limited_jobs AS (
- SELECT
- q.id,
- q.owner_id,
- q.group_id,
- q.priority,
- q.created_at,
- ROW_NUMBER() OVER (
- PARTITION BY q.owner_id, q.group_id
- ORDER BY q.priority ASC, q.created_at ASC, q.id ASC
- ) as group_rank,
- GREATEST(
- COALESCE(gc.max_concurrency, 999999)
- - COALESCE(gac.active_count, 0),
- 0
- ) as group_limit
- FROM ${this.queueName} q
- LEFT JOIN ${this.queueName}_group_concurrency gc ON gc.id = q.group_id
- LEFT JOIN group_active_counts gac ON gac.group_id = q.group_id
- WHERE q.status = 'queued'::nuq.job_status
- ),
- owner_limited_jobs AS (
- SELECT
- glj.id,
- glj.owner_id,
- glj.group_id,
- ROW_NUMBER() OVER (
- PARTITION BY glj.owner_id
- ORDER BY glj.priority ASC, glj.created_at ASC, glj.id ASC
- ) as owner_rank,
- GREATEST(
- COALESCE(oc.max_concurrency, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(glj.owner_id))
- - COALESCE(oac.active_count, 0),
- 0
- ) as owner_limit
- FROM group_limited_jobs glj
- LEFT JOIN ${this.queueName}_owner_concurrency oc ON oc.id = glj.owner_id
- LEFT JOIN owner_active_counts oac ON oac.owner_id = glj.owner_id
- WHERE glj.group_rank <= glj.group_limit
- ),
- selected_jobs_with_metadata AS (
- SELECT id, owner_id, group_id
- FROM owner_limited_jobs
- WHERE owner_rank <= owner_limit
- LIMIT 1
- ),
- locked_jobs AS (
- SELECT q.id, q.owner_id, q.group_id
- FROM ${this.queueName} q
- WHERE q.id IN (SELECT id FROM selected_jobs_with_metadata)
- AND q.status = 'queued'::nuq.job_status
- FOR UPDATE SKIP LOCKED
- ),
- missing_owners AS (
- SELECT DISTINCT owner_id
- FROM locked_jobs
- WHERE owner_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_owner_concurrency oc
- WHERE oc.id = owner_id
- )
- ),
- ensure_owner_rows AS (
- INSERT INTO ${this.queueName}_owner_concurrency (id, max_concurrency)
- SELECT owner_id, ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(owner_id)
- FROM missing_owners
- ON CONFLICT (id) DO NOTHING
- ),
- missing_groups AS (
- SELECT DISTINCT group_id
- FROM locked_jobs
- WHERE group_id IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM ${this.queueName}_group_concurrency gc
- WHERE gc.id = group_id
- )
- ),
- ensure_group_rows AS (
- INSERT INTO ${this.queueName}_group_concurrency (id, max_concurrency)
- SELECT group_id, NULL
- FROM missing_groups
- ON CONFLICT (id) DO NOTHING
- ),
- updated AS (
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM locked_jobs)
- RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}
- )
- SELECT ${this.jobReturning.map(x => `updated.${x}`).join(", ")} FROM updated;
- `;
- } else {
- updateQuery = `
- WITH selected_jobs AS (
- SELECT j.id
- FROM ${this.queueName} j
- WHERE j.status = 'queued'::nuq.job_status
- ORDER BY j.priority ASC, j.created_at ASC
- FOR UPDATE SKIP LOCKED
- LIMIT 1
+ return this.rowToJob(
+ (
+ await nuqPool.query(
+ `
+ WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1)
+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};
+ `,
)
- UPDATE ${this.queueName} q
- SET status = 'active'::nuq.job_status, lock = gen_random_uuid(), locked_at = now()
- WHERE q.id IN (SELECT id FROM selected_jobs)
- RETURNING ${this.jobReturning.join(", ")};
- `;
- }
-
- return this.rowToJob((await nuqPool.query(updateQuery)).rows[0])!;
+ ).rows[0],
+ )!;
} finally {
logger.info("nuqGetJobToProcess metrics", {
module: "nuq/metrics",
@@ -1410,36 +782,26 @@ class NuQ<JobData = any, JobReturnValue = any> {
const start = Date.now();
try {
- const updateQuery = `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;
-
- const result = await nuqPool.query(updateQuery, [
- id,
- lock,
- returnvalue,
- ]);
+ const result = await nuqPool.query(
+ `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,
+ [id, lock, returnvalue],
+ );
const success = result.rowCount !== 0;
if (success) {
const job = result.rows[0];
-
- if (job) {
- if (
- this.nuqWaitMode === "listen" &&
- !process.env.NUQ_RABBITMQ_URL
- ) {
- await nuqPool.query(
- `SELECT pg_notify('${this.queueName}', $1);`,
- [job.id + "|completed"],
- );
- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
- await this.sendJobEnd(
- job.id,
- "completed",
- job.listen_channel_id,
- _logger,
- );
- }
+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
+ job.id + "|completed",
+ ]);
+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
+ await this.sendJobEnd(
+ job.id,
+ "completed",
+ job.listen_channel_id,
+ _logger,
+ );
}
}
@@ -1478,36 +840,26 @@ class NuQ<JobData = any, JobReturnValue = any> {
const start = Date.now();
try {
- const updateQuery = `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;
-
- const result = await nuqPool.query(updateQuery, [
- id,
- lock,
- failedReason,
- ]);
+ const result = await nuqPool.query(
+ `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`,
+ [id, lock, failedReason],
+ );
const success = result.rowCount !== 0;
if (success) {
const job = result.rows[0];
-
- if (job) {
- if (
- this.nuqWaitMode === "listen" &&
- !process.env.NUQ_RABBITMQ_URL
- ) {
- await nuqPool.query(
- `SELECT pg_notify('${this.queueName}', $1);`,
- [job.id + "|failed"],
- );
- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
- await this.sendJobEnd(
- job.id,
- "failed",
- job.listen_channel_id,
- _logger,
- );
- }
+ if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
+ await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
+ job.id + "|failed",
+ ]);
+ } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
+ await this.sendJobEnd(
+ job.id,
+ "failed",
+ job.listen_channel_id,
+ _logger,
+ );
}
}
@@ -1531,223 +883,12 @@ class NuQ<JobData = any, JobReturnValue = any> {
});
}
- public async getOwnerJobCounts(
- ownerId: string,
- _logger: Logger = logger,
- ): Promise<{ active: number; queued: number }> {
- const start = Date.now();
- try {
- const result = await nuqPool.query(
- `SELECT
- COALESCE(SUM(CASE WHEN status = 'active'::nuq.job_status THEN 1 ELSE 0 END), 0)::int as active,
- COALESCE(SUM(CASE WHEN status = 'queued'::nuq.job_status THEN 1 ELSE 0 END), 0)::int as queued
- FROM ${this.queueName}
- WHERE owner_id = $1;`,
- [ownerId],
- );
-
- return {
- active: result.rows[0]?.active ?? 0,
- queued: result.rows[0]?.queued ?? 0,
- };
- } finally {
- _logger.info("nuqGetOwnerJobCounts metrics", {
- module: "nuq/metrics",
- method: "nuqGetOwnerJobCounts",
- duration: Date.now() - start,
- ownerId,
- });
- }
- }
-
- public async getOwnerConcurrency(
- ownerId: string,
- _logger: Logger = logger,
- ): Promise<{
- currentConcurrency: number;
- maxConcurrency: number;
- } | null> {
- const start = Date.now();
- try {
- const result = await nuqPool.query(
- `SELECT
- (SELECT COUNT(*)::int8 FROM ${this.queueName} WHERE owner_id = $1 AND status = 'active'::nuq.job_status) as current_concurrency,
- max_concurrency
- FROM ${this.queueName}_owner_concurrency
- WHERE id = $1;`,
- [ownerId],
- );
-
- if (result.rows.length === 0) {
- return null;
- }
-
- return {
- currentConcurrency: result.rows[0].current_concurrency,
- maxConcurrency: result.rows[0].max_concurrency,
- };
- } finally {
- _logger.info("nuqGetOwnerConcurrency metrics", {
- module: "nuq/metrics",
- method: "nuqGetOwnerConcurrency",
- duration: Date.now() - start,
- ownerId,
- });
- }
- }
-
// === Metrics
public async getMetrics(): Promise<string> {
const start = Date.now();
-
- let query: string;
- if (this.options.concurrencyLimit === "per-owner") {
- query = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*) as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- owner_status AS (
- SELECT
- oc.id,
- GREATEST(0, oc.max_concurrency - COALESCE(oac.active_count, 0)) as available_slots
- FROM ${this.queueName}_owner_concurrency oc
- LEFT JOIN owner_active_counts oac ON oac.owner_id = oc.id
- ),
- queued_per_owner AS (
- SELECT
- owner_id,
- COUNT(*) as queued_count
- FROM ${this.queueName}
- WHERE status = 'queued'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- owner_breakdown AS (
- SELECT
- LEAST(qpo.queued_count, COALESCE(os.available_slots, 999999)) as can_run,
- GREATEST(0, qpo.queued_count - COALESCE(os.available_slots, 999999)) as blocked
- FROM queued_per_owner qpo
- LEFT JOIN owner_status os ON qpo.owner_id = os.id
- ),
- non_queued AS (
- SELECT status::text, COUNT(*) as count
- FROM ${this.queueName}
- WHERE status != 'queued'::nuq.job_status
- GROUP BY status
- ),
- totals AS (
- SELECT
- COALESCE(SUM(can_run), 0) as total_queued,
- COALESCE(SUM(blocked), 0) as total_limited
- FROM owner_breakdown
- )
- SELECT status, count FROM non_queued
- UNION ALL
- SELECT 'queued' as status, total_queued::bigint as count
- FROM totals
- WHERE total_queued > 0
- UNION ALL
- SELECT 'concurrency-limited' as status, total_limited::bigint as count
- FROM totals
- WHERE total_limited > 0
- ORDER BY count DESC;
- `;
- } else if (this.options.concurrencyLimit === "per-owner-per-group") {
- query = `
- WITH owner_active_counts AS (
- SELECT
- owner_id,
- COUNT(*) as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND owner_id IS NOT NULL
- GROUP BY owner_id
- ),
- group_active_counts AS (
- SELECT
- group_id,
- COUNT(*) as active_count
- FROM ${this.queueName}
- WHERE status = 'active'::nuq.job_status AND group_id IS NOT NULL
- GROUP BY group_id
- ),
- owner_status AS (
- SELECT
- oc.id,
- GREATEST(0, oc.max_concurrency - COALESCE(oac.active_count, 0)) as available_slots
- FROM ${this.queueName}_owner_concurrency oc
- LEFT JOIN owner_active_counts oac ON oac.owner_id = oc.id
- ),
- group_status AS (
- SELECT
- gc.id,
- GREATEST(0, gc.max_concurrency - COALESCE(gac.active_count, 0)) as available_slots
- FROM ${this.queueName}_group_concurrency gc
- LEFT JOIN group_active_counts gac ON gac.group_id = gc.id
- ),
- queued_per_owner_group AS (
- SELECT
- owner_id,
- group_id,
- COUNT(*) as queued_count
- FROM ${this.queueName}
- WHERE status = 'queued'::nuq.job_status
- GROUP BY owner_id, group_id
- ),
- group_capacity AS (
- SELECT
- qpog.owner_id,
- qpog.group_id,
- qpog.queued_count,
- LEAST(qpog.queued_count, COALESCE(gs.available_slots, 999999)) as group_can_run,
- GREATEST(0, qpog.queued_count - COALESCE(gs.available_slots, 999999)) as group_blocked
- FROM queued_per_owner_group qpog
- LEFT JOIN group_status gs ON qpog.group_id = gs.id
- ),
- owner_breakdown AS (
- SELECT
- gc.owner_id,
- SUM(gc.group_can_run) as total_demand,
- COALESCE(os.available_slots, 999999) as owner_capacity,
- LEAST(SUM(gc.group_can_run), COALESCE(os.available_slots, 999999)) as owner_can_run,
- GREATEST(0, SUM(gc.group_can_run) - COALESCE(os.available_slots, 999999)) as owner_blocked,
- SUM(gc.group_blocked) as group_blocked
- FROM group_capacity gc
- LEFT JOIN owner_status os ON gc.owner_id = os.id
- GROUP BY gc.owner_id, os.available_slots
- ),
- non_queued AS (
- SELECT status::text, COUNT(*) as count
- FROM ${this.queueName}
- WHERE status != 'queued'::nuq.job_status
- GROUP BY status
- ),
- totals AS (
- SELECT
- COALESCE(SUM(owner_can_run), 0) as total_queued,
- COALESCE(SUM(owner_blocked + group_blocked), 0) as total_limited
- FROM owner_breakdown
- )
- SELECT status, count FROM non_queued
- UNION ALL
- SELECT 'queued' as status, total_queued::bigint as count
- FROM totals
- WHERE total_queued > 0
- UNION ALL
- SELECT 'concurrency-limited' as status, total_limited::bigint as count
- FROM totals
- WHERE total_limited > 0
- ORDER BY count DESC;
- `;
- } else {
- query = `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`;
- }
-
- const result = await nuqPool.query(query);
-
+ const result = await nuqPool.query(
+ `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,
+ );
logger.info("nuqGetMetrics metrics", {
module: "nuq/metrics",
method: "nuqGetMetrics",
@@ -1782,139 +923,6 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
-// === Group
-
-type NuQGroupOptions = {
- memberQueues: NuQ[];
- finishQueue?: NuQ;
- groupTTL: number;
-};
-
-type NuQGroupConcurrencySettings = {
- queue: NuQ;
- maxConcurrency?: number;
-};
-
-type NuQGroupInstanceOptions = {
- concurrency: NuQGroupConcurrencySettings[];
- ownerId: string;
-};
-
-type NuQGroupInstance = {
- id: string;
- status: "active" | "completed";
- createdAt: Date;
- finishedAt?: Date;
- expiresAt?: Date;
- ownerId?: string;
-};
-
-class NuQGroup {
- constructor(
- public readonly groupName: string,
- public readonly options: NuQGroupOptions,
- ) {}
-
- private groupReturning = [
- "id",
- "status",
- "created_at",
- "finished_at",
- "expires_at",
- "owner_id",
- ];
-
- private rowToGroup(row: any): NuQGroupInstance | null {
- if (!row) return null;
- return {
- id: row.id,
- status: row.status,
- createdAt: new Date(row.created_at),
- finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
- expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
- ownerId: row.owner_id ?? undefined,
- };
- }
-
- public async addGroup(
- id: string,
- options: NuQGroupInstanceOptions,
- ): Promise<NuQGroupInstance> {
- const client = await nuqPool.connect();
-
- await client.query("BEGIN");
-
- try {
- const insert = await client.query(
- `INSERT INTO ${this.groupName} (id, owner_id) VALUES ($1, $2) RETURNING ${this.groupReturning.join(", ")};`,
- [id, normalizeOwnerId(options.ownerId) ?? null],
- );
-
- for (const entry of options.concurrency) {
- if (entry.queue.options.concurrencyLimit === "per-owner-per-group") {
- await client.query(
- `INSERT INTO ${entry.queue.queueName}_group_concurrency (id, max_concurrency) VALUES ($1, $2);`,
- [id, entry.maxConcurrency ?? null],
- );
- }
- }
-
- await client.query("COMMIT");
- client.release();
-
- return this.rowToGroup(insert.rows[0])!;
- } catch (e) {
- await client.query("ROLLBACK");
- client.release(e);
- throw e;
- }
- }
-
- public async getGroup(id: string): Promise<NuQGroupInstance | null> {
- return this.rowToGroup(
- (
- await nuqPool.query(
- `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE id = $1 LIMIT 1;`,
- [id],
- )
- ).rows[0],
- );
- }
-
- public async cancelGroup(id: string): Promise<boolean> {
- const client = await nuqPool.connect();
-
- await client.query("BEGIN");
-
- try {
- const updateOp = await client.query(
- `UPDATE ${this.groupName} SET status = 'cancelled'::nuq.group_status WHERE id = $1 AND status = 'active'::nuq.group_status`,
- [id],
- );
-
- if (updateOp.rowCount === 0) {
- client.release();
- return false;
- }
-
- for (const queue of this.options.memberQueues) {
- await client.query(
- `UPDATE ${queue.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = 'CANCELLED' WHERE group_id = $1 AND status = 'queued'::nuq.job_status`,
- [id],
- );
- }
-
- client.release();
- return true;
- } catch (e) {
- client.release(e);
- throw e;
- }
- }
-}
-
-// === Utilities
-
export function nuqGetLocalMetrics(): string {
return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n
# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n
@@ -1936,16 +944,7 @@ export async function nuqHealthCheck(): Promise<boolean> {
// === Instances
-export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {
- concurrencyLimit: "per-owner-per-group",
-});
-// export const crawlFinishQueue = new NuQ("nuq.queue_crawl_finish");
-
-export const crawlGroup = new NuQGroup("nuq.group_crawl", {
- memberQueues: [scrapeQueue],
- // finishQueue: crawlFinishQueue,
- groupTTL: 24 * 60 * 60,
-});
+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");
// === Cleanup
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index f86dfef41..14b888a40 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -4,6 +4,10 @@ import http from "http";
import https from "https";
import { logger as _logger } from "../../lib/logger";
+import {
+ concurrentJobDone,
+ pushConcurrencyLimitActiveJob,
+} from "../../lib/concurrency-limit";
import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";
import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";
import { v4 as uuidv4 } from "uuid";
@@ -23,13 +27,17 @@ import {
StoredCrawl,
} from "../../lib/crawl-redis";
import { redisEvictConnection } from "../redis";
-import { addScrapeJob, addScrapeJobs } from "../queue-jobs";
+import {
+ _addScrapeJobToBullMQ,
+ addScrapeJob,
+ addScrapeJobs,
+} from "../queue-jobs";
import psl from "psl";
import { getJobPriority } from "../../lib/job-priority";
import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";
import { hasFormatOfType } from "../../lib/format-utils";
import { getACUCTeam } from "../../controllers/auth";
-import { createWebhookSender, WebhookEvent } from "../webhook/index";
+import { createWebhookSender, WebhookEvent } from "../webhook";
import { CustomError } from "../../lib/custom-error";
import { startWebScraperPipeline } from "../../main/runWebScraper";
import { CostTracking } from "../../lib/cost-tracking";
@@ -330,7 +338,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
// Store robots blocked URLs in Redis set
for (const [url, reason] of links.denialReasons) {
if (reason === "URL blocked by robots.txt") {
- await recordRobotsBlocked(job.data.crawl_id, url);
+ await recordRobotsBlocked(
+ job.data.crawl_id,
+ url
+ );
}
}
@@ -541,12 +552,15 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
error instanceof Error &&
error.message === "URL blocked by robots.txt"
) {
- await recordRobotsBlocked(job.data.crawl_id, job.data.url);
+ await recordRobotsBlocked(
+ job.data.crawl_id,
+ job.data.url,
+ );
}
} catch (e) {
logger.debug("Failed to record top-level robots block", { e });
}
-
+
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
@@ -750,7 +764,7 @@ async function addKickoffSitemapJob(
}
const jobId = uuidv4();
- await addScrapeJob(
+ await _addScrapeJobToBullMQ(
{
mode: "kickoff_sitemap" as const,
team_id: sourceJob.data.team_id,
@@ -1097,55 +1111,75 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
try {
- await addJobPriority(job.data.team_id, job.id);
try {
- if (job.data.mode === "kickoff") {
- const result = await processKickoffJob(job as NuQJob<ScrapeJobKickoff>);
- if (result.success) {
- return null;
- } else {
- throw (result as any).error;
- }
- } else if (job.data.mode === "kickoff_sitemap") {
- const result = await processKickoffSitemapJob(
- job as NuQJob<ScrapeJobKickoffSitemap>,
- );
- if (result.success) {
- return null;
+ let extendLockInterval: NodeJS.Timeout | null = null;
+ if (job.data?.mode !== "kickoff" && job.data?.team_id) {
+ extendLockInterval = setInterval(async () => {
+ await pushConcurrencyLimitActiveJob(
+ job.data.team_id,
+ job.id,
+ 60 * 1000,
+ ); // 60s lock renew, just like in the queue
+ }, jobLockExtendInterval);
+ }
+
+ await addJobPriority(job.data.team_id, job.id);
+ try {
+ if (job.data.mode === "kickoff") {
+ const result = await processKickoffJob(
+ job as NuQJob<ScrapeJobKickoff>,
+ );
+ if (result.success) {
+ return null;
+ } else {
+ throw (result as any).error;
+ }
+ } else if (job.data.mode === "kickoff_sitemap") {
+ const result = await processKickoffSitemapJob(
+ job as NuQJob<ScrapeJobKickoffSitemap>,
+ );
+ if (result.success) {
+ return null;
+ } else {
+ throw (result as any).error;
+ }
} else {
- throw (result as any).error;
- }
- } else {
- const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);
- if (result.success) {
- try {
- if (job.data.team_id) {
- await redisEvictConnection.set(
- "most-recent-success:" + job.data.team_id,
- new Date().toISOString(),
- "EX",
- 60 * 60 * 24,
- );
+ const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);
+ if (result.success) {
+ try {
+ if (job.data.team_id) {
+ await redisEvictConnection.set(
+ "most-recent-success:" + job.data.team_id,
+ new Date().toISOString(),
+ "EX",
+ 60 * 60 * 24,
+ );
+ }
+ } catch (e) {
+ logger.warn("Failed to set most recent success", { error: e });
}
- } catch (e) {
- logger.warn("Failed to set most recent success", { error: e });
- }
- try {
- if (process.env.GCS_BUCKET_NAME) {
- logger.debug("Job succeeded -- putting null in Redis");
- return null;
- } else {
- logger.debug("Job succeeded -- putting result in Redis");
- return result.document;
- }
- } catch (e) {}
- } else {
- throw (result as any).error;
+ try {
+ if (process.env.GCS_BUCKET_NAME) {
+ logger.debug("Job succeeded -- putting null in Redis");
+ return null;
+ } else {
+ logger.debug("Job succeeded -- putting result in Redis");
+ return result.document;
+ }
+ } catch (e) {}
+ } else {
+ throw (result as any).error;
+ }
+ }
+ } finally {
+ await deleteJobPriority(job.data.team_id, job.id);
+ if (extendLockInterval) {
+ clearInterval(extendLockInterval);
}
}
} finally {
- await deleteJobPriority(job.data.team_id, job.id);
+ await concurrentJobDone(job);
}
} catch (error) {
logger.debug("Job failed", { error });
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 1697a1f8c..f22a6b83a 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -9,12 +9,6 @@ EXCEPTION
WHEN duplicate_object THEN null;
END $$;
-DO $$ BEGIN
- CREATE TYPE nuq.group_status AS ENUM ('active', 'completed', 'cancelled');
-EXCEPTION
- WHEN duplicate_object THEN null;
-END $$;
-
CREATE TABLE IF NOT EXISTS nuq.queue_scrape (
id uuid NOT NULL DEFAULT gen_random_uuid(),
status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,
@@ -28,9 +22,6 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (
listen_channel_id text, -- for listenable jobs over rabbitmq
returnvalue jsonb, -- only for selfhost
failedreason text, -- only for selfhost
- owner_id uuid,
- group_id uuid,
- times_out_at timestamp with time zone,
CONSTRAINT queue_scrape_pkey PRIMARY KEY (id)
);
@@ -44,146 +35,21 @@ CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_idx ON nuq.queue_scrape (owner_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status);
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_group_idx ON nuq.queue_scrape (owner_id, group_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status);
-
--- Performance indexes for prefetchJobs query optimization
--- This index dramatically speeds up the "SELECT DISTINCT owner_id, group_id" query in the queued_combinations CTE
--- by allowing an index-only scan instead of a sequential scan of all queued jobs
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_group_distinct_idx ON nuq.queue_scrape (owner_id, group_id) WHERE (status = 'queued'::nuq.job_status);
-
--- This index helps with group-specific queries and provides better coverage for per-group capacity lookups
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_group_priority_idx ON nuq.queue_scrape (group_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status AND group_id IS NOT NULL);
-
--- This index helps with owner-specific queries when group_id is NULL
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_owner_no_group_idx ON nuq.queue_scrape (owner_id, priority ASC, created_at ASC) WHERE (status = 'queued'::nuq.job_status AND group_id IS NULL);
-
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_complete_idx ON nuq.queue_scrape (owner_id, group_id, priority ASC, created_at ASC, id ASC) WHERE (status = 'queued'::nuq.job_status);
-
--- Indexes for dynamic concurrency counting (replacing current_concurrency counters)
--- This allows fast COUNT(*) queries for active jobs per owner/group without maintaining counters
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_owner_idx ON nuq.queue_scrape (owner_id) WHERE (status = 'active'::nuq.job_status AND owner_id IS NOT NULL);
-CREATE INDEX IF NOT EXISTS nuq_queue_scrape_active_group_idx ON nuq.queue_scrape (group_id) WHERE (status = 'active'::nuq.job_status AND group_id IS NOT NULL);
-CREATE INDEX IF NOT EXISTS idx_queue_scrape_prefetch_composite ON queue_scrape (status, owner_id, group_id, priority, created_at, id) WHERE status = 'queued';
-
-CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (
- id uuid NOT NULL,
- max_concurrency int8 NOT NULL,
- CONSTRAINT queue_scrape_owner_concurrency_pkey PRIMARY KEY (id)
-);
-
--- fake concurrency limit source for tests
-CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency_source (
- id uuid NOT NULL,
- max_concurrency int8 NOT NULL,
- CONSTRAINT queue_scrape_owner_concurrency_source_pkey PRIMARY KEY (id)
-);
-
-CREATE OR REPLACE FUNCTION nuq_queue_scrape_owner_resolve_max_concurrency(owner_id uuid)
-RETURNS int8
-LANGUAGE sql
-STABLE
-AS $$
- SELECT COALESCE((SELECT max_concurrency FROM nuq.queue_scrape_owner_concurrency_source WHERE id = owner_id LIMIT 1), 100)::int8;
-$$;
-
-CREATE TABLE IF NOT EXISTS nuq.queue_scrape_group_concurrency (
- id uuid NOT NULL,
- max_concurrency int8,
- CONSTRAINT queue_scrape_group_concurrency_pkey PRIMARY KEY (id)
-);
SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$
- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.group_id IS NULL AND nuq.queue_scrape.created_at < now() - interval '1 hour';
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';
$$);
SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$
- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.group_id IS NULL AND nuq.queue_scrape.created_at < now() - interval '6 hours';
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';
$$);
SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
- WITH stalled_jobs AS (
- SELECT id, owner_id, group_id, stalls
- FROM nuq.queue_scrape
- WHERE locked_at <= now() - interval '1 minute'
- AND status = 'active'::nuq.job_status
- ),
- requeued AS (
- UPDATE nuq.queue_scrape
- SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1
- WHERE id IN (
- SELECT sj.id
- FROM stalled_jobs sj
- WHERE COALESCE(sj.stalls, 0) < 9
- )
- RETURNING id
- ),
- stallfail AS (
- UPDATE nuq.queue_scrape
- SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1
- WHERE id IN (
- SELECT sj.id
- FROM stalled_jobs sj
- WHERE COALESCE(sj.stalls, 0) >= 9
- )
- RETURNING id
- )
+ UPDATE nuq.queue_scrape SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) < 9;
+ WITH stallfail AS (UPDATE nuq.queue_scrape SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9 RETURNING id)
SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;
$$);
SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$
REINDEX TABLE CONCURRENTLY nuq.queue_scrape;
$$);
-
-SELECT cron.schedule('nuq_queue_scrape_max_concurrency_sync', '*/5 * * * *', $$
- UPDATE nuq.queue_scrape_owner_concurrency
- SET max_concurrency = (SELECT nuq_queue_scrape_owner_resolve_max_concurrency(nuq.queue_scrape_owner_concurrency.id));
-$$);
-
-SELECT cron.schedule('nuq_queue_scrape_timeout', '* * * * *', $$
- UPDATE nuq.queue_scrape
- SET status = 'failed', finished_at = now(), failedreason = 'SCRAPE_TIMEOUT|{"stack":"Error: Scrape timed out\n in DB","message":"Scrape timed out"}'
- WHERE status = 'queued' AND times_out_at < now();
-$$);
-
-CREATE TABLE IF NOT EXISTS nuq.group_crawl (
- id uuid NOT NULL DEFAULT gen_random_uuid(),
- status nuq.group_status NOT NULL DEFAULT 'active'::nuq.group_status,
- created_at timestamp with time zone NOT NULL DEFAULT now(),
- finished_at timestamp with time zone,
- expires_at timestamp with time zone,
- owner_id uuid,
- CONSTRAINT group_crawl_pkey PRIMARY KEY (id)
-);
-
-CREATE OR REPLACE FUNCTION nuq_queue_scrape_check_group_completion()
-RETURNS TRIGGER
-LANGUAGE plpgsql
-AS $$
-BEGIN
- IF NEW.group_id IS NOT NULL THEN
- UPDATE nuq.group_crawl
- SET status = 'completed'::nuq.group_status,
- finished_at = now(),
- expires_at = now() + interval '24 hours'
- WHERE id = NEW.group_id
- AND status != 'completed'::nuq.group_status
- AND NOT EXISTS (
- SELECT 1
- FROM nuq.queue_scrape
- WHERE group_id = NEW.group_id
- AND status NOT IN ('completed'::nuq.job_status, 'failed'::nuq.job_status)
- );
- END IF;
-
- RETURN NEW;
-END;
-$$;
-
--- Trigger to automatically mark groups as completed
-CREATE OR REPLACE TRIGGER nuq_queue_scrape_group_completion_trigger
-AFTER UPDATE OF status ON nuq.queue_scrape
-FOR EACH ROW
-WHEN (NEW.status IN ('completed'::nuq.job_status, 'failed'::nuq.job_status)
- AND OLD.status NOT IN ('completed'::nuq.job_status, 'failed'::nuq.job_status))
-EXECUTE FUNCTION nuq_queue_scrape_check_group_completion();