Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): concurrency queue reconciler (#2929)
tomsideguide1b3494f181a2
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 76e29e4e3..38715d488 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -27,6 +27,8 @@"nuq-worker:production": "node dist/src/services/worker/nuq-worker.js","nuq-prefetch-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-prefetch-worker.js\"","nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js",+ "nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"",+ "nuq-reconciler-worker:production": "node dist/src/services/worker/nuq-reconciler-worker.js","extract-worker": "tsc-watch --onSuccess \"node dist/src/services/extract-worker.js\"","extract-worker:production": "node dist/src/services/extract-worker.js","index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 0f1cbc167..88c5e5dcf 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -121,6 +121,7 @@ const configSchema = z.object({NUQ_WORKER_START_PORT: z.coerce.number().default(3006),NUQ_WORKER_COUNT: z.coerce.number().default(5),NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011).catch(3011), // todo: investigate why .catch is needed+ NUQ_RECONCILER_WORKER_PORT: z.coerce.number().default(3012).catch(3012),EXTRACT_WORKER_PORT: z.coerce.number().default(3004),NUQ_WAIT_MODE: z.string().optional(),diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.tsindex 3f941aa52..ee810a7b0 100644--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts@@ -1,14 +1,6 @@import { logger as _logger } from "../../../lib/logger";import { Request, Response } from "express";-import { getRedisConnection } from "../../../services/queue-service";-import { scrapeQueue } from "../../../services/worker/nuq";-import {- pushConcurrencyLimitedJobs,- pushConcurrencyLimitActiveJob,- getConcurrencyLimitActiveJobsCount,-} from "../../../lib/concurrency-limit";-import { RateLimiterMode } from "../../../types";-import { getACUCTeam } from "../../auth";+import { reconcileConcurrencyQueue } from "../../../lib/concurrency-queue-reconciler";export async function concurrencyQueueBackfillController(req: Request,@@ -20,157 +12,16 @@ export async function concurrencyQueueBackfillController(logger.info("Starting concurrency queue backfill");- // Skip the precrawl team β it has millions of backlogged jobs that cause- // this endpoint to time out before it can process any other teams.- const IGNORED_TEAM_IDS = new Set([- "86100d9a-54ec-43ab-8f90-ba5852aede34", // Precrawl team- ]);-- const backloggedOwnerIDs = (- req.query.teamId- ? [req.query.teamId as string]- : await scrapeQueue.getBackloggedOwnerIDs(logger)- ).filter(id => !IGNORED_TEAM_IDS.has(id));-- const teamResults: {- teamId: string;- backloggedJobs: number;- alreadyQueued: number;- queued: number;- started: number;- }[] = [];-- for (const ownerId of backloggedOwnerIDs) {- logger.info("Backfilling concurrency queue for team", { teamId: ownerId });-- try {- const backloggedJobIDs = new Set(- await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),- );- const queuedJobIDs = new Set<string>();-- let cursor = "0";-- do {- const result = await getRedisConnection().zscan(- `concurrency-limit-queue:${ownerId}`,- cursor,- "COUNT",- 1000,- );- cursor = result[0];- const results = result[1];-- // zscan returns [member1, score1, member2, score2, ...]- // Only parse members (even indices), skip scores (odd indices)- for (let i = 0; i < results.length; i += 2) {- queuedJobIDs.add(results[i]);- }- } while (cursor !== "0");-- const jobIDsToAdd = new Set(- [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),- );-- logger.info("Team statistics", {- teamId: ownerId,- backloggedJobIDs: backloggedJobIDs.size,- queuedJobIDs: queuedJobIDs.size,- jobIDsToAdd: jobIDsToAdd.size,- });-- const jobsToAdd = await scrapeQueue.getJobsFromBacklog(- Array.from(jobIDsToAdd),- logger,- );-- // Get concurrency limit (only crawl β extract ACUC can overflow integer for high-usage teams)- const crawlACUC = await getACUCTeam(- ownerId,- false,- true,- RateLimiterMode.Crawl,- );- const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;- const maxExtractConcurrency = crawlACUC?.concurrency ?? 2;-- const currentActiveConcurrency =- await getConcurrencyLimitActiveJobsCount(ownerId);-- const jobsToStart: typeof jobsToAdd = [];- const jobsToQueue: typeof jobsToAdd = [];-- let activeCount = currentActiveConcurrency;- for (const job of jobsToAdd) {- const isExtract = "is_extract" in job.data && job.data.is_extract;- const limit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;-- if (activeCount < limit) {- jobsToStart.push(job);- activeCount++;- } else {- jobsToQueue.push(job);- }- }-- if (jobsToQueue.length > 0) {- await pushConcurrencyLimitedJobs(- ownerId,- jobsToQueue.map(job => ({- job: {- id: job.id,- data: job.data,- priority: job.priority,- listenable: job.listenChannelId !== undefined,- },- timeout: Infinity,- })),- );- }-- // Promote jobs that can start immediately- // These involve DB transactions per job so they remain sequential- for (const job of jobsToStart) {- await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {- priority: job.priority,- listenable: job.listenChannelId !== undefined,- ownerId: job.data.team_id ?? undefined,- groupId: job.data.crawl_id ?? undefined,- });-- await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);- }-- logger.info("Finished backfilling concurrency queue for team", {- teamId: ownerId,- startedCount: jobsToStart.length,- queuedCount: jobsToQueue.length,- });-- teamResults.push({- teamId: ownerId,- backloggedJobs: backloggedJobIDs.size,- alreadyQueued: queuedJobIDs.size,- queued: jobsToQueue.length,- started: jobsToStart.length,- });- } catch (e) {- logger.error("Failed to backfill team, skipping", {- teamId: ownerId,- error: e,- });- }- }-- logger.info("Finished backfilling all teams", {- teamsProcessed: teamResults.length,+ const teamId =+ typeof req.query.teamId === "string"+ ? (req.query.teamId as string)+ : undefined;+ const summary = await reconcileConcurrencyQueue({+ teamId,+ logger,});- res.json({- ok: true,- teamsProcessed: teamResults.length,- totalJobsQueued: teamResults.reduce((sum, t) => sum + t.queued, 0),- totalJobsStarted: teamResults.reduce((sum, t) => sum + t.started, 0),- teams: teamResults,- });+ logger.info("Finished backfilling all teams", summary);++ res.json({ ok: true, ...summary });}diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 71dda3794..1deef110a 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -34,6 +34,7 @@ interface Services {worker?: ProcessResult;nuqWorkers: ProcessResult[];nuqPrefetchWorker?: ProcessResult;+ nuqReconcilerWorker?: ProcessResult;extractWorker?: ProcessResult;indexWorker?: ProcessResult;command?: ProcessResult;@@ -101,6 +102,7 @@ const EXTRACT_WORKER_PORT = config.EXTRACT_WORKER_PORT;const NUQ_WORKER_START_PORT = config.NUQ_WORKER_START_PORT;const NUQ_WORKER_COUNT = config.NUQ_WORKER_COUNT;const NUQ_PREFETCH_WORKER_PORT = NUQ_WORKER_START_PORT + NUQ_WORKER_COUNT;+const NUQ_RECONCILER_WORKER_PORT = NUQ_PREFETCH_WORKER_PORT + 1;// PostgreSQL credentials (with defaults for backward compatibility)const POSTGRES_USER = config.POSTGRES_USER;@@ -779,6 +781,18 @@ async function startServices(command?: string[]): Promise<Services> {},);+ const nuqReconcilerWorker = execForward(+ "nuq-reconciler",+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/worker/nuq-reconciler-worker.js"+ : "pnpm nuq-reconciler-worker:production",+ {+ NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "nuq-reconciler-worker-0",+ },+ );+const indexWorker = config.USE_DB_AUTHENTICATION? execForward("index-worker",@@ -813,6 +827,7 @@ async function startServices(command?: string[]): Promise<Services> {worker,nuqWorkers,nuqPrefetchWorker,+ nuqReconcilerWorker,indexWorker,extractWorker,command: commandProcess,@@ -829,6 +844,7 @@ async function stopDevelopmentServices(services: Services) {services.worker?.process,...services.nuqWorkers.map(w => w.process),services.nuqPrefetchWorker?.process,+ services.nuqReconcilerWorker?.process,services.indexWorker?.process,services.extractWorker?.process,services.command?.process,@@ -992,6 +1008,8 @@ async function waitForTermination(services: Services): Promise<void> {if (services.extractWorker) promises.push(services.extractWorker.promise);if (services.nuqPrefetchWorker)promises.push(services.nuqPrefetchWorker.promise);+ if (services.nuqReconcilerWorker)+ promises.push(services.nuqReconcilerWorker.promise);promises.push(...services.nuqWorkers.map(w => w.promise));diff --git a/apps/api/src/lib/concurrency-queue-reconciler.ts b/apps/api/src/lib/concurrency-queue-reconciler.tsnew file mode 100644index 000000000..8c3975116--- /dev/null+++ b/apps/api/src/lib/concurrency-queue-reconciler.ts@@ -0,0 +1,232 @@+import { Logger } from "winston";+import { getACUCTeam } from "../controllers/auth";+import { getRedisConnection } from "../services/queue-service";+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";+import { RateLimiterMode, type ScrapeJobData } from "../types";+import {+ getConcurrencyLimitActiveJobs,+ pushConcurrencyLimitActiveJob,+ pushConcurrencyLimitedJob,+ pushCrawlConcurrencyLimitActiveJob,+} from "./concurrency-limit";+import { getCrawl } from "./crawl-redis";+import { logger as _logger } from "./logger";++interface ReconcileOptions {+ teamId?: string;+ logger?: Logger;+}++interface ReconcileResult {+ teamsScanned: number;+ teamsWithDrift: number;+ jobsRequeued: number;+ jobsStarted: number;+}++function isExtractJob(data: ScrapeJobData): boolean {+ return "is_extract" in data && !!data.is_extract;+}++function getBacklogJobTimeout(jobData: ScrapeJobData): number {+ if (jobData.crawl_id) return Infinity;++ if ("scrapeOptions" in jobData && jobData.scrapeOptions?.timeout)+ return jobData.scrapeOptions.timeout;++ return 60 * 1000;+}++async function requeueJob(+ ownerId: string,+ job: NuQJob<ScrapeJobData>,+): Promise<void> {+ await pushConcurrencyLimitedJob(+ ownerId,+ {+ id: job.id,+ data: job.data,+ priority: job.priority,+ listenable: job.listenChannelId !== undefined,+ },+ getBacklogJobTimeout(job.data),+ );+}++async function getQueuedJobIDs(teamId: string): Promise<Set<string>> {+ const queuedJobIDs = new Set<string>();+ let cursor = "0";++ do {+ const [nextCursor, results] = await getRedisConnection().zscan(+ `concurrency-limit-queue:${teamId}`,+ cursor,+ "COUNT",+ 100,+ );+ cursor = nextCursor;++ // zscan returns [member1, score1, member2, score2, ...]+ for (let i = 0; i < results.length; i += 2) {+ queuedJobIDs.add(results[i]);+ }+ } while (cursor !== "0");++ return queuedJobIDs;+}++async function reconcileTeam(+ ownerId: string,+ teamLogger: Logger,+): Promise<{ jobsStarted: number; jobsRequeued: number } | null> {+ const backloggedJobIDs = new Set(+ await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, teamLogger),+ );+ if (backloggedJobIDs.size === 0) {+ return null;+ }++ const queuedJobIDs = await getQueuedJobIDs(ownerId);+ const missingJobIDs = [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x));++ if (missingJobIDs.length === 0) {+ return null;+ }++ const jobsToRecover = await scrapeQueue.getJobsFromBacklog(+ missingJobIDs,+ teamLogger,+ );+ if (jobsToRecover.length === 0) {+ return null;+ }++ const maxCrawlConcurrency =+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))+ ?.concurrency ?? 2;+ const maxExtractConcurrency =+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))+ ?.concurrency ?? 2;++ // Split active count by type so one type's active jobs don't gate the other+ const activeJobIds = await getConcurrencyLimitActiveJobs(ownerId);+ const activeJobs = await scrapeQueue.getJobs(activeJobIds, teamLogger);+ let activeCrawlCount = 0;+ let activeExtractCount = 0;+ for (const aj of activeJobs) {+ if (isExtractJob(aj.data)) {+ activeExtractCount++;+ } else {+ activeCrawlCount++;+ }+ }++ const jobsToStart: typeof jobsToRecover = [];+ const jobsToQueue: typeof jobsToRecover = [];++ for (const job of jobsToRecover) {+ const isExtract = isExtractJob(job.data);+ const teamLimit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;+ const activeCount = isExtract ? activeExtractCount : activeCrawlCount;++ if (activeCount < teamLimit) {+ jobsToStart.push(job);+ if (isExtract) activeExtractCount++;+ else activeCrawlCount++;+ } else {+ jobsToQueue.push(job);+ }+ }++ let jobsStarted = 0;+ let jobsRequeued = 0;++ for (const job of jobsToQueue) {+ await requeueJob(ownerId, job);+ jobsRequeued++;+ }++ for (const job of jobsToStart) {+ const promoted = await scrapeQueue.promoteJobFromBacklogOrAdd(+ job.id,+ job.data,+ {+ priority: job.priority,+ listenable: job.listenChannelId !== undefined,+ ownerId: job.data.team_id ?? undefined,+ groupId: job.data.crawl_id ?? undefined,+ },+ );++ if (promoted !== null) {+ await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);++ if (job.data.crawl_id) {+ const sc = await getCrawl(job.data.crawl_id);+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {+ await pushCrawlConcurrencyLimitActiveJob(+ job.data.crawl_id,+ job.id,+ 60 * 1000,+ );+ }+ }++ jobsStarted++;+ } else {+ teamLogger.warn("Job promotion failed, re-queuing job", {+ jobId: job.id,+ });+ await requeueJob(ownerId, job);+ jobsRequeued++;+ }+ }++ teamLogger.info("Recovered drift in concurrency queue", {+ missingJobs: missingJobIDs.length,+ recoveredJobs: jobsToRecover.length,+ requeuedJobs: jobsRequeued,+ startedJobs: jobsStarted,+ });++ return { jobsStarted, jobsRequeued };+}++export async function reconcileConcurrencyQueue(+ options: ReconcileOptions = {},+): Promise<ReconcileResult> {+ const logger = (options.logger ?? _logger).child({+ module: "concurrencyQueueReconciler",+ scopedTeamId: options.teamId,+ });++ const owners = options.teamId+ ? [options.teamId]+ : await scrapeQueue.getBackloggedOwnerIDs(logger);++ const ownerIds = owners.filter((x): x is string => typeof x === "string");++ const result: ReconcileResult = {+ teamsScanned: ownerIds.length,+ teamsWithDrift: 0,+ jobsRequeued: 0,+ jobsStarted: 0,+ };++ for (const ownerId of ownerIds) {+ const teamLogger = logger.child({ teamId: ownerId });++ try {+ const teamResult = await reconcileTeam(ownerId, teamLogger);+ if (teamResult !== null) {+ result.teamsWithDrift++;+ result.jobsStarted += teamResult.jobsStarted;+ result.jobsRequeued += teamResult.jobsRequeued;+ }+ } catch (error) {+ teamLogger.error("Failed to reconcile team, skipping", { error });+ }+ }++ return result;+}diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.tsnew file mode 100644index 000000000..19100c479--- /dev/null+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts@@ -0,0 +1,99 @@+import "dotenv/config";+import { config } from "../../config";+import "../sentry";+import { setSentryServiceTag } from "../sentry";+import { logger as _logger } from "../../lib/logger";+import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";+import { Counter, register } from "prom-client";+import Express from "express";++const RECONCILER_INTERVAL_MS = 60 * 1000;++const reconcilerRunsTotal = new Counter({+ name: "concurrency_queue_reconciler_runs_total",+ help: "Total completed concurrency queue reconciler runs",+});++const reconcilerFailuresTotal = new Counter({+ name: "concurrency_queue_reconciler_failures_total",+ help: "Total failed concurrency queue reconciler runs",+});++const reconcilerJobsRecoveredTotal = new Counter({+ name: "concurrency_queue_reconciler_jobs_recovered_total",+ help: "Total drifted jobs recovered by the reconciler",+});++(async () => {+ setSentryServiceTag("nuq-reconciler-worker");++ let isShuttingDown = false;+ let reconcilerInFlight = false;++ const app = Express();++ app.get("/metrics", async (_, res) => {+ try {+ res.contentType("text/plain").send(await register.metrics());+ } catch (error) {+ _logger.error("Failed to collect metrics", { error });+ res.status(500).send("Failed to collect metrics");+ }+ });+ app.get("/health", (_, res) => {+ res.status(200).send("OK");+ });++ const server = app.listen(config.NUQ_RECONCILER_WORKER_PORT, () => {+ _logger.info("NuQ reconciler worker started", {+ port: config.NUQ_RECONCILER_WORKER_PORT,+ });+ });++ async function shutdown() {+ if (isShuttingDown) return;+ isShuttingDown = true;+ _logger.info("NuQ reconciler worker shutting down");++ while (reconcilerInFlight) {+ _logger.info("Waiting for in-flight reconciliation to complete...");+ await new Promise(resolve => setTimeout(resolve, 1000));+ }++ server.close(() => {+ _logger.info("NuQ reconciler worker shut down");+ process.exit(0);+ });+ }++ if (require.main === module) {+ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);+ }++ while (!isShuttingDown) {+ if (!reconcilerInFlight) {+ reconcilerInFlight = true;++ try {+ const summary = await reconcileConcurrencyQueue({+ logger: _logger,+ });++ reconcilerRunsTotal.inc();+ reconcilerJobsRecoveredTotal.inc(+ summary.jobsRequeued + summary.jobsStarted,+ );++ _logger.info("Concurrency queue reconciler run complete", summary);+ } catch (error) {+ reconcilerFailuresTotal.inc();+ _logger.error("Concurrency queue reconciler run failed", { error });+ } finally {+ reconcilerInFlight = false;+ }+ }++ await new Promise(resolve => setTimeout(resolve, RECONCILER_INTERVAL_MS));+ }+})();diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 5e7318082..69f68eb19 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -27,9 +27,9 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fconst app = Express();- app.get("/metrics", (_, res) =>- res.contentType("text/plain").send(nuqGetLocalMetrics()),- );+ app.get("/metrics", (_, res) => {+ res.contentType("text/plain").send(nuqGetLocalMetrics());+ });app.get("/health", async (_, res) => {if (await nuqHealthCheck()) {res.status(200).send("OK");