Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): promote from reconciler (#2943)

tomsideguidee946bd0300fa
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 778ad0fe0..267c4738f 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -173,7 +173,7 @@ export async function pushCrawlConcurrencyLimitActiveJob(
);
}
-async function removeCrawlConcurrencyLimitActiveJob(
+export async function removeCrawlConcurrencyLimitActiveJob(
crawl_id: string,
id: string,
) {
@@ -188,7 +188,7 @@ async function removeCrawlConcurrencyLimitActiveJob(
* @param teamId
* @returns A job that can be run, or null if there are no more jobs to run.
*/
-async function getNextConcurrentJob(teamId: string): Promise<{
+export async function getNextConcurrentJob(teamId: string): Promise<{
job: ConcurrencyLimitedJob;
timeout: number;
} | null> {
@@ -283,6 +283,11 @@ async function getNextConcurrentJob(teamId: string): Promise<{
export async function concurrentJobDone(job: NuQJob<any>) {
if (job.id && job.data && job.data.team_id) {
await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
+ await getRedisConnection().zrem(
+ constructQueueKey(job.data.team_id),
+ job.id,
+ );
+ await getRedisConnection().del(constructJobKey(job.id));
await cleanOldConcurrencyLimitEntries(job.data.team_id);
await cleanOldConcurrencyLimitedJobs(job.data.team_id);
@@ -291,89 +296,92 @@ export async function concurrentJobDone(job: NuQJob<any>) {
await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);
}
- let i = 0;
- for (; i < 10; i++) {
- const maxTeamConcurrency =
- (
- await getACUCTeam(
- job.data.team_id,
- false,
- true,
- job.data.is_extract
- ? RateLimiterMode.Extract
- : RateLimiterMode.Crawl,
- )
- )?.concurrency ?? 2;
+ const maxTeamConcurrency =
+ (
+ await getACUCTeam(
+ job.data.team_id,
+ false,
+ true,
+ job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+
+ let staleSkipped = 0;
+ while (staleSkipped < 100) {
const currentActiveConcurrency = (
await getConcurrencyLimitActiveJobs(job.data.team_id)
).length;
- if (currentActiveConcurrency < maxTeamConcurrency) {
- const nextJob = await getNextConcurrentJob(job.data.team_id);
- if (nextJob !== null) {
- await pushConcurrencyLimitActiveJob(
- job.data.team_id,
- nextJob.job.id,
- 60 * 1000,
- );
+ if (currentActiveConcurrency >= maxTeamConcurrency) break;
- if (nextJob.job.data.crawl_id) {
- await pushCrawlConcurrencyLimitActiveJob(
- nextJob.job.data.crawl_id,
- nextJob.job.id,
- 60 * 1000,
- );
-
- const sc = await getCrawl(nextJob.job.data.crawl_id);
- if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {
- await new Promise(resolve =>
- setTimeout(resolve, sc.crawlerOptions.delay * 1000),
- );
- }
- }
+ const nextJob = await getNextConcurrentJob(job.data.team_id);
+ if (nextJob === null) break;
- abTestJob(nextJob.job.data);
-
- const promotedSuccessfully =
- (await scrapeQueue.promoteJobFromBacklogOrAdd(
- nextJob.job.id,
- nextJob.job.data,
- {
- priority: nextJob.job.priority,
- listenable: nextJob.job.listenable,
- ownerId: nextJob.job.data.team_id ?? undefined,
- groupId: nextJob.job.data.crawl_id ?? undefined,
- },
- )) !== null;
-
- if (promotedSuccessfully) {
- logger.debug("Successfully promoted concurrent queued job", {
- teamId: job.data.team_id,
- jobId: nextJob.job.id,
- zeroDataRetention: nextJob.job.data?.zeroDataRetention,
- });
- break;
- } else {
- logger.warn(
- "Was unable to promote concurrent queued job as it already exists in the database",
- {
- teamId: job.data.team_id,
- jobId: nextJob.job.id,
- zeroDataRetention: nextJob.job.data?.zeroDataRetention,
- },
- );
- }
- } else {
- break;
+ await pushConcurrencyLimitActiveJob(
+ job.data.team_id,
+ nextJob.job.id,
+ 60 * 1000,
+ );
+
+ if (nextJob.job.data.crawl_id) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ 60 * 1000,
+ );
+
+ const sc = await getCrawl(nextJob.job.data.crawl_id);
+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {
+ await new Promise(resolve =>
+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),
+ );
}
- } else {
+ }
+
+ abTestJob(nextJob.job.data);
+
+ const promotedSuccessfully =
+ (await scrapeQueue.promoteJobFromBacklogOrAdd(
+ nextJob.job.id,
+ nextJob.job.data,
+ {
+ priority: nextJob.job.priority,
+ listenable: nextJob.job.listenable,
+ ownerId: nextJob.job.data.team_id ?? undefined,
+ groupId: nextJob.job.data.crawl_id ?? undefined,
+ },
+ )) !== null;
+
+ if (promotedSuccessfully) {
+ logger.debug("Successfully promoted concurrent queued job", {
+ teamId: job.data.team_id,
+ jobId: nextJob.job.id,
+ zeroDataRetention: nextJob.job.data?.zeroDataRetention,
+ });
break;
+ } else {
+ logger.warn(
+ "Was unable to promote concurrent queued job as it already exists in the database",
+ {
+ teamId: job.data.team_id,
+ jobId: nextJob.job.id,
+ zeroDataRetention: nextJob.job.data?.zeroDataRetention,
+ },
+ );
+ await removeConcurrencyLimitActiveJob(job.data.team_id, nextJob.job.id);
+ if (nextJob.job.data.crawl_id) {
+ await removeCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ );
+ }
+ staleSkipped++;
}
}
- if (i === 10) {
+ if (staleSkipped >= 100) {
logger.warn(
- "Failed to promote a concurrent job after 10 iterations, bailing!",
+ "Skipped 100 stale entries in concurrency queue without a successful promotion",
{
teamId: job.data.team_id,
},
diff --git a/apps/api/src/lib/concurrency-queue-reconciler.ts b/apps/api/src/lib/concurrency-queue-reconciler.ts
index b22bf4e1e..3edc047b9 100644
--- a/apps/api/src/lib/concurrency-queue-reconciler.ts
+++ b/apps/api/src/lib/concurrency-queue-reconciler.ts
@@ -5,9 +5,12 @@ import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
import { RateLimiterMode, type ScrapeJobData } from "../types";
import {
getConcurrencyLimitActiveJobs,
+ getNextConcurrentJob,
pushConcurrencyLimitActiveJob,
pushConcurrencyLimitedJob,
pushCrawlConcurrencyLimitActiveJob,
+ removeConcurrencyLimitActiveJob,
+ removeCrawlConcurrencyLimitActiveJob,
} from "./concurrency-limit";
import { getCrawl } from "./crawl-redis";
import { logger as _logger } from "./logger";
@@ -192,6 +195,105 @@ async function reconcileTeam(
return { jobsStarted, jobsRequeued };
}
+async function drainQueue(
+ ownerId: string,
+ teamLogger: Logger,
+): Promise<{ jobsPromoted: number; staleSkipped: number }> {
+ const maxCrawlConcurrency =
+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))
+ ?.concurrency ?? 2;
+ const maxExtractConcurrency =
+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))
+ ?.concurrency ?? 2;
+
+ const activeIds = await getConcurrencyLimitActiveJobs(ownerId);
+ const activeJobs = await scrapeQueue.getJobs(activeIds, teamLogger);
+ let crawlCount = 0;
+ let extractCount = 0;
+ for (const aj of activeJobs) {
+ if (isExtractJob(aj.data)) extractCount++;
+ else crawlCount++;
+ }
+
+ let jobsPromoted = 0;
+ let staleSkipped = 0;
+ let typeBlocked = 0;
+
+ while (staleSkipped + typeBlocked < 100) {
+ if (
+ crawlCount >= maxCrawlConcurrency &&
+ extractCount >= maxExtractConcurrency
+ )
+ break;
+
+ const nextJob = await getNextConcurrentJob(ownerId);
+ if (nextJob === null) break;
+
+ const isExtract = isExtractJob(nextJob.job.data);
+ const typeLimit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;
+ const typeCount = isExtract ? extractCount : crawlCount;
+
+ if (typeCount >= typeLimit) {
+ await pushConcurrencyLimitedJob(
+ ownerId,
+ {
+ id: nextJob.job.id,
+ data: nextJob.job.data,
+ priority: nextJob.job.priority,
+ listenable: nextJob.job.listenable,
+ },
+ nextJob.timeout === Infinity ? 172800000 : nextJob.timeout,
+ );
+ typeBlocked++;
+ continue;
+ }
+
+ await pushConcurrencyLimitActiveJob(ownerId, nextJob.job.id, 60 * 1000);
+ if (nextJob.job.data.crawl_id) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ 60 * 1000,
+ );
+ }
+
+ const promoted = await scrapeQueue.promoteJobFromBacklogOrAdd(
+ nextJob.job.id,
+ nextJob.job.data,
+ {
+ priority: nextJob.job.priority,
+ listenable: nextJob.job.listenable,
+ ownerId: nextJob.job.data.team_id ?? undefined,
+ groupId: nextJob.job.data.crawl_id ?? undefined,
+ },
+ );
+
+ if (promoted !== null) {
+ if (isExtract) extractCount++;
+ else crawlCount++;
+ jobsPromoted++;
+ } else {
+ await removeConcurrencyLimitActiveJob(ownerId, nextJob.job.id);
+ if (nextJob.job.data.crawl_id) {
+ await removeCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ );
+ }
+ staleSkipped++;
+ }
+ }
+
+ if (staleSkipped >= 100) {
+ teamLogger.warn(
+ "Queue drain hit 100 stale entries without fully draining",
+ { ownerId },
+ );
+ }
+
+ return { jobsPromoted, staleSkipped };
+}
+
export async function reconcileConcurrencyQueue(
options: ReconcileOptions = {},
): Promise<ReconcileResult> {
@@ -200,11 +302,21 @@ export async function reconcileConcurrencyQueue(
scopedTeamId: options.teamId,
});
- const owners = options.teamId
- ? [options.teamId]
- : await scrapeQueue.getBackloggedOwnerIDs(logger);
-
- const ownerIds = owners.filter((x): x is string => typeof x === "string");
+ let ownerIds: string[];
+ if (options.teamId) {
+ ownerIds = [options.teamId];
+ } else {
+ const backlogOwners = (
+ await scrapeQueue.getBackloggedOwnerIDs(logger)
+ ).filter((x): x is string => typeof x === "string");
+ const queueKeys = await getRedisConnection().smembers(
+ "concurrency-limit-queues",
+ );
+ const queueOwners = queueKeys
+ .map(k => k.replace("concurrency-limit-queue:", ""))
+ .filter(id => id.length > 0);
+ ownerIds = [...new Set([...backlogOwners, ...queueOwners])];
+ }
const result: ReconcileResult = {
teamsScanned: ownerIds.length,
@@ -223,6 +335,15 @@ export async function reconcileConcurrencyQueue(
result.jobsStarted += teamResult.jobsStarted;
result.jobsRequeued += teamResult.jobsRequeued;
}
+
+ const drainResult = await drainQueue(ownerId, teamLogger);
+ if (drainResult.jobsPromoted > 0 || drainResult.staleSkipped > 0) {
+ result.jobsStarted += drainResult.jobsPromoted;
+ teamLogger.info("Queue drain promoted jobs", {
+ jobsPromoted: drainResult.jobsPromoted,
+ staleSkipped: drainResult.staleSkipped,
+ });
+ }
} catch (error) {
teamLogger.error("Failed to reconcile team, skipping", { error });
}