Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): promote from reconciler (#2943)
tomsideguidee946bd0300fa
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 778ad0fe0..267c4738f 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -173,7 +173,7 @@ export async function pushCrawlConcurrencyLimitActiveJob();}-async function removeCrawlConcurrencyLimitActiveJob(+export async function removeCrawlConcurrencyLimitActiveJob(crawl_id: string,id: string,) {@@ -188,7 +188,7 @@ async function removeCrawlConcurrencyLimitActiveJob(* @param teamId* @returns A job that can be run, or null if there are no more jobs to run.*/-async function getNextConcurrentJob(teamId: string): Promise<{+export async function getNextConcurrentJob(teamId: string): Promise<{job: ConcurrencyLimitedJob;timeout: number;} | null> {@@ -283,6 +283,11 @@ async function getNextConcurrentJob(teamId: string): Promise<{export async function concurrentJobDone(job: NuQJob<any>) {if (job.id && job.data && job.data.team_id) {await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);+ await getRedisConnection().zrem(+ constructQueueKey(job.data.team_id),+ job.id,+ );+ await getRedisConnection().del(constructJobKey(job.id));await cleanOldConcurrencyLimitEntries(job.data.team_id);await cleanOldConcurrencyLimitedJobs(job.data.team_id);@@ -291,89 +296,92 @@ export async function concurrentJobDone(job: NuQJob<any>) {await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);}- let i = 0;- for (; i < 10; i++) {- const maxTeamConcurrency =- (- await getACUCTeam(- job.data.team_id,- false,- true,- job.data.is_extract- ? RateLimiterMode.Extract- : RateLimiterMode.Crawl,- )- )?.concurrency ?? 2;+ const maxTeamConcurrency =+ (+ await getACUCTeam(+ job.data.team_id,+ false,+ true,+ job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;++ let staleSkipped = 0;+ while (staleSkipped < 100) {const currentActiveConcurrency = (await getConcurrencyLimitActiveJobs(job.data.team_id)).length;- if (currentActiveConcurrency < maxTeamConcurrency) {- const nextJob = await getNextConcurrentJob(job.data.team_id);- if (nextJob !== null) {- await pushConcurrencyLimitActiveJob(- job.data.team_id,- nextJob.job.id,- 60 * 1000,- );+ if (currentActiveConcurrency >= maxTeamConcurrency) break;- if (nextJob.job.data.crawl_id) {- await pushCrawlConcurrencyLimitActiveJob(- nextJob.job.data.crawl_id,- nextJob.job.id,- 60 * 1000,- );-- const sc = await getCrawl(nextJob.job.data.crawl_id);- if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {- await new Promise(resolve =>- setTimeout(resolve, sc.crawlerOptions.delay * 1000),- );- }- }+ const nextJob = await getNextConcurrentJob(job.data.team_id);+ if (nextJob === null) break;- abTestJob(nextJob.job.data);-- const promotedSuccessfully =- (await scrapeQueue.promoteJobFromBacklogOrAdd(- nextJob.job.id,- nextJob.job.data,- {- priority: nextJob.job.priority,- listenable: nextJob.job.listenable,- ownerId: nextJob.job.data.team_id ?? undefined,- groupId: nextJob.job.data.crawl_id ?? undefined,- },- )) !== null;-- if (promotedSuccessfully) {- logger.debug("Successfully promoted concurrent queued job", {- teamId: job.data.team_id,- jobId: nextJob.job.id,- zeroDataRetention: nextJob.job.data?.zeroDataRetention,- });- break;- } else {- logger.warn(- "Was unable to promote concurrent queued job as it already exists in the database",- {- teamId: job.data.team_id,- jobId: nextJob.job.id,- zeroDataRetention: nextJob.job.data?.zeroDataRetention,- },- );- }- } else {- break;+ await pushConcurrencyLimitActiveJob(+ job.data.team_id,+ nextJob.job.id,+ 60 * 1000,+ );++ if (nextJob.job.data.crawl_id) {+ await pushCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ 60 * 1000,+ );++ const sc = await getCrawl(nextJob.job.data.crawl_id);+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {+ await new Promise(resolve =>+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),+ );}- } else {+ }++ abTestJob(nextJob.job.data);++ const promotedSuccessfully =+ (await scrapeQueue.promoteJobFromBacklogOrAdd(+ nextJob.job.id,+ nextJob.job.data,+ {+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ ownerId: nextJob.job.data.team_id ?? undefined,+ groupId: nextJob.job.data.crawl_id ?? undefined,+ },+ )) !== null;++ if (promotedSuccessfully) {+ logger.debug("Successfully promoted concurrent queued job", {+ teamId: job.data.team_id,+ jobId: nextJob.job.id,+ zeroDataRetention: nextJob.job.data?.zeroDataRetention,+ });break;+ } else {+ logger.warn(+ "Was unable to promote concurrent queued job as it already exists in the database",+ {+ teamId: job.data.team_id,+ jobId: nextJob.job.id,+ zeroDataRetention: nextJob.job.data?.zeroDataRetention,+ },+ );+ await removeConcurrencyLimitActiveJob(job.data.team_id, nextJob.job.id);+ if (nextJob.job.data.crawl_id) {+ await removeCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ );+ }+ staleSkipped++;}}- if (i === 10) {+ if (staleSkipped >= 100) {logger.warn(- "Failed to promote a concurrent job after 10 iterations, bailing!",+ "Skipped 100 stale entries in concurrency queue without a successful promotion",{teamId: job.data.team_id,},diff --git a/apps/api/src/lib/concurrency-queue-reconciler.ts b/apps/api/src/lib/concurrency-queue-reconciler.tsindex b22bf4e1e..3edc047b9 100644--- a/apps/api/src/lib/concurrency-queue-reconciler.ts+++ b/apps/api/src/lib/concurrency-queue-reconciler.ts@@ -5,9 +5,12 @@ import { scrapeQueue, type NuQJob } from "../services/worker/nuq";import { RateLimiterMode, type ScrapeJobData } from "../types";import {getConcurrencyLimitActiveJobs,+ getNextConcurrentJob,pushConcurrencyLimitActiveJob,pushConcurrencyLimitedJob,pushCrawlConcurrencyLimitActiveJob,+ removeConcurrencyLimitActiveJob,+ removeCrawlConcurrencyLimitActiveJob,} from "./concurrency-limit";import { getCrawl } from "./crawl-redis";import { logger as _logger } from "./logger";@@ -192,6 +195,105 @@ async function reconcileTeam(return { jobsStarted, jobsRequeued };}+async function drainQueue(+ ownerId: string,+ teamLogger: Logger,+): Promise<{ jobsPromoted: number; staleSkipped: number }> {+ const maxCrawlConcurrency =+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))+ ?.concurrency ?? 2;+ const maxExtractConcurrency =+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))+ ?.concurrency ?? 2;++ const activeIds = await getConcurrencyLimitActiveJobs(ownerId);+ const activeJobs = await scrapeQueue.getJobs(activeIds, teamLogger);+ let crawlCount = 0;+ let extractCount = 0;+ for (const aj of activeJobs) {+ if (isExtractJob(aj.data)) extractCount++;+ else crawlCount++;+ }++ let jobsPromoted = 0;+ let staleSkipped = 0;+ let typeBlocked = 0;++ while (staleSkipped + typeBlocked < 100) {+ if (+ crawlCount >= maxCrawlConcurrency &&+ extractCount >= maxExtractConcurrency+ )+ break;++ const nextJob = await getNextConcurrentJob(ownerId);+ if (nextJob === null) break;++ const isExtract = isExtractJob(nextJob.job.data);+ const typeLimit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;+ const typeCount = isExtract ? extractCount : crawlCount;++ if (typeCount >= typeLimit) {+ await pushConcurrencyLimitedJob(+ ownerId,+ {+ id: nextJob.job.id,+ data: nextJob.job.data,+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ },+ nextJob.timeout === Infinity ? 172800000 : nextJob.timeout,+ );+ typeBlocked++;+ continue;+ }++ await pushConcurrencyLimitActiveJob(ownerId, nextJob.job.id, 60 * 1000);+ if (nextJob.job.data.crawl_id) {+ await pushCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ 60 * 1000,+ );+ }++ const promoted = await scrapeQueue.promoteJobFromBacklogOrAdd(+ nextJob.job.id,+ nextJob.job.data,+ {+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ ownerId: nextJob.job.data.team_id ?? undefined,+ groupId: nextJob.job.data.crawl_id ?? undefined,+ },+ );++ if (promoted !== null) {+ if (isExtract) extractCount++;+ else crawlCount++;+ jobsPromoted++;+ } else {+ await removeConcurrencyLimitActiveJob(ownerId, nextJob.job.id);+ if (nextJob.job.data.crawl_id) {+ await removeCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ );+ }+ staleSkipped++;+ }+ }++ if (staleSkipped >= 100) {+ teamLogger.warn(+ "Queue drain hit 100 stale entries without fully draining",+ { ownerId },+ );+ }++ return { jobsPromoted, staleSkipped };+}+export async function reconcileConcurrencyQueue(options: ReconcileOptions = {},): Promise<ReconcileResult> {@@ -200,11 +302,21 @@ export async function reconcileConcurrencyQueue(scopedTeamId: options.teamId,});- const owners = options.teamId- ? [options.teamId]- : await scrapeQueue.getBackloggedOwnerIDs(logger);-- const ownerIds = owners.filter((x): x is string => typeof x === "string");+ let ownerIds: string[];+ if (options.teamId) {+ ownerIds = [options.teamId];+ } else {+ const backlogOwners = (+ await scrapeQueue.getBackloggedOwnerIDs(logger)+ ).filter((x): x is string => typeof x === "string");+ const queueKeys = await getRedisConnection().smembers(+ "concurrency-limit-queues",+ );+ const queueOwners = queueKeys+ .map(k => k.replace("concurrency-limit-queue:", ""))+ .filter(id => id.length > 0);+ ownerIds = [...new Set([...backlogOwners, ...queueOwners])];+ }const result: ReconcileResult = {teamsScanned: ownerIds.length,@@ -223,6 +335,15 @@ export async function reconcileConcurrencyQueue(result.jobsStarted += teamResult.jobsStarted;result.jobsRequeued += teamResult.jobsRequeued;}++ const drainResult = await drainQueue(ownerId, teamLogger);+ if (drainResult.jobsPromoted > 0 || drainResult.staleSkipped > 0) {+ result.jobsStarted += drainResult.jobsPromoted;+ teamLogger.info("Queue drain promoted jobs", {+ jobsPromoted: drainResult.jobsPromoted,+ staleSkipped: drainResult.staleSkipped,+ });+ }} catch (error) {teamLogger.error("Failed to reconcile team, skipping", { error });}