Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { RequestWithAuth } from "./types";
import { Response } from "express";
import { redisEvictConnection } from "../../services/redis";
import { isFdbTeam } from "../../services/worker/nuq-router";
import {
nuqFdbHealthCheck,
scrapeQueueFdb,
withFdbTimeout,
} from "../../services/worker/nuq-fdb";
import { logger } from "../../lib/logger";
import {
cleanOldConcurrencyLimitedJobs,
cleanOldConcurrencyLimitEntries,
getConcurrencyLimitActiveJobsCount,
getConcurrencyQueueJobsCount,
} from "../../lib/concurrency-limit";
import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";
type QueueStatusResponse = {
success: boolean;
jobsInQueue: number;
activeJobsInQueue: number;
waitingJobsInQueue: number;
maxConcurrency: number;
mostRecentSuccess: string | null;
};
const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;
export async function queueStatusController(
req: RequestWithAuth<{}, undefined, QueueStatusResponse>,
res: Response<QueueStatusResponse>,
) {
await cleanOldConcurrencyLimitEntries(req.auth.team_id);
let activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
req.auth.team_id,
);
await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
// during the FDB migration a team can have load on both ledgers
if (await isFdbTeam(req.auth.team_id)) {
try {
if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {
const [fdbActive, fdbPending] = await Promise.all([
withFdbTimeout(
scrapeQueueFdb.getTeamActiveCount(req.auth.team_id),
FDB_OPTIONAL_COUNT_TIMEOUT_MS,
),
withFdbTimeout(
scrapeQueueFdb.getTeamPendingCount(req.auth.team_id),
FDB_OPTIONAL_COUNT_TIMEOUT_MS,
),
]);
activeJobsOfTeam += fdbActive;
queuedJobsOfTeam += fdbPending;
}
} catch (error) {
logger.warn("Failed to read FDB queue counts, falling back to Redis", {
module: "queue-status",
version: "v1",
error,
});
}
}
// most-recent-success is written by the scrape worker via redisEvictConnection
// (REDIS_EVICT_URL), which is a different instance from getRedisConnection()
// (REDIS_URL). Read it from the same connection it is written to.
const mostRecentSuccess = await redisEvictConnection.get(
"most-recent-success:" + req.auth.team_id,
);
return res.status(200).json({
success: true,
jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,
activeJobsInQueue: activeJobsOfTeam,
waitingJobsInQueue: queuedJobsOfTeam,
maxConcurrency:
req.acuc?.concurrency_limit ?? DEFAULT_TEAM_LIMITS.concurrency_limit,
mostRecentSuccess: mostRecentSuccess
? new Date(mostRecentSuccess).toISOString()
: null,
});
}