Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: always populate nuq metrics (#2328)

tom222056c6c283
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 1b256ea67..6bc0875f2 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -61,10 +61,6 @@ export async function markCrawlActive(id: string) {
await redisEvictConnection.sadd("active_crawls", id);
}
-export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {
- return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);
-}
-
export async function getCrawl(id: string): Promise<StoredCrawl | null> {
return await withSpan("firecrawl-redis-get-crawl", async span => {
setSpanAttributes(span, {
@@ -273,34 +269,6 @@ export async function finishCrawlKickoff(id: string) {
);
}
-export async function finishCrawlPre(id: string, __logger: Logger = _logger) {
- if (await isCrawlFinished(id)) {
- __logger.debug("Marking crawl as pre-finished.", {
- module: "crawl-redis",
- method: "finishCrawlPre",
- crawlId: id,
- });
- const set = await redisEvictConnection.setnx(
- "crawl:" + id + ":finished_pre",
- "yes",
- );
- await redisEvictConnection.expire(
- "crawl:" + id + ":finished_pre",
- 24 * 60 * 60,
- );
- return set === 1;
- }
-}
-
-export async function unPreFinishCrawl(id: string) {
- _logger.debug("Un-pre-finishing crawl.", {
- module: "crawl-redis",
- method: "unPreFinishCrawl",
- crawlId: id,
- });
- await redisEvictConnection.del("crawl:" + id + ":finished_pre");
-}
-
export async function finishCrawl(id: string, __logger: Logger = _logger) {
__logger.debug("Marking crawl as finished.", {
module: "crawl-redis",
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index a0463dd2d..80658758e 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -1242,16 +1242,35 @@ class NuQ<JobData = any, JobReturnValue = any> {
// === Metrics
public async getMetrics(): Promise<string> {
const start = Date.now();
- const result = await nuqPool.query(
- `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,
- );
+ const result = await nuqPool.query(`
+ SELECT status::text, COUNT(id) as count FROM ${this.queueName} GROUP BY status
+ ${this.options.backlog ? `UNION ALL SELECT 'backlog'::text as status, COUNT(id) as count FROM ${this.queueName}_backlog` : ""}
+ `);
logger.info("nuqGetMetrics metrics", {
module: "nuq/metrics",
method: "nuqGetMetrics",
duration: Date.now() - start,
});
const prometheusQueueName = this.queueName.replace(".", "_");
- return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${result.rows.map(x => `${prometheusQueueName}_job_count{status="${x.status}"} ${x.count}`).join("\n")}\n`;
+
+ const statusCounts = new Map<NuQJobStatus, number>([
+ ["queued", 0],
+ ["active", 0],
+ ["completed", 0],
+ ["failed", 0],
+ ["backlog", 0],
+ ]);
+
+ result.rows.forEach(x => statusCounts.set(x.status, parseInt(x.count, 10)));
+
+ return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${Array.from(
+ statusCounts.entries(),
+ )
+ .map(
+ ([status, count]) =>
+ `${prometheusQueueName}_job_count{status="${status}"} ${count}`,
+ )
+ .join("\n")}\n`;
}
// === Cleanup