Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: always populate nuq metrics (#2328)
tom222056c6c283
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 1b256ea67..6bc0875f2 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -61,10 +61,6 @@ export async function markCrawlActive(id: string) {await redisEvictConnection.sadd("active_crawls", id);}-export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {- return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);-}-export async function getCrawl(id: string): Promise<StoredCrawl | null> {return await withSpan("firecrawl-redis-get-crawl", async span => {setSpanAttributes(span, {@@ -273,34 +269,6 @@ export async function finishCrawlKickoff(id: string) {);}-export async function finishCrawlPre(id: string, __logger: Logger = _logger) {- if (await isCrawlFinished(id)) {- __logger.debug("Marking crawl as pre-finished.", {- module: "crawl-redis",- method: "finishCrawlPre",- crawlId: id,- });- const set = await redisEvictConnection.setnx(- "crawl:" + id + ":finished_pre",- "yes",- );- await redisEvictConnection.expire(- "crawl:" + id + ":finished_pre",- 24 * 60 * 60,- );- return set === 1;- }-}--export async function unPreFinishCrawl(id: string) {- _logger.debug("Un-pre-finishing crawl.", {- module: "crawl-redis",- method: "unPreFinishCrawl",- crawlId: id,- });- await redisEvictConnection.del("crawl:" + id + ":finished_pre");-}-export async function finishCrawl(id: string, __logger: Logger = _logger) {__logger.debug("Marking crawl as finished.", {module: "crawl-redis",diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex a0463dd2d..80658758e 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -1242,16 +1242,35 @@ class NuQ<JobData = any, JobReturnValue = any> {// === Metricspublic async getMetrics(): Promise<string> {const start = Date.now();- const result = await nuqPool.query(- `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,- );+ const result = await nuqPool.query(`+ SELECT status::text, COUNT(id) as count FROM ${this.queueName} GROUP BY status+ ${this.options.backlog ? `UNION ALL SELECT 'backlog'::text as status, COUNT(id) as count FROM ${this.queueName}_backlog` : ""}+ `);logger.info("nuqGetMetrics metrics", {module: "nuq/metrics",method: "nuqGetMetrics",duration: Date.now() - start,});const prometheusQueueName = this.queueName.replace(".", "_");- return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${result.rows.map(x => `${prometheusQueueName}_job_count{status="${x.status}"} ${x.count}`).join("\n")}\n`;++ const statusCounts = new Map<NuQJobStatus, number>([+ ["queued", 0],+ ["active", 0],+ ["completed", 0],+ ["failed", 0],+ ["backlog", 0],+ ]);++ result.rows.forEach(x => statusCounts.set(x.status, parseInt(x.count, 10)));++ return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${Array.from(+ statusCounts.entries(),+ )+ .map(+ ([status, count]) =>+ `${prometheusQueueName}_job_count{status="${status}"} ${count}`,+ )+ .join("\n")}\n`;}// === Cleanup