Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2/crawl): expose createdAt, completedAt, and duration on crawl status
firecrawl-spring[bot]dad61eb89e01
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 8c6c8771c..101a62aeb 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -2648,6 +2648,20 @@"format": "date-time","description": "The date and time when the crawl will expire."},+ "createdAt": {+ "type": "string",+ "format": "date-time",+ "description": "The date and time when the crawl was started."+ },+ "completedAt": {+ "type": "string",+ "format": "date-time",+ "description": "The date and time when the crawl finished. Present only when the crawl is in a terminal state (`completed`, `failed`, or `cancelled`)."+ },+ "duration": {+ "type": "number",+ "description": "Crawl duration in seconds. For terminal crawls, this is the elapsed time from `createdAt` to `completedAt`. For in-progress crawls, it is the elapsed time from `createdAt` to now."+ },"next": {"type": "string","nullable": true,diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 53c70e4db..6a389e7e2 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -216,6 +216,44 @@ describe("Crawl tests", () => {// 10 * scrapeTimeout,// );+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "crawl status returns createdAt, completedAt, and duration",+ async () => {+ const beforeCrawl = Date.now();++ const results = await crawl(+ {+ url: base,+ limit: 3,+ },+ identity,+ );++ const afterCrawl = Date.now();++ expect(results.success).toBe(true);+ if (results.success) {+ expect(typeof results.createdAt).toBe("string");+ expect(typeof results.completedAt).toBe("string");+ expect(typeof results.duration).toBe("number");++ const createdAtMs = new Date(results.createdAt!).getTime();+ const completedAtMs = new Date(results.completedAt!).getTime();++ expect(createdAtMs).not.toBeNaN();+ expect(completedAtMs).not.toBeNaN();+ expect(completedAtMs).toBeGreaterThanOrEqual(createdAtMs);+ expect(createdAtMs).toBeGreaterThanOrEqual(beforeCrawl - 1000);+ expect(completedAtMs).toBeLessThanOrEqual(afterCrawl + 1000);++ expect(results.duration).toBeGreaterThanOrEqual(0);+ const expectedDuration = (completedAtMs - createdAtMs) / 1000;+ expect(Math.abs(results.duration! - expectedDuration)).toBeLessThan(1);+ }+ },+ 10 * scrapeTimeout,+ );+concurrentIf(ALLOW_TEST_SUITE_WEBSITE)("delay parameter works",async () => {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 784804e78..6883ab7e4 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -13,6 +13,7 @@ import {getCrawlQualifiedJobCount,getDoneJobsOrderedLength,getDoneJobsOrderedUntil,+ getLastDoneJobTimestamp,isCrawlKickoffFinished,} from "../../lib/crawl-redis";import {@@ -254,6 +255,7 @@ export async function crawlStatusController(// if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)if (outputBulkA.status === "failed" && crawlError) {+ const createdAtMs = sc?.createdAt;return res.status(200).json({success: false,error: crawlError,@@ -263,6 +265,10 @@ export async function crawlStatusController(creditsUsed: outputBulkA.creditsUsed ?? 0,expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),data: [],+ ...(createdAtMs && {+ createdAt: new Date(createdAtMs).toISOString(),+ duration: Math.max(0, (Date.now() - createdAtMs) / 1000),+ }),});}@@ -364,15 +370,33 @@ export async function crawlStatusController(}}+ const status = outputBulkA.status ?? "scraping";+ const createdAtMs = sc?.createdAt;+ const lastDoneMs =+ status !== "scraping" ? await getLastDoneJobTimestamp(req.params.jobId) : null;+ const completedAtMs =+ status === "completed" || status === "failed" || status === "cancelled"+ ? lastDoneMs ?? createdAtMs ?? null+ : null;+ const durationSeconds = createdAtMs+ ? Math.max(+ 0,+ ((completedAtMs ?? Date.now()) - createdAtMs) / 1000,+ )+ : undefined;+return res.status(200).json({success: true,- status: outputBulkA.status ?? "scraping",+ status,completed: outputBulkA.completed ?? 0,total: outputBulkA.total ?? 0,creditsUsed: outputBulkA.creditsUsed ?? 0,expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),next: outputBulkB.next,data: outputBulkB.data,+ ...(createdAtMs && { createdAt: new Date(createdAtMs).toISOString() }),+ ...(completedAtMs && { completedAt: new Date(completedAtMs).toISOString() }),+ ...(durationSeconds !== undefined && { duration: durationSeconds }),...(warning && { warning }),});}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 22ac101ba..4748ac1d0 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1436,6 +1436,9 @@ export type CrawlStatusResponse =total: number;creditsUsed: number;expiresAt: string;+ createdAt?: string;+ completedAt?: string;+ duration?: number;next?: string;data: Document[];warning?: string;@@ -1448,6 +1451,8 @@ export type CrawlStatusResponse =total: number;creditsUsed: number;expiresAt: string;+ createdAt?: string;+ duration?: number;data: Document[];};diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 5699c6657..eddbb4e59 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -206,6 +206,24 @@ export async function getDoneJobsOrdered();}+export async function getLastDoneJobTimestamp(+ id: string,+): Promise<number | null> {+ await redisEvictConnection.expire(+ "crawl:" + id + ":jobs_donez_ordered",+ 24 * 60 * 60,+ );+ const result = await redisEvictConnection.zrange(+ "crawl:" + id + ":jobs_donez_ordered",+ -1,+ -1,+ "WITHSCORES",+ );+ if (!result || result.length < 2) return null;+ const score = parseInt(result[1], 10);+ return Number.isFinite(score) ? score : null;+}+export async function getDoneJobsOrderedUntil(id: string,until: number = Infinity,