Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2/crawl): expose createdAt, completedAt, and duration on crawl status

firecrawl-spring[bot]dad61eb89e01
diff --git a/apps/api/openapi.json b/apps/api/openapi.json
index 8c6c8771c..101a62aeb 100644
--- a/apps/api/openapi.json
+++ b/apps/api/openapi.json
@@ -2648,6 +2648,20 @@
"format": "date-time",
"description": "The date and time when the crawl will expire."
},
+ "createdAt": {
+ "type": "string",
+ "format": "date-time",
+ "description": "The date and time when the crawl was started."
+ },
+ "completedAt": {
+ "type": "string",
+ "format": "date-time",
+ "description": "The date and time when the crawl finished. Present only when the crawl is in a terminal state (`completed`, `failed`, or `cancelled`)."
+ },
+ "duration": {
+ "type": "number",
+ "description": "Crawl duration in seconds. For terminal crawls, this is the elapsed time from `createdAt` to `completedAt`. For in-progress crawls, it is the elapsed time from `createdAt` to now."
+ },
"next": {
"type": "string",
"nullable": true,
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 53c70e4db..6a389e7e2 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -216,6 +216,44 @@ describe("Crawl tests", () => {
// 10 * scrapeTimeout,
// );
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "crawl status returns createdAt, completedAt, and duration",
+ async () => {
+ const beforeCrawl = Date.now();
+
+ const results = await crawl(
+ {
+ url: base,
+ limit: 3,
+ },
+ identity,
+ );
+
+ const afterCrawl = Date.now();
+
+ expect(results.success).toBe(true);
+ if (results.success) {
+ expect(typeof results.createdAt).toBe("string");
+ expect(typeof results.completedAt).toBe("string");
+ expect(typeof results.duration).toBe("number");
+
+ const createdAtMs = new Date(results.createdAt!).getTime();
+ const completedAtMs = new Date(results.completedAt!).getTime();
+
+ expect(createdAtMs).not.toBeNaN();
+ expect(completedAtMs).not.toBeNaN();
+ expect(completedAtMs).toBeGreaterThanOrEqual(createdAtMs);
+ expect(createdAtMs).toBeGreaterThanOrEqual(beforeCrawl - 1000);
+ expect(completedAtMs).toBeLessThanOrEqual(afterCrawl + 1000);
+
+ expect(results.duration).toBeGreaterThanOrEqual(0);
+ const expectedDuration = (completedAtMs - createdAtMs) / 1000;
+ expect(Math.abs(results.duration! - expectedDuration)).toBeLessThan(1);
+ }
+ },
+ 10 * scrapeTimeout,
+ );
+
concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
"delay parameter works",
async () => {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 784804e78..6883ab7e4 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -13,6 +13,7 @@ import {
getCrawlQualifiedJobCount,
getDoneJobsOrderedLength,
getDoneJobsOrderedUntil,
+ getLastDoneJobTimestamp,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
import {
@@ -254,6 +255,7 @@ export async function crawlStatusController(
// if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)
if (outputBulkA.status === "failed" && crawlError) {
+ const createdAtMs = sc?.createdAt;
return res.status(200).json({
success: false,
error: crawlError,
@@ -263,6 +265,10 @@ export async function crawlStatusController(
creditsUsed: outputBulkA.creditsUsed ?? 0,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
data: [],
+ ...(createdAtMs && {
+ createdAt: new Date(createdAtMs).toISOString(),
+ duration: Math.max(0, (Date.now() - createdAtMs) / 1000),
+ }),
});
}
@@ -364,15 +370,33 @@ export async function crawlStatusController(
}
}
+ const status = outputBulkA.status ?? "scraping";
+ const createdAtMs = sc?.createdAt;
+ const lastDoneMs =
+ status !== "scraping" ? await getLastDoneJobTimestamp(req.params.jobId) : null;
+ const completedAtMs =
+ status === "completed" || status === "failed" || status === "cancelled"
+ ? lastDoneMs ?? createdAtMs ?? null
+ : null;
+ const durationSeconds = createdAtMs
+ ? Math.max(
+ 0,
+ ((completedAtMs ?? Date.now()) - createdAtMs) / 1000,
+ )
+ : undefined;
+
return res.status(200).json({
success: true,
- status: outputBulkA.status ?? "scraping",
+ status,
completed: outputBulkA.completed ?? 0,
total: outputBulkA.total ?? 0,
creditsUsed: outputBulkA.creditsUsed ?? 0,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
next: outputBulkB.next,
data: outputBulkB.data,
+ ...(createdAtMs && { createdAt: new Date(createdAtMs).toISOString() }),
+ ...(completedAtMs && { completedAt: new Date(completedAtMs).toISOString() }),
+ ...(durationSeconds !== undefined && { duration: durationSeconds }),
...(warning && { warning }),
});
}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 22ac101ba..4748ac1d0 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1436,6 +1436,9 @@ export type CrawlStatusResponse =
total: number;
creditsUsed: number;
expiresAt: string;
+ createdAt?: string;
+ completedAt?: string;
+ duration?: number;
next?: string;
data: Document[];
warning?: string;
@@ -1448,6 +1451,8 @@ export type CrawlStatusResponse =
total: number;
creditsUsed: number;
expiresAt: string;
+ createdAt?: string;
+ duration?: number;
data: Document[];
};
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 5699c6657..eddbb4e59 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -206,6 +206,24 @@ export async function getDoneJobsOrdered(
);
}
+export async function getLastDoneJobTimestamp(
+ id: string,
+): Promise<number | null> {
+ await redisEvictConnection.expire(
+ "crawl:" + id + ":jobs_donez_ordered",
+ 24 * 60 * 60,
+ );
+ const result = await redisEvictConnection.zrange(
+ "crawl:" + id + ":jobs_donez_ordered",
+ -1,
+ -1,
+ "WITHSCORES",
+ );
+ if (!result || result.length < 2) return null;
+ const score = parseInt(result[1], 10);
+ return Number.isFinite(score) ? score : null;
+}
+
export async function getDoneJobsOrderedUntil(
id: string,
until: number = Infinity,