Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add created_at field to /crawl/active endpoint response (#1718)

devin-ai-integration[bot]8922d7eb5dca
diff --git a/apps/api/requests.http b/apps/api/requests.http
index 55d315d1b..5f55c4975 100644
--- a/apps/api/requests.http
+++ b/apps/api/requests.http
@@ -126,3 +126,8 @@ content-type: application/json
"query": "firecrawl",
"limit": 50
}
+
+### Get Active Crawls
+# @name activeCrawls
+GET {{baseUrl}}/v1/crawl/active HTTP/1.1
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 584c8b14c..907018350 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -60,14 +60,32 @@ describe("Crawl tests", () => {
}, 3 * scrapeTimeout + 3 * 5000);
it.concurrent("ongoing crawls endpoint works", async () => {
+ const beforeCrawl = new Date();
+
const res = await asyncCrawl({
url: "https://firecrawl.dev",
limit: 3,
}, identity);
const ongoing = await crawlOngoing(identity);
-
- expect(ongoing.crawls.find(x => x.id === res.id)).toBeDefined();
+ const afterCrawl = new Date();
+
+ const crawlItem = ongoing.crawls.find(x => x.id === res.id);
+ expect(crawlItem).toBeDefined();
+
+ if (crawlItem) {
+ expect(crawlItem.created_at).toBeDefined();
+ expect(typeof crawlItem.created_at).toBe("string");
+
+ const createdAtDate = new Date(crawlItem.created_at);
+ expect(createdAtDate).toBeInstanceOf(Date);
+ expect(createdAtDate.getTime()).not.toBeNaN();
+
+ expect(crawlItem.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);
+
+ expect(createdAtDate.getTime()).toBeGreaterThanOrEqual(beforeCrawl.getTime() - 1000);
+ expect(createdAtDate.getTime()).toBeLessThanOrEqual(afterCrawl.getTime() + 1000);
+ }
await asyncCrawlWaitForFinish(res.id, identity);
diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.ts
index 7839d5fc1..a9dc52134 100644
--- a/apps/api/src/controllers/v1/crawl-ongoing.ts
+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts
@@ -25,6 +25,7 @@ export async function ongoingCrawlsController(
id: x.id,
teamId: x.team_id!,
url: x.originUrl!,
+ created_at: new Date(x.createdAt || Date.now()).toISOString(),
options: {
...toNewCrawlerOptions(x.crawlerOptions),
scrapeOptions: x.scrapeOptions,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 2c7989270..7064ce9e1 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -976,6 +976,7 @@ export type OngoingCrawlsResponse =
id: string;
teamId: string;
url: string;
+ created_at: string;
options: CrawlerOptions;
}[];
};