Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2): Set default maxAge to 4 hours for scrape endpoint (#1915)

Gergő Móriczff87297ea9de
diff --git a/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.ts b/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.ts
new file mode 100644
index 000000000..05fd0aa3d
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.ts
@@ -0,0 +1,142 @@
+import request from "supertest";
+import { Identity, idmux, scrapeTimeout } from "./lib";
+import crypto from "crypto";
+
+const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";
+
+describe("V2 Scrape Default maxAge", () => {
+ if (!process.env.TEST_SUITE_SELF_HOSTED) {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "v2-scrape-default-maxage",
+ concurrency: 100,
+ credits: 1000000,
+ });
+ }, 10000);
+
+ test("should use default maxAge of 4 hours when not specified", async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ // First scrape to populate cache
+ const response1 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ });
+
+ expect(response1.status).toBe(200);
+ expect(response1.body.success).toBe(true);
+ expect(response1.body.data).toBeDefined();
+ expect(response1.body.data.metadata.cacheState).toBe("miss");
+
+ // Wait for index to be populated
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ // Second scrape should hit cache with default maxAge
+ const response2 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ });
+
+ expect(response2.status).toBe(200);
+ expect(response2.body.success).toBe(true);
+ expect(response2.body.data).toBeDefined();
+ expect(response2.body.data.metadata.cacheState).toBe("hit");
+ expect(response2.body.data.metadata.cachedAt).toBeDefined();
+ }, scrapeTimeout * 2 + 20000);
+
+ test("should respect explicitly set maxAge of 0", async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ // First scrape to populate cache
+ const response1 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ });
+
+ expect(response1.status).toBe(200);
+ expect(response1.body.success).toBe(true);
+ expect(response1.body.data).toBeDefined();
+ expect(response1.body.data.metadata.cacheState).toBe("miss");
+
+ // Wait for index to be populated
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ // Second scrape with maxAge=0 should miss cache
+ const response2 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ maxAge: 0,
+ });
+
+ expect(response2.status).toBe(200);
+ expect(response2.body.success).toBe(true);
+ expect(response2.body.data).toBeDefined();
+ expect(response2.body.data.metadata.cacheState).toBeUndefined();
+ }, scrapeTimeout * 2 + 20000);
+
+ test("should respect custom maxAge value", async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ // First scrape to populate cache
+ const response1 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ maxAge: 3600000, // 1 hour in milliseconds
+ });
+
+ expect(response1.status).toBe(200);
+ expect(response1.body.success).toBe(true);
+ expect(response1.body.data).toBeDefined();
+ expect(response1.body.data.metadata.cacheState).toBe("miss");
+
+ // Wait for index to be populated
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ // Second scrape with same maxAge should hit cache
+ const response2 = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url,
+ timeout: scrapeTimeout,
+ maxAge: 3600000, // 1 hour in milliseconds
+ });
+
+ expect(response2.status).toBe(200);
+ expect(response2.body.success).toBe(true);
+ expect(response2.body.data).toBeDefined();
+ expect(response2.body.data.metadata.cacheState).toBe("hit");
+ expect(response2.body.data.metadata.cachedAt).toBeDefined();
+ }, scrapeTimeout * 2 + 20000);
+ } else {
+ it("mocked", () => {
+ expect(true).toBe(true);
+ });
+ }
+});
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 843bddc7e..90b4b5eb1 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -296,7 +296,7 @@ const baseScrapeOptions = z
useMock: z.string().optional(),
blockAds: z.boolean().default(true),
proxy: z.enum(["basic", "stealth", "auto"]).optional(),
- maxAge: z.number().int().gte(0).safe().default(0),
+ maxAge: z.number().int().gte(0).safe().default(14400000),
storeInCache: z.boolean().default(true),
// @deprecated
__searchPreviewToken: z.string().optional(),
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 1718fbc02..50ac88839 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -92,6 +92,7 @@ function execForward(fancyName: string, command: string): Promise<void> {
const api = execForward("api", "pnpm start:production:nobuild");
const worker = execForward("worker", "pnpm worker:production");
+ const indexWorker = execForward("indexWorker", "pnpm index-worker:production");
try {
await Promise.race([
@@ -107,14 +108,17 @@ function execForward(fancyName: string, command: string): Promise<void> {
cmd,
api,
worker,
+ indexWorker,
]);
} finally {
console.log("=== Tearing down API and Worker...");
exec("pkill -f 'queue-worker.js'");
exec("pkill -f 'index.js'");
+ exec("pkill -f 'index-worker.js'");
await Promise.all([
api,
worker,
+ indexWorker,
]);
}