Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2): Set default maxAge to 4 hours for scrape endpoint (#1915)
Gergő Móriczff87297ea9de
diff --git a/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.ts b/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.tsnew file mode 100644index 000000000..05fd0aa3d--- /dev/null+++ b/apps/api/src/__tests__/snips/v2-scrape-default-maxage.test.ts@@ -0,0 +1,142 @@+import request from "supertest";+import { Identity, idmux, scrapeTimeout } from "./lib";+import crypto from "crypto";++const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";++describe("V2 Scrape Default maxAge", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "v2-scrape-default-maxage",+ concurrency: 100,+ credits: 1000000,+ });+ }, 10000);++ test("should use default maxAge of 4 hours when not specified", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ // First scrape to populate cache+ const response1 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ });++ expect(response1.status).toBe(200);+ expect(response1.body.success).toBe(true);+ expect(response1.body.data).toBeDefined();+ expect(response1.body.data.metadata.cacheState).toBe("miss");++ // Wait for index to be populated+ await new Promise(resolve => setTimeout(resolve, 20000));++ // Second scrape should hit cache with default maxAge+ const response2 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ });++ expect(response2.status).toBe(200);+ expect(response2.body.success).toBe(true);+ expect(response2.body.data).toBeDefined();+ expect(response2.body.data.metadata.cacheState).toBe("hit");+ expect(response2.body.data.metadata.cachedAt).toBeDefined();+ }, scrapeTimeout * 2 + 20000);++ test("should respect explicitly set maxAge of 0", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ // First scrape to populate cache+ const response1 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ });++ expect(response1.status).toBe(200);+ expect(response1.body.success).toBe(true);+ expect(response1.body.data).toBeDefined();+ expect(response1.body.data.metadata.cacheState).toBe("miss");++ // Wait for index to be populated+ await new Promise(resolve => setTimeout(resolve, 20000));++ // Second scrape with maxAge=0 should miss cache+ const response2 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ maxAge: 0,+ });++ expect(response2.status).toBe(200);+ expect(response2.body.success).toBe(true);+ expect(response2.body.data).toBeDefined();+ expect(response2.body.data.metadata.cacheState).toBeUndefined();+ }, scrapeTimeout * 2 + 20000);++ test("should respect custom maxAge value", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ // First scrape to populate cache+ const response1 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ maxAge: 3600000, // 1 hour in milliseconds+ });++ expect(response1.status).toBe(200);+ expect(response1.body.success).toBe(true);+ expect(response1.body.data).toBeDefined();+ expect(response1.body.data.metadata.cacheState).toBe("miss");++ // Wait for index to be populated+ await new Promise(resolve => setTimeout(resolve, 20000));++ // Second scrape with same maxAge should hit cache+ const response2 = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ url,+ timeout: scrapeTimeout,+ maxAge: 3600000, // 1 hour in milliseconds+ });++ expect(response2.status).toBe(200);+ expect(response2.body.success).toBe(true);+ expect(response2.body.data).toBeDefined();+ expect(response2.body.data.metadata.cacheState).toBe("hit");+ expect(response2.body.data.metadata.cachedAt).toBeDefined();+ }, scrapeTimeout * 2 + 20000);+ } else {+ it("mocked", () => {+ expect(true).toBe(true);+ });+ }+});\ No newline at end of filediff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 843bddc7e..90b4b5eb1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -296,7 +296,7 @@ const baseScrapeOptions = zuseMock: z.string().optional(),blockAds: z.boolean().default(true),proxy: z.enum(["basic", "stealth", "auto"]).optional(),- maxAge: z.number().int().gte(0).safe().default(0),+ maxAge: z.number().int().gte(0).safe().default(14400000),storeInCache: z.boolean().default(true),// @deprecated__searchPreviewToken: z.string().optional(),diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 1718fbc02..50ac88839 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -92,6 +92,7 @@ function execForward(fancyName: string, command: string): Promise<void> {const api = execForward("api", "pnpm start:production:nobuild");const worker = execForward("worker", "pnpm worker:production");+ const indexWorker = execForward("indexWorker", "pnpm index-worker:production");try {await Promise.race([@@ -107,14 +108,17 @@ function execForward(fancyName: string, command: string): Promise<void> {cmd,api,worker,+ indexWorker,]);} finally {console.log("=== Tearing down API and Worker...");exec("pkill -f 'queue-worker.js'");exec("pkill -f 'index.js'");+ exec("pkill -f 'index-worker.js'");await Promise.all([api,worker,+ indexWorker,]);}