Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(changeTracking): support tags (FIR-1940) (#1631)
GergΕ MΓ³ricz34a18a2d2feb
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex f135c0b5c..7402f13c6 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -10,46 +10,51 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",useMock: "mocking-works-properly",+ timeout: 60000,});expect(response.markdown).toBe("this is fake data coming from the mocking system!",);- }, 30000);+ }, 60000);it.concurrent("works", async () => {const response = await scrape({- url: "http://firecrawl.dev"+ url: "http://firecrawl.dev",+ timeout: 60000,});expect(response.markdown).toContain("Firecrawl");- }, 30000);+ }, 60000);it.concurrent("handles non-UTF-8 encodings", async () => {const response = await scrape({url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",+ timeout: 60000,});expect(response.markdown).toContain("γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ");- }, 30000);+ }, 60000);if (process.env.TEST_SUITE_SELF_HOSTED && process.env.PROXY_SERVER) {it.concurrent("self-hosted proxy works", async () => {const response = await scrape({- url: "https://icanhazip.com"+ url: "https://icanhazip.com",+ timeout: 60000,});expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);- }, 30000);+ }, 70000);it.concurrent("self-hosted proxy works on playwright", async () => {const response = await scrape({url: "https://icanhazip.com",waitFor: 100,+ timeout: 60000,});expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);- }, 30000);+ }, 70000);}if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.PLAYWRIGHT_MICROSERVICE_URL) {@@ -57,10 +62,11 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",waitFor: 2000,+ timeout: 60000,});expect(response.markdown).toContain("Firecrawl");- }, 30000);+ }, 60000);}describe("JSON scrape support", () => {@@ -68,17 +74,19 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["rawHtml"],+ timeout: 60000,});const obj = JSON.parse(response.rawHtml!);expect(obj.id).toBe(1);- }, 30000);+ }, 70000);});if (!process.env.TEST_SUITE_SELF_HOSTED) {it.concurrent("scrape status works", async () => {const response = await scrape({- url: "http://firecrawl.dev"+ url: "http://firecrawl.dev",+ timeout: 60000,});expect(response.markdown).toContain("Firecrawl");@@ -88,7 +96,7 @@ describe("Scrape tests", () => {const status = await scrapeStatus(response.metadata.scrapeId!);expect(JSON.stringify(status)).toBe(JSON.stringify(response));- }, 60000);+ }, 70000);// describe("Ad blocking (f-e dependant)", () => {// it.concurrent("blocks ads by default", async () => {@@ -118,6 +126,7 @@ describe("Scrape tests", () => {url,maxAge: 120000,storeInCache: false,+ timeout: 60000,});expect(response1.metadata.cacheState).toBe("miss");@@ -127,6 +136,7 @@ describe("Scrape tests", () => {const response2 = await scrape({url,maxAge: 120000,+ timeout: 60000,});expect(response2.metadata.cacheState).toBe("miss");@@ -136,6 +146,7 @@ describe("Scrape tests", () => {const response3 = await scrape({url,maxAge: 120000,+ timeout: 60000,});expect(response3.metadata.cacheState).toBe("hit");@@ -144,6 +155,7 @@ describe("Scrape tests", () => {const response4 = await scrape({url,maxAge: 1,+ timeout: 60000,});expect(response4.metadata.cacheState).toBe("miss");@@ -155,11 +167,12 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://example.com",formats: ["markdown", "changeTracking"],+ timeout: 60000,});expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();- }, 30000);+ }, 60000);it.concurrent("includes git diff when requested", async () => {const response = await scrape({@@ -167,7 +180,8 @@ describe("Scrape tests", () => {formats: ["markdown", "changeTracking"],changeTrackingOptions: {modes: ["git-diff"]- }+ },+ timeout: 60000,});expect(response.changeTracking).toBeDefined();@@ -179,7 +193,7 @@ describe("Scrape tests", () => {expect(response.changeTracking?.diff?.json).toBeDefined();expect(response.changeTracking?.diff?.json.files).toBeInstanceOf(Array);}- }, 30000);+ }, 60000);it.concurrent("includes structured output when requested", async () => {const response = await scrape({@@ -188,7 +202,8 @@ describe("Scrape tests", () => {changeTrackingOptions: {modes: ["json"],prompt: "Summarize the changes between the previous and current content",- }+ },+ timeout: 60000,});expect(response.changeTracking).toBeDefined();@@ -197,7 +212,7 @@ describe("Scrape tests", () => {if (response.changeTracking?.changeStatus === "changed") {expect(response.changeTracking?.json).toBeDefined();}- }, 30000);+ }, 60000);it.concurrent("supports schema-based extraction for change tracking", async () => {const response = await scrape({@@ -221,7 +236,8 @@ describe("Scrape tests", () => {}}}- }+ },+ timeout: 60000,});expect(response.changeTracking).toBeDefined();@@ -238,7 +254,7 @@ describe("Scrape tests", () => {expect(response.changeTracking?.json.features).toHaveProperty("new");}}- }, 30000);+ }, 60000);it.concurrent("supports both git-diff and structured modes together", async () => {const response = await scrape({@@ -253,7 +269,8 @@ describe("Scrape tests", () => {changes: { type: "array", items: { type: "string" } }}}- }+ },+ timeout: 60000,});expect(response.changeTracking).toBeDefined();@@ -268,59 +285,99 @@ describe("Scrape tests", () => {expect(response.changeTracking?.json).toHaveProperty("summary");expect(response.changeTracking?.json).toHaveProperty("changes");}- }, 30000);+ }, 60000);++ it.concurrent("supports tags properly", async () => {+ const uuid1 = crypto.randomUUID();+ const uuid2 = crypto.randomUUID();++ const response1 = await scrape({+ url: "https://firecrawl.dev/",+ formats: ["markdown", "changeTracking"],+ changeTrackingOptions: { tag: uuid1 },+ timeout: 60000,+ });++ const response2 = await scrape({+ url: "https://firecrawl.dev/",+ formats: ["markdown", "changeTracking"],+ changeTrackingOptions: { tag: uuid2 },+ timeout: 60000,+ });++ expect(response1.changeTracking?.previousScrapeAt).toBeNull();+ expect(response1.changeTracking?.changeStatus).toBe("new");+ expect(response2.changeTracking?.previousScrapeAt).toBeNull();+ expect(response2.changeTracking?.changeStatus).toBe("new");++ const response3 = await scrape({+ url: "https://firecrawl.dev/",+ formats: ["markdown", "changeTracking"],+ changeTrackingOptions: { tag: uuid1 },+ timeout: 60000,+ });++ expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();+ expect(response3.changeTracking?.changeStatus).not.toBe("new");+ }, 180000);});describe("Location API (f-e dependant)", () => {it.concurrent("works without specifying an explicit location", async () => {await scrape({url: "https://iplocation.com",+ timeout: 60000,});- }, 30000);+ }, 70000);it.concurrent("works with country US", async () => {const response = await scrape({url: "https://iplocation.com",location: { country: "US" },+ timeout: 60000,});expect(response.markdown).toContain("| Country | United States |");- }, 30000);+ }, 70000);});describe("Screenshot (f-e/sb dependant)", () => {it.concurrent("screenshot format works", async () => {const response = await scrape({url: "http://firecrawl.dev",- formats: ["screenshot"]+ formats: ["screenshot"],+ timeout: 60000,});expect(typeof response.screenshot).toBe("string");- }, 30000);+ }, 70000);it.concurrent("screenshot@fullPage format works", async () => {const response = await scrape({url: "http://firecrawl.dev",- formats: ["screenshot@fullPage"]+ formats: ["screenshot@fullPage"],+ timeout: 60000,});expect(typeof response.screenshot).toBe("string");- }, 30000);+ }, 70000);});describe("Proxy API (f-e dependant)", () => {it.concurrent("undefined works", async () => {await scrape({url: "http://firecrawl.dev",+ timeout: 60000,});- }, 30000);+ }, 70000);it.concurrent("basic works", async () => {await scrape({url: "http://firecrawl.dev",proxy: "basic",+ timeout: 60000,});- }, 30000);+ }, 70000);it.concurrent("stealth works", async () => {await scrape({@@ -328,7 +385,7 @@ describe("Scrape tests", () => {proxy: "stealth",timeout: 120000,});- }, 130000);+ }, 140000);it.concurrent("auto works properly on non-stealth site", async () => {const res = await scrape({@@ -338,7 +395,7 @@ describe("Scrape tests", () => {});expect(res.metadata.proxyUsed).toBe("basic");- }, 130000);+ }, 140000);it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {const res = await scrape({@@ -348,7 +405,7 @@ describe("Scrape tests", () => {});expect(res.metadata.proxyUsed).toBe("stealth");- }, 130000);+ }, 140000);});describe("PDF (f-e dependant)", () => {@@ -406,6 +463,7 @@ describe("Scrape tests", () => {required: ["company_mission", "supports_sso", "is_open_source"],},},+ timeout: 60000,});expect(response).toHaveProperty("json");@@ -417,24 +475,26 @@ describe("Scrape tests", () => {expect(response.json).toHaveProperty("is_open_source");expect(response.json.is_open_source).toBe(true);expect(typeof response.json.is_open_source).toBe("boolean");- }, 30000);+ }, 60000);});}it.concurrent("sourceURL stays unnormalized", async () => {const response = await scrape({url: "https://firecrawl.dev/?pagewanted=all&et_blog",+ timeout: 60000,});expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");- }, 30000);+ }, 60000);it.concurrent("application/json content type is markdownified properly", async () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["markdown"],+ timeout: 60000,});expect(response.markdown).toContain("```json");- }, 30000);+ }, 60000);});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex c36821e48..ba68c9953 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -260,6 +260,7 @@ const baseScrapeOptions = zprompt: z.string().optional(),schema: z.any().optional(),modes: z.enum(["json", "git-diff"]).array().optional().default([]),+ tag: z.string().or(z.null()).default(null),}).optional(),mobile: z.boolean().default(false),diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 505480130..6284d9a6f 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -61,9 +61,10 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeif (meta.options.formats.includes("changeTracking")) {const start = Date.now();const res = await supabase_service- .rpc("diff_get_last_scrape_3", {+ .rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,i_url: document.metadata.sourceURL ?? meta.url,+ i_tag: meta.options.changeTrackingOptions?.tag ?? null,});const end = Date.now();if (end - start > 100) {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 262d4015d..b26a3a9a3 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -65,6 +65,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLocost_tracking: job.cost_tracking,pdf_num_pages: job.pdf_num_pages ?? null,credits_billed: job.credits_billed ?? null,+ change_tracking_tag: job.change_tracking_tag ?? null,};if (process.env.GCS_BUCKET_NAME) {@@ -145,6 +146,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLotokens_billed: job.tokens_billed,cost_tracking: job.cost_tracking,pdf_num_pages: job.pdf_num_pages,+ change_tracking_tag: job.change_tracking_tag ?? null,},};if (job.mode !== "single_urls") {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 2404e9171..4f025e38a 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1460,6 +1460,7 @@ async function processJob(job: Job & { id: string }, token: string) {cost_tracking: costTracking,pdf_num_pages: doc.metadata.numPages,credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,},true,job.data.internalOptions?.bypassBilling ?? false,@@ -1509,6 +1510,7 @@ async function processJob(job: Job & { id: string }, token: string) {cost_tracking: costTracking,pdf_num_pages: doc.metadata.numPages,credits_billed,+ change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null,}, false, job.data.internalOptions?.bypassBilling ?? false);}diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 2e18ba3d9..1d488ff01 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -102,6 +102,7 @@ export interface FirecrawlJob {cost_tracking?: CostTracking;pdf_num_pages?: number;credits_billed?: number | null;+ change_tracking_tag?: string | null;}export interface FirecrawlScrapeResponse {