Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Index testing improvements (FIR-2214) (#1637)
Gergő Móricz1de0ae392cc9
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 7402f13c6..759f20cad 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -160,6 +160,351 @@ describe("Scrape tests", () => {expect(response4.metadata.cacheState).toBe("miss");}, 150000 + 2 * 17000);++ it.concurrent("respects screenshot", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const response1 = await scrape({+ url,+ formats: ["screenshot"],+ timeout: 60000,+ });++ expect(response1.screenshot).toBeDefined();++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ formats: ["screenshot"],+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response2.screenshot).toBe(response1.screenshot);++ const response3 = await scrape({+ url,+ formats: ["screenshot@fullPage"],+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response3.screenshot).not.toBe(response1.screenshot);+ expect(response3.metadata.cacheState).toBe("miss");+ }, 207000);++ it.concurrent("respects screenshot@fullPage", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const response1 = await scrape({+ url,+ formats: ["screenshot@fullPage"],+ timeout: 60000,+ });++ expect(response1.screenshot).toBeDefined();++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ formats: ["screenshot@fullPage"],+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response2.screenshot).toBe(response1.screenshot);++ const response3 = await scrape({+ url,+ formats: ["screenshot"],+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response3.screenshot).not.toBe(response1.screenshot);+ expect(response3.metadata.cacheState).toBe("miss");+ }, 207000);++ it.concurrent("respects changeTracking", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ await scrape({+ url,+ formats: ["markdown", "changeTracking"],+ timeout: 60000,+ });++ const response1 = await scrape({+ url,+ formats: ["markdown", "changeTracking"],+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response1.metadata.cacheState).not.toBeDefined();++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ formats: ["markdown"],+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response2.metadata.cacheState).toBe("hit");+ }, 147000);++ it.concurrent("respects headers", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ await scrape({+ url,+ headers: {+ "X-Test": "test",+ },+ timeout: 60000,+ });++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response.metadata.cacheState).toBe("miss");+ }, 147000);++ it.concurrent("respects mobile", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ await scrape({+ url,+ timeout: 60000,+ });++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response1 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ mobile: true,+ });++ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ timeout: 60000,+ maxAge: 180000,+ mobile: true,+ });++ expect(response2.metadata.cacheState).toBe("hit");+ }, 224000);++ it.concurrent("respects actions", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const response1 = await scrape({+ url,+ timeout: 60000,+ maxAge: 60000,+ actions: [{+ "type": "wait",+ "milliseconds": 1000,+ }]+ });++ expect(response1.metadata.cacheState).not.toBeDefined();++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response2.metadata.cacheState).toBe("miss");+ }, 147000);++ it.concurrent("respects location", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ await scrape({+ url,+ timeout: 60000,+ });++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response1 = await scrape({+ url,+ location: { country: "BR" },+ maxAge: 120000,+ timeout: 60000,+ });++ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ location: { country: "BR" },+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response2.metadata.cacheState).toBe("hit");+ }, 224000);++ it.concurrent("respects blockAds", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ await scrape({+ url,+ blockAds: true,+ timeout: 60000,+ });++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response0 = await scrape({+ url,+ blockAds: true,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response0.metadata.cacheState).toBe("hit");++ const response1 = await scrape({+ url,+ blockAds: false,+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ blockAds: false,+ timeout: 60000,+ maxAge: 240000,+ });++ expect(response2.metadata.cacheState).toBe("hit");+ }, 284000);++ it.concurrent("respects proxy: stealth", async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const response1 = await scrape({+ url,+ proxy: "stealth",+ timeout: 60000,+ maxAge: 60000,+ });++ expect(response1.metadata.proxyUsed).toBe("stealth");+ expect(response1.metadata.cacheState).not.toBeDefined();++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response2.metadata.cacheState).toBe("hit");++ const response3 = await scrape({+ url,+ proxy: "stealth",+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response3.metadata.cacheState).not.toBeDefined();+ }, 207000);++ it.concurrent("works properly on pages returning 200", async () => {+ const id = crypto.randomUUID();+ const url = "https://httpstat.us/200?testId=" + id;++ await scrape({+ url,+ timeout: 60000,+ });++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response.metadata.cacheState).toBe("hit");+ }, 147000);++ it.concurrent("works properly on pages returning errors", async () => {+ const id = crypto.randomUUID();+ const url = "https://httpstat.us/404?testId=" + id;++ await scrape({+ url,+ timeout: 60000,+ });++ const response1 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response1.metadata.cacheState).toBe("miss");++ const response2 = await scrape({+ url,+ timeout: 60000,+ maxAge: 180000,+ });++ expect(response2.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response3 = await scrape({+ url,+ timeout: 60000,+ maxAge: 240000,+ });++ expect(response3.metadata.cacheState).toBe("hit");+ }, 284000);});describe("Change Tracking format", () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 684682581..57edbf75c 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -191,7 +191,7 @@ export const engineOptions: {skipTlsVerification: true,useFastMode: true,stealthProxy: false,- disableAdblock: false,+ disableAdblock: true,},quality: 999, // index should always be tried second ? - MG},