Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add waitFor of 5000ms for changeTracking format
Devin AI9ab2a2663613
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex b9f1401e2..ebb311f9c 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -95,6 +95,23 @@ describe("Scrape tests", () => {expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();}, 30000);++ it.concurrent("enforces a minimum waitFor of 5000ms", async () => {+ const response1 = await scrape({+ url: "https://example.com",+ formats: ["markdown", "changeTracking"],+ });++ expect(response1.changeTracking).toBeDefined();++ const response2 = await scrape({+ url: "https://example.com",+ formats: ["markdown", "changeTracking"],+ waitFor: 1000,+ });++ expect(response2.changeTracking).toBeDefined();+ }, 30000);it.concurrent("includes git diff when requested", async () => {const response = await scrape({diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex cedd275e2..1408dde12 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -390,6 +390,9 @@ export async function scrapeURL(options: ScrapeOptions,internalOptions: InternalOptions,): Promise<ScrapeUrlResponse> {+ if (options.formats.includes("changeTracking") && options.waitFor < 5000) {+ options.waitFor = 5000;+ }const meta = await buildMetaObject(id, url, options, internalOptions);try {while (true) {