Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add waitFor of 5000ms for changeTracking format

Devin AI9ab2a2663613
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index b9f1401e2..ebb311f9c 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -95,6 +95,23 @@ describe("Scrape tests", () => {
expect(response.changeTracking).toBeDefined();
expect(response.changeTracking?.previousScrapeAt).not.toBeNull();
}, 30000);
+
+ it.concurrent("enforces a minimum waitFor of 5000ms", async () => {
+ const response1 = await scrape({
+ url: "https://example.com",
+ formats: ["markdown", "changeTracking"],
+ });
+
+ expect(response1.changeTracking).toBeDefined();
+
+ const response2 = await scrape({
+ url: "https://example.com",
+ formats: ["markdown", "changeTracking"],
+ waitFor: 1000,
+ });
+
+ expect(response2.changeTracking).toBeDefined();
+ }, 30000);
it.concurrent("includes git diff when requested", async () => {
const response = await scrape({
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index cedd275e2..1408dde12 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -390,6 +390,9 @@ export async function scrapeURL(
options: ScrapeOptions,
internalOptions: InternalOptions,
): Promise<ScrapeUrlResponse> {
+ if (options.formats.includes("changeTracking") && options.waitFor < 5000) {
+ options.waitFor = 5000;
+ }
const meta = await buildMetaObject(id, url, options, internalOptions);
try {
while (true) {