Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add waitFor validation rule to enforce waitFor <= timeout/2 (#1751)

devin-ai-integration[bot]251e42c2a3f6
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 9d1e9d81b..69ffc8492 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -74,7 +74,7 @@ export type Identity = {
// Scrape API
// =========================================
-async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {
+export async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {
return await request(TEST_URL)
.post("/v1/scrape")
.set("Authorization", `Bearer ${identity.apiKey}`)
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index cbcea150e..a8e0b1f49 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -1,4 +1,4 @@
-import { scrape, scrapeStatus, scrapeWithFailure, scrapeTimeout, indexCooldown, idmux, Identity } from "./lib";
+import { scrape, scrapeWithFailure, scrapeStatus, scrapeTimeout, indexCooldown, idmux, Identity, scrapeRaw } from "./lib";
import crypto from "crypto";
let identity: Identity;
@@ -46,6 +46,70 @@ describe("Scrape tests", () => {
expect(response.markdown).toContain("Firecrawl");
}, scrapeTimeout);
+ describe("waitFor validation", () => {
+ it.concurrent("allows waitFor when it's less than half of timeout", async () => {
+ const response = await scrape({
+ url: "http://firecrawl.dev",
+ waitFor: 5000,
+ timeout: 15000,
+ }, identity);
+
+ expect(response.markdown).toContain("Firecrawl");
+ }, scrapeTimeout);
+
+ it.concurrent("allows waitFor when it's exactly half of timeout", async () => {
+ const response = await scrape({
+ url: "http://firecrawl.dev",
+ waitFor: 7500,
+ timeout: 15000,
+ }, identity);
+
+ expect(response.markdown).toContain("Firecrawl");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects waitFor when it exceeds half of timeout", async () => {
+ const raw = await scrapeRaw({
+ url: "http://firecrawl.dev",
+ waitFor: 8000,
+ timeout: 15000,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ expect(raw.body.details).toBeDefined();
+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects waitFor when it equals timeout", async () => {
+ const raw = await scrapeRaw({
+ url: "http://firecrawl.dev",
+ waitFor: 15000,
+ timeout: 15000,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ expect(raw.body.details).toBeDefined();
+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects waitFor when it exceeds timeout", async () => {
+ const raw = await scrapeRaw({
+ url: "http://firecrawl.dev",
+ waitFor: 20000,
+ timeout: 15000,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ expect(raw.body.details).toBeDefined();
+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");
+ }, scrapeTimeout);
+ });
+
it.concurrent("works with Punycode domains", async () => {
await scrape({
url: "http://xn--1lqv92a901a.xn--ses554g/",
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 82a3e76b4..ec5660dde 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -355,6 +355,19 @@ const fire1Refine = (obj) => {
const fire1RefineOpts = {
message: "You may only specify the FIRE-1 model in agent or jsonOptions.agent, but not both.",
};
+const waitForRefine = (obj) => {
+ if (obj.waitFor && obj.timeout) {
+ if (typeof obj.timeout !== 'number' || obj.timeout <= 0) {
+ return false;
+ }
+ return obj.waitFor <= obj.timeout / 2;
+ }
+ return true;
+};
+const waitForRefineOpts = {
+ message: "waitFor must not exceed half of timeout",
+ path: ["waitFor"],
+};
const extractRefine = (obj) => {
const hasExtractFormat = obj.formats?.includes("extract");
const hasExtractOptions = obj.extract !== undefined;
@@ -447,6 +460,7 @@ export const scrapeOptions = baseScrapeOptions
)
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
@@ -531,6 +545,10 @@ export const extractV1Options = z
(x) => (x.scrapeOptions ? fire1Refine(x.scrapeOptions) : true),
fire1RefineOpts,
)
+ .refine(
+ (x) => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),
+ waitForRefineOpts,
+ )
.transform((x) => ({
...x,
scrapeOptions: x.scrapeOptions
@@ -565,6 +583,7 @@ export const scrapeRequestSchema = baseScrapeOptions
.strict(strictMessage)
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
@@ -604,6 +623,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions
.strict(strictMessage)
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
@@ -620,6 +640,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
.strict(strictMessage)
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;
@@ -672,6 +693,7 @@ export const crawlRequestSchema = crawlerOptions
.strict(strictMessage)
.refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
.refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)
+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.refine(
(data) => {
const urlDepth = getURLDepth(data.url);
@@ -1299,6 +1321,7 @@ export const searchRequestSchema = z
)
.refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
.refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)
+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform((x) => ({
...x,
scrapeOptions: extractTransform(x.scrapeOptions),