Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add waitFor validation rule to enforce waitFor <= timeout/2 (#1751)
devin-ai-integration[bot]251e42c2a3f6
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 9d1e9d81b..69ffc8492 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -74,7 +74,7 @@ export type Identity = {// Scrape API// =========================================-async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {+export async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/scrape").set("Authorization", `Bearer ${identity.apiKey}`)diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex cbcea150e..a8e0b1f49 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -1,4 +1,4 @@-import { scrape, scrapeStatus, scrapeWithFailure, scrapeTimeout, indexCooldown, idmux, Identity } from "./lib";+import { scrape, scrapeWithFailure, scrapeStatus, scrapeTimeout, indexCooldown, idmux, Identity, scrapeRaw } from "./lib";import crypto from "crypto";let identity: Identity;@@ -46,6 +46,70 @@ describe("Scrape tests", () => {expect(response.markdown).toContain("Firecrawl");}, scrapeTimeout);+ describe("waitFor validation", () => {+ it.concurrent("allows waitFor when it's less than half of timeout", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev",+ waitFor: 5000,+ timeout: 15000,+ }, identity);++ expect(response.markdown).toContain("Firecrawl");+ }, scrapeTimeout);++ it.concurrent("allows waitFor when it's exactly half of timeout", async () => {+ const response = await scrape({+ url: "http://firecrawl.dev",+ waitFor: 7500,+ timeout: 15000,+ }, identity);++ expect(response.markdown).toContain("Firecrawl");+ }, scrapeTimeout);++ it.concurrent("rejects waitFor when it exceeds half of timeout", async () => {+ const raw = await scrapeRaw({+ url: "http://firecrawl.dev",+ waitFor: 8000,+ timeout: 15000,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ expect(raw.body.details).toBeDefined();+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");+ }, scrapeTimeout);++ it.concurrent("rejects waitFor when it equals timeout", async () => {+ const raw = await scrapeRaw({+ url: "http://firecrawl.dev",+ waitFor: 15000,+ timeout: 15000,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ expect(raw.body.details).toBeDefined();+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");+ }, scrapeTimeout);++ it.concurrent("rejects waitFor when it exceeds timeout", async () => {+ const raw = await scrapeRaw({+ url: "http://firecrawl.dev",+ waitFor: 20000,+ timeout: 15000,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ expect(raw.body.details).toBeDefined();+ expect(JSON.stringify(raw.body.details)).toContain("waitFor must not exceed half of timeout");+ }, scrapeTimeout);+ });+it.concurrent("works with Punycode domains", async () => {await scrape({url: "http://xn--1lqv92a901a.xn--ses554g/",diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 82a3e76b4..ec5660dde 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -355,6 +355,19 @@ const fire1Refine = (obj) => {const fire1RefineOpts = {message: "You may only specify the FIRE-1 model in agent or jsonOptions.agent, but not both.",};+const waitForRefine = (obj) => {+ if (obj.waitFor && obj.timeout) {+ if (typeof obj.timeout !== 'number' || obj.timeout <= 0) {+ return false;+ }+ return obj.waitFor <= obj.timeout / 2;+ }+ return true;+};+const waitForRefineOpts = {+ message: "waitFor must not exceed half of timeout",+ path: ["waitFor"],+};const extractRefine = (obj) => {const hasExtractFormat = obj.formats?.includes("extract");const hasExtractOptions = obj.extract !== undefined;@@ -447,6 +460,7 @@ export const scrapeOptions = baseScrapeOptions).refine(extractRefine, extractRefineOpts).refine(fire1Refine, fire1RefineOpts)+ .refine(waitForRefine, waitForRefineOpts).transform(extractTransform);export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;@@ -531,6 +545,10 @@ export const extractV1Options = z(x) => (x.scrapeOptions ? fire1Refine(x.scrapeOptions) : true),fire1RefineOpts,)+ .refine(+ (x) => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),+ waitForRefineOpts,+ ).transform((x) => ({...x,scrapeOptions: x.scrapeOptions@@ -565,6 +583,7 @@ export const scrapeRequestSchema = baseScrapeOptions.strict(strictMessage).refine(extractRefine, extractRefineOpts).refine(fire1Refine, fire1RefineOpts)+ .refine(waitForRefine, waitForRefineOpts).transform(extractTransform);export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;@@ -604,6 +623,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions.strict(strictMessage).refine(extractRefine, extractRefineOpts).refine(fire1Refine, fire1RefineOpts)+ .refine(waitForRefine, waitForRefineOpts).transform(extractTransform);export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions@@ -620,6 +640,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions.strict(strictMessage).refine(extractRefine, extractRefineOpts).refine(fire1Refine, fire1RefineOpts)+ .refine(waitForRefine, waitForRefineOpts).transform(extractTransform);export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;@@ -672,6 +693,7 @@ export const crawlRequestSchema = crawlerOptions.strict(strictMessage).refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts).refine((data) => {const urlDepth = getURLDepth(data.url);@@ -1299,6 +1321,7 @@ export const searchRequestSchema = z).refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform((x) => ({...x,scrapeOptions: extractTransform(x.scrapeOptions),