Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/batch/scrape): maxConcurrency field support when using ignoreInvalidURLs (#1705)

Gergő Móricz11f469488e4f
diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.ts
index 113cf651d..9a3f3cba6 100644
--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts
+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts
@@ -1,4 +1,4 @@
-import { batchScrape } from "./lib";
+import { batchScrape, scrapeTimeout } from "./lib";
describe("Batch scrape tests", () => {
it.concurrent("works", async () => {
@@ -8,7 +8,7 @@ describe("Batch scrape tests", () => {
expect(response.data[0]).toHaveProperty("markdown");
expect(response.data[0].markdown).toContain("Firecrawl");
- }, 180000);
+ }, scrapeTimeout);
if (!process.env.TEST_SUITE_SELF_HOSTED) {
describe("JSON format", () => {
@@ -55,5 +55,5 @@ describe("Batch scrape tests", () => {
});
expect(response.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
- }, 35000);
+ }, scrapeTimeout);
});
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 8903458a3..5f57776b9 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -1,4 +1,4 @@
-import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing } from "./lib";
+import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, scrapeTimeout } from "./lib";
import { describe, it, expect } from "@jest/globals";
describe("Crawl tests", () => {
@@ -7,7 +7,7 @@ describe("Crawl tests", () => {
url: "https://firecrawl.dev",
limit: 10,
});
- }, 120000);
+ }, 10 * scrapeTimeout);
it.concurrent("filters URLs properly", async () => {
const res = await crawl({
@@ -24,7 +24,7 @@ describe("Crawl tests", () => {
expect(url.pathname).toMatch(/^\/pricing$/);
}
}
- }, 120000);
+ }, 10 * scrapeTimeout);
it.concurrent("filters URLs properly when using regexOnFullURL", async () => {
const res = await crawl({
@@ -39,7 +39,7 @@ describe("Crawl tests", () => {
expect(res.completed).toBe(1);
expect(res.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/pricing");
}
- }, 120000);
+ }, 10 * scrapeTimeout);
it.concurrent("delay parameter works", async () => {
await crawl({
@@ -47,7 +47,7 @@ describe("Crawl tests", () => {
limit: 3,
delay: 5,
});
- }, 300000);
+ }, 3 * scrapeTimeout + 3 * 5000);
it.concurrent("ongoing crawls endpoint works", async () => {
const res = await asyncCrawl({
@@ -64,7 +64,7 @@ describe("Crawl tests", () => {
const ongoing2 = await crawlOngoing();
expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();
- }, 120000);
+ }, 3 * scrapeTimeout);
// TEMP: Flaky
// it.concurrent("discovers URLs properly when origin is not included", async () => {
@@ -112,7 +112,7 @@ describe("Crawl tests", () => {
if (res.success) {
expect(res.completed).toBeGreaterThan(0);
}
- }, 120000);
+ }, 5 * scrapeTimeout);
it.concurrent("crawlEntireDomain takes precedence over allowBackwardLinks", async () => {
const res = await crawl({
@@ -126,7 +126,7 @@ describe("Crawl tests", () => {
if (res.success) {
expect(res.completed).toBeGreaterThan(0);
}
- }, 120000);
+ }, 5 * scrapeTimeout);
it.concurrent("backward compatibility - allowBackwardLinks still works", async () => {
const res = await crawl({
@@ -139,5 +139,5 @@ describe("Crawl tests", () => {
if (res.success) {
expect(res.completed).toBeGreaterThan(0);
}
- }, 120000);
+ }, 5 * scrapeTimeout);
});
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index a21d9f312..6d0e28fd2 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -18,6 +18,10 @@ export const defaultIdentity: Identity = {
apiKey: process.env.TEST_API_KEY!,
};
+// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests
+export const scrapeTimeout = 75000;
+export const indexCooldown = 30000;
+
// =========================================
// Scrape API
// =========================================
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 77944e7f6..6c8c80d7e 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -1,10 +1,6 @@
-import { scrape, scrapeStatus, scrapeWithFailure } from "./lib";
+import { scrape, scrapeStatus, scrapeWithFailure, scrapeTimeout, indexCooldown } from "./lib";
import crypto from "crypto";
-// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests
-const scrapeTimeout = 75000;
-const indexCooldown = 30000;
-
describe("Scrape tests", () => {
it.concurrent("mocking works properly", async () => {
// depends on falsified mock mocking-works-properly
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 4682a9e9f..ab4e39343 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -610,6 +610,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
ignoreInvalidURLs: z.boolean().default(false),
+ maxConcurrency: z.number().positive().int().optional(),
})
.strict(strictMessage)
.refine(extractRefine, extractRefineOpts)