Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): reintroduce default timeout for simple queries (#1440)

Gergő Móricz4294face784e
diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.ts
index 1890b08bf..f3e9e585b 100644
--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts
+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts
@@ -8,7 +8,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0]).toHaveProperty("markdown");
expect(response.body.data[0].markdown).toContain("Firecrawl");
- }, 30000);
+ }, 180000);
if (!process.env.TEST_SUITE_SELF_HOSTED) {
describe("JSON format", () => {
@@ -45,7 +45,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0].json).toHaveProperty("is_open_source");
expect(response.body.data[0].json.is_open_source).toBe(true);
expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");
- }, 30000);
+ }, 180000);
});
}
});
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index e047d1cb6..cedd275e2 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -230,7 +230,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const timeToRun =
meta.options.timeout !== undefined
? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))
- : undefined;
+ : (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract)
+ ? Math.round(120000 / Math.min(fallbackList.length, 2))
+ : undefined;
for (const { engine, unsupportedFeatures } of fallbackList) {
meta.internalOptions.abort?.throwIfAborted();