Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/sb): enforce timeout (FIR-980) (#1183)
Gergő Móricz1491b5b14164
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 0c53edb88..41326face 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -34,7 +34,7 @@ describe("Scrape tests", () => {expect(response.body.data.markdown).toBe("this is fake data coming from the mocking system!",);- });+ }, 10000);describe("Ad blocking (f-e dependant)", () => {it.concurrent("blocks ads by default", async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.tsindex 38c43878d..39805a2fd 100644--- a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts@@ -3,7 +3,7 @@ import { Meta } from "../..";import { EngineScrapeResult } from "..";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { AxiosError, type AxiosResponse } from "axios";-import { EngineError } from "../../error";+import { EngineError, TimeoutError } from "../../error";const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);@@ -17,23 +17,26 @@ export function scrapeURLWithScrapingBee(let response: AxiosResponse<any>;const timeout = (timeToRun ?? 300000) + meta.options.waitFor;try {- response = await client.get({- url: meta.url,- params: {- timeout,- wait_browser: wait_browser,- wait: meta.options.waitFor,- transparent_status_code: true,- json_response: true,- screenshot: meta.options.formats.includes("screenshot"),- screenshot_full_page: meta.options.formats.includes(- "screenshot@fullPage",- ),- },- headers: {- "ScrapingService-Request": "TRUE", // this is sent to the page, not to ScrapingBee - mogery- },- });+ response = await Promise.race<AxiosResponse<any>>([+ client.get({+ url: meta.url,+ params: {+ timeout,+ wait_browser: wait_browser,+ wait: meta.options.waitFor,+ transparent_status_code: true,+ json_response: true,+ screenshot: meta.options.formats.includes("screenshot"),+ screenshot_full_page: meta.options.formats.includes(+ "screenshot@fullPage",+ ),+ },+ headers: {+ "ScrapingService-Request": "TRUE", // this is sent to the page, not to ScrapingBee - mogery+ },+ }),+ new Promise((_, reject) => setTimeout(() => reject(new TimeoutError("ScrapingBee timed out")), timeout + 5000)),+ ]);} catch (error) {if (error instanceof AxiosError && error.response !== undefined) {response = error.response;