Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: fix minor inconveniences
Gergő Móricz2c5723175665
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex a28b469ca..031dab909 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -57,15 +57,16 @@ describe("Scrape tests", () => {expect(response.markdown).toContain("Firecrawl");}, scrapeTimeout);- it.concurrent("allows waitFor when it's exactly half of timeout", async () => {- const response = await scrape({- url: "http://firecrawl.dev",- waitFor: 7500,- timeout: 15000,- }, identity);-- expect(response.markdown).toContain("Firecrawl");- }, scrapeTimeout);+ // TODO: does not make sense. reevaluate - mogery+ // it.concurrent("allows waitFor when it's exactly half of timeout", async () => {+ // const response = await scrape({+ // url: "http://firecrawl.dev",+ // waitFor: 7500,+ // timeout: 15000,+ // }, identity);++ // expect(response.markdown).toContain("Firecrawl");+ // }, scrapeTimeout);it.concurrent("rejects waitFor when it exceeds half of timeout", async () => {const raw = await scrapeRaw({diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex cc1a1e4aa..b4e16860e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -135,8 +135,6 @@ export async function fireEngineCheckStatus(abort?: AbortSignal,production = true,): Promise<FireEngineCheckStatusSuccess> {- const x = Date.now();- meta.logger.debug("Checking status (" + x + ")");let status = await robustFetch({url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,method: "GET",@@ -145,7 +143,6 @@ export async function fireEngineCheckStatus(mock,abort,});- meta.logger.debug("Status came back (" + x + ")");// Fire-engine now saves the content to GCSif (!status.content && status.docUrl) {