Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(test): skip robotsUserAgent test in self-hosted CI (#3376)
firecrawl-spring[bot]12bcbbf8e68f
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 15de2f77e..287b28e9e 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -128,8 +128,7 @@ describe("Crawl tests", () => {expect(results.success).toBe(true);if (results.success) {for (const page of results.data) {- const pageUrl =- page.metadata.url ?? page.metadata.sourceURL ?? base;+ const pageUrl = page.metadata.url ?? page.metadata.sourceURL ?? base;const normalized = normalizeUrlForCompare(pageUrl);expect(normalized === baseNormalized || sitemapUrls.has(normalized),@@ -529,7 +528,7 @@ describe("Crawl tests", () => {10 * scrapeTimeout,);- concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ concurrentIf(TEST_PRODUCTION)("accepts robotsUserAgent parameter",async () => {const robotsIdentity = await idmux({