Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test(api): respect local fixture availability in robots tests (#4544)
Gergő Móricz5ce5972bb06b
diff --git a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts b/apps/api/src/__tests__/snips/v2/scrape-robots.test.tsindex 9d43f98d5..b5ab41b40 100644--- a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts@@ -5,7 +5,7 @@ import { CrawlDenialError } from "../../../lib/error";import * as robots from "../../../lib/robots-txt";import { scrapeURL } from "../../../scraper/scrapeURL";import * as engines from "../../../scraper/scrapeURL/engines";-import { TEST_SUITE_WEBSITE } from "../lib";+import { ALLOW_TEST_SUITE_WEBSITE, itIf, TEST_SUITE_WEBSITE } from "../lib";import { scrapeTimeout } from "./lib";// Exercise the pipeline directly so team flags are covered in self-hosted@@ -53,7 +53,9 @@ describe("Scrape robots policy", () => {scrapeTimeout,);- it(+ // External proxies cannot reach the local test site. Keep the denial test+ // above unconditional because it must finish before any engine is invoked.+ itIf(ALLOW_TEST_SUITE_WEBSITE)("scrapes an allowed URL",async () => {vi.mocked(robots.fetchRobotsTxt).mockResolvedValue({@@ -70,7 +72,7 @@ describe("Scrape robots policy", () => {scrapeTimeout,);- it(+ itIf(ALLOW_TEST_SUITE_WEBSITE)("scrapes without checking robots when the team flag is disabled",async () => {const result = await runScrape(false);@@ -82,7 +84,7 @@ describe("Scrape robots policy", () => {scrapeTimeout,);- it(+ itIf(ALLOW_TEST_SUITE_WEBSITE)("allows scraping when robots.txt retrieval fails",async () => {vi.mocked(robots.fetchRobotsTxt).mockRejectedValue(