Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
// @ts-check
import mdx from "@astrojs/mdx";
import sitemap from "@astrojs/sitemap";
import { defineConfig } from "astro/config";
// TEST_WEBSITE_URL required for deployment
const SITE_URL =
(process.env.VERCEL_URL && `https://${process.env.VERCEL_URL}`) ||
process.env.TEST_WEBSITE_URL ||
process.env.TEST_SUITE_WEBSITE ||
"http://127.0.0.1:4321";
export default defineConfig({
site: SITE_URL,
output: 'static',
integrations: [
mdx(),
// Keep the robots.txt fixture out of the sitemap so that only crawls
// that start at /robots-test/ see it.
sitemap({ filter: page => !page.includes("/robots-test/") }),
],
});