Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolas9512d81e05f4
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 96e1ee78a..1dff52360 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -36,6 +36,8 @@ export async function extractController(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>) {+ const selfHosted = process.env.USE_DB_AUTHENTICATION !== "true";+req.body = extractRequestSchema.parse(req.body);const id = crypto.randomUUID();@@ -65,7 +67,8 @@ export async function extractController(allowExternalLinks,origin: req.body.origin,limit: req.body.limit,- ignoreSitemap: true,+ // If we're self-hosted, we don't want to ignore the sitemap, due to our fire-engine mapping+ ignoreSitemap: !selfHosted ? true : false,includeMetadata: true,includeSubdomains: req.body.includeSubdomains,});