Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
skipTlsVerification on robots.txt scraping
Thomas Kosmasacde353e5652
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex c8e449f01..0000b6fea 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -78,7 +78,7 @@ export async function crawlController(const crawler = crawlToCrawler(id, sc);try {- sc.robots = await crawler.getRobotsTxt();+ sc.robots = await crawler.getRobotsTxt(pageOptions.skipTlsVerification);} catch (e) {Logger.debug(`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 009a5933d..72a49fd87 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -9,7 +9,7 @@ import robotsParser from "robots-parser";import { getURLDepth } from "./utils/maxDepthUtils";import { axiosTimeout } from "../../../src/lib/timeout";import { Logger } from "../../../src/lib/logger";-+import https from "https";export class WebCrawler {private jobId: string;private initialUrl: string;@@ -145,8 +145,14 @@ export class WebCrawler {.slice(0, limit);}- public async getRobotsTxt(): Promise<string> {- const response = await axios.get(this.robotsTxtUrl, { timeout: axiosTimeout });+ public async getRobotsTxt(skipTlsVerification = false): Promise<string> {+ let extraArgs = {};+ if(skipTlsVerification) {+ extraArgs["httpsAgent"] = new https.Agent({+ rejectUnauthorized: false+ });+ }+ const response = await axios.get(this.robotsTxtUrl, { timeout: axiosTimeout, ...extraArgs });return response.data;}