Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

skipTlsVerification on robots.txt scraping

Thomas Kosmasacde353e5652
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index c8e449f01..0000b6fea 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -78,7 +78,7 @@ export async function crawlController(
const crawler = crawlToCrawler(id, sc);
try {
- sc.robots = await crawler.getRobotsTxt();
+ sc.robots = await crawler.getRobotsTxt(pageOptions.skipTlsVerification);
} catch (e) {
Logger.debug(
`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 009a5933d..72a49fd87 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -9,7 +9,7 @@ import robotsParser from "robots-parser";
import { getURLDepth } from "./utils/maxDepthUtils";
import { axiosTimeout } from "../../../src/lib/timeout";
import { Logger } from "../../../src/lib/logger";
-
+import https from "https";
export class WebCrawler {
private jobId: string;
private initialUrl: string;
@@ -145,8 +145,14 @@ export class WebCrawler {
.slice(0, limit);
}
- public async getRobotsTxt(): Promise<string> {
- const response = await axios.get(this.robotsTxtUrl, { timeout: axiosTimeout });
+ public async getRobotsTxt(skipTlsVerification = false): Promise<string> {
+ let extraArgs = {};
+ if(skipTlsVerification) {
+ extraArgs["httpsAgent"] = new https.Agent({
+ rejectUnauthorized: false
+ });
+ }
+ const response = await axios.get(this.robotsTxtUrl, { timeout: axiosTimeout, ...extraArgs });
return response.data;
}