Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
skipTlsVerification
Thomas Kosmasbd55464b52e2
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 033de6e0c..2bf0dc346 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -117,6 +117,7 @@ export const scrapeOptions = z.object({}).transform(val => val ? val.toUpperCase() : 'US')}).optional(),+ skipTlsVerification: z.boolean().default(false),}).strict(strictMessage)@@ -433,6 +434,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {parsePDF: x.parsePDF,actions: x.actions as Action[], // no strict null checking grrrr - mogerygeolocation: x.geolocation,+ skipTlsVerification: x.skipTlsVerification};}diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex ca6142ec0..8aa1d004a 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -54,6 +54,7 @@ export type PageOptions = {geolocation?: {country?: string;};+ skipTlsVerification?: boolean;};export type ExtractorOptions = {diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.tsindex 5285a9f42..1817a07b8 100644--- a/apps/api/src/scraper/WebScraper/index.ts+++ b/apps/api/src/scraper/WebScraper/index.ts@@ -594,6 +594,7 @@ export class WebScraperDataProvider {atsv: options.pageOptions?.atsv ?? false,actions: options.pageOptions?.actions ?? undefined,geolocation: options.pageOptions?.geolocation ?? undefined,+ skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,};this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };this.replaceAllPathsWithAbsolutePaths =diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.tsindex d1dafd5f7..3bbd74eb1 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts@@ -28,7 +28,7 @@ export async function scrapWithFireEngine({waitFor = 0,screenshot = false,fullPageScreenshot = false,- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" } },+ pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false },fireEngineOptions = {},headers,options,@@ -40,7 +40,7 @@ export async function scrapWithFireEngine({waitFor?: number;screenshot?: boolean;fullPageScreenshot?: boolean;- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string } };+ pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean };fireEngineOptions?: FireEngineOptions;headers?: Record<string, string>;options?: any;@@ -119,6 +119,7 @@ export async function scrapWithFireEngine({atsv: pageOptions?.atsv ?? false,scrollXPaths: pageOptions?.scrollXPaths ?? [],geolocation: pageOptions?.geolocation,+ skipTlsVerification: pageOptions?.skipTlsVerification ?? false,actions: actions,},{diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex b4141dc19..cd76793c3 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -157,6 +157,7 @@ export async function scrapSingleUrl(atsv: pageOptions.atsv ?? false,actions: pageOptions.actions ?? undefined,geolocation: pageOptions.geolocation ?? undefined,+ skipTlsVerification: pageOptions.skipTlsVerification ?? false,}if (extractorOptions) {