Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Temporarily comment out TLS client engine options in the scraper
rafaelmmiller1da56fe21956
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 6f2745879..c72e86b49 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -26,8 +26,8 @@ export type Engine =| "fire-engine(retry);chrome-cdp;stealth"| "fire-engine;playwright"| "fire-engine;playwright;stealth"- | "fire-engine;tlsclient"- | "fire-engine;tlsclient;stealth"+ // | "fire-engine;tlsclient"+ // | "fire-engine;tlsclient;stealth"| "playwright"| "fetch"| "pdf"@@ -167,9 +167,9 @@ const engineMRTs: {"fire-engine;playwright;stealth": meta =>fireEngineMaxReasonableTime(meta, "playwright"),// "fire-engine;tlsclient": meta =>- // fireEngineMaxReasonableTime(meta, "tlsclient"),+ // fireEngineMaxReasonableTime(meta, "tlsclient"),// "fire-engine;tlsclient;stealth": meta =>- // fireEngineMaxReasonableTime(meta, "tlsclient"),+ // fireEngineMaxReasonableTime(meta, "tlsclient"),playwright: playwrightMaxReasonableTime,fetch: fetchMaxReasonableTime,pdf: pdfMaxReasonableTime,@@ -348,42 +348,42 @@ export const engineOptions: {},quality: 20,},- "fire-engine;tlsclient": {- features: {- actions: false,- waitFor: false,- screenshot: false,- "screenshot@fullScreen": false,- pdf: false,- docx: false,- atsv: true,- location: true,- mobile: false,- skipTlsVerification: true,- useFastMode: true,- stealthProxy: false,- disableAdblock: false,- },- quality: 10,- },- "fire-engine;tlsclient;stealth": {- features: {- actions: false,- waitFor: false,- screenshot: false,- "screenshot@fullScreen": false,- pdf: false,- docx: false,- atsv: true,- location: true,- mobile: false,- skipTlsVerification: true,- useFastMode: true,- stealthProxy: true,- disableAdblock: false,- },- quality: -15,- },+ // "fire-engine;tlsclient": {+ // features: {+ // actions: false,+ // waitFor: false,+ // screenshot: false,+ // "screenshot@fullScreen": false,+ // pdf: false,+ // docx: false,+ // atsv: true,+ // location: true,+ // mobile: false,+ // skipTlsVerification: true,+ // useFastMode: true,+ // stealthProxy: false,+ // disableAdblock: false,+ // },+ // quality: 10,+ // },+ // "fire-engine;tlsclient;stealth": {+ // features: {+ // actions: false,+ // waitFor: false,+ // screenshot: false,+ // "screenshot@fullScreen": false,+ // pdf: false,+ // docx: false,+ // atsv: true,+ // location: true,+ // mobile: false,+ // skipTlsVerification: true,+ // useFastMode: true,+ // stealthProxy: true,+ // disableAdblock: false,+ // },+ // quality: -15,+ // },fetch: {features: {actions: false,