Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Remove TLS client engine options from the scraper temporarily
rafaelmmiller0a60985040b0
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 6c7d4c9d3..6f2745879 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -5,7 +5,7 @@ import {fireEngineMaxReasonableTime,scrapeURLWithFireEngineChromeCDP,scrapeURLWithFireEnginePlaywright,- scrapeURLWithFireEngineTLSClient,+ // scrapeURLWithFireEngineTLSClient,} from "./fire-engine";import { pdfMaxReasonableTime, scrapePDF } from "./pdf";import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";@@ -52,8 +52,8 @@ export const engines: Engine[] = ["fire-engine(retry);chrome-cdp;stealth" as const,"fire-engine;playwright" as const,"fire-engine;playwright;stealth" as const,- "fire-engine;tlsclient" as const,- "fire-engine;tlsclient;stealth" as const,+ // "fire-engine;tlsclient" as const,+ // "fire-engine;tlsclient;stealth" as const,]: []),...(usePlaywright ? ["playwright" as const] : []),@@ -141,8 +141,8 @@ const engineHandlers: {"fire-engine(retry);chrome-cdp;stealth": scrapeURLWithFireEngineChromeCDP,"fire-engine;playwright": scrapeURLWithFireEnginePlaywright,"fire-engine;playwright;stealth": scrapeURLWithFireEnginePlaywright,- "fire-engine;tlsclient": scrapeURLWithFireEngineTLSClient,- "fire-engine;tlsclient;stealth": scrapeURLWithFireEngineTLSClient,+ // "fire-engine;tlsclient": scrapeURLWithFireEngineTLSClient,+ // "fire-engine;tlsclient;stealth": scrapeURLWithFireEngineTLSClient,playwright: scrapeURLWithPlaywright,fetch: scrapeURLWithFetch,pdf: scrapePDF,@@ -166,10 +166,10 @@ const engineMRTs: {fireEngineMaxReasonableTime(meta, "playwright"),"fire-engine;playwright;stealth": meta =>fireEngineMaxReasonableTime(meta, "playwright"),- "fire-engine;tlsclient": meta =>- fireEngineMaxReasonableTime(meta, "tlsclient"),- "fire-engine;tlsclient;stealth": meta =>- fireEngineMaxReasonableTime(meta, "tlsclient"),+ // "fire-engine;tlsclient": meta =>+ // fireEngineMaxReasonableTime(meta, "tlsclient"),+ // "fire-engine;tlsclient;stealth": meta =>+ // fireEngineMaxReasonableTime(meta, "tlsclient"),playwright: playwrightMaxReasonableTime,fetch: fetchMaxReasonableTime,pdf: pdfMaxReasonableTime,