Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): pdf + document engines not respecting skipTlsVerification flag
delong350b7f1eafedf
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex 8a8cfc8f4..195acb440 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -108,10 +108,14 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {proxyUsed = meta.documentPrefetch.proxyUsed;} else {// Fetch the document normally- const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- });+ const result = await fetchFileToBuffer(+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ );response = result.response;buffer = result.buffer;@@ -141,14 +145,18 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {};} finally {// Clean up temporary file if it was created by prefetch- if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {+ if (+ tempFilePath &&+ meta.documentPrefetch !== undefined &&+ meta.documentPrefetch !== null+ ) {try {await unlink(tempFilePath);} catch (error) {// Ignore errors when cleaning up temp files- meta.logger?.warn("Failed to clean up temporary document file", {- error,- tempFilePath+ meta.logger?.warn("Failed to clean up temporary document file", {+ error,+ tempFilePath,});}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex fc6fd343d..e06fa94fc 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -266,10 +266,14 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {proxyUsed: meta.pdfPrefetch.proxyUsed,};} else {- const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- });+ const file = await fetchFileToBuffer(+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ );const ct = file.response.headers.get("Content-Type");if (ct && !ct.includes("application/pdf")) {@@ -302,10 +306,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const { response, tempFilePath } =meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }- : await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- });+ : await downloadFile(+ meta.id,+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ );if ((response as any).headers) {// if downloadFile was useddiff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex 21b14338b..66f3d9eb2 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -1,33 +1,87 @@import path from "path";import os from "os";import { createWriteStream, promises as fs } from "node:fs";-import { EngineError } from "../../error";+import {+ AddFeatureError,+ DNSResolutionError,+ EngineError,+ SiteError,+ SSLError,+} from "../../error";import { Writable } from "stream";import { v4 as uuid } from "uuid";import * as undici from "undici";import { getSecureDispatcher } from "./safeFetch";+const mapUndiciError = (url: string, skipTlsVerification: boolean, e: any) => {+ const code = e?.code ?? e?.cause?.code ?? e?.errno ?? e?.name;+ if (e?.name === "AbortError") {+ return e;+ }++ switch (code) {+ case "UND_ERR_CONNECT_TIMEOUT":+ case "UND_ERR_HEADERS_TIMEOUT":+ case "UND_ERR_BODY_TIMEOUT":+ case "ETIMEDOUT":+ return new SiteError("ERR_TIMED_OUT");++ case "ECONNREFUSED":+ case "EHOSTUNREACH":+ case "ENETUNREACH":+ return new SiteError("ERR_CONNECT_REFUSED");++ case "ENOTFOUND":+ case "EAI_AGAIN": {+ let hostname = url;+ try {+ hostname = new URL(url).hostname;+ } catch {}+ return new DNSResolutionError(hostname);+ }++ case "ECONNRESET":+ case "EPIPE":+ case "ECONNABORTED":+ return new SiteError("ERR_CONNECTION_RESET");++ case "UNABLE_TO_VERIFY_LEAF_SIGNATURE":+ case "DEPTH_ZERO_SELF_SIGNED_CERT":+ case "ERR_TLS_CERT_ALTNAME_INVALID":+ return new SSLError(skipTlsVerification);++ default:+ return e;+ }+};+export async function fetchFileToBuffer(url: string,+ skipTlsVerification: boolean = false,init?: undici.RequestInit,): Promise<{response: undici.Response;buffer: Buffer;}> {- const response = await undici.fetch(url, {- ...init,- redirect: "follow",- dispatcher: getSecureDispatcher(),- });- return {- response,- buffer: Buffer.from(await response.arrayBuffer()),- };+ try {+ const response = await undici.fetch(url, {+ ...init,+ redirect: "follow",+ dispatcher: getSecureDispatcher(skipTlsVerification),+ });+ return {+ response,+ buffer: Buffer.from(await response.arrayBuffer()),+ };+ } catch (e) {+ throw mapUndiciError(url, skipTlsVerification, e);+ }}export async function downloadFile(id: string,url: string,+ skipTlsVerification: boolean = false,init?: undici.RequestInit,): Promise<{response: undici.Response;@@ -37,18 +91,18 @@ export async function downloadFile(const tempFileWrite = createWriteStream(tempFilePath);// TODO: maybe we could use tlsclient for this? for proxying- const response = await undici.fetch(url, {- ...init,- redirect: "follow",- dispatcher: getSecureDispatcher(),- });+ try {+ const response = await undici.fetch(url, {+ ...init,+ redirect: "follow",+ dispatcher: getSecureDispatcher(skipTlsVerification),+ });- // This should never happen in the current state of JS/Undici (2024), but let's check anyways.- if (response.body === null) {- throw new EngineError("Response body was null", { cause: { response } });- }+ // This should never happen in the current state of JS/Undici (2024), but let's check anyways.+ if (response.body === null) {+ throw new EngineError("Response body was null", { cause: { response } });+ }- try {await response.body.pipeTo(Writable.toWeb(tempFileWrite), {signal: init?.signal || undefined,@@ -58,12 +112,14 @@ export async function downloadFile(cause: { error },});});++ return {+ response,+ tempFilePath,+ };+ } catch (e) {+ throw mapUndiciError(url, skipTlsVerification, e);} finally {tempFileWrite.close();}-- return {- response,- tempFilePath,- };}