Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): pdf + document engines not respecting skipTlsVerification flag

delong350b7f1eafedf
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index 8a8cfc8f4..195acb440 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -108,10 +108,14 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
proxyUsed = meta.documentPrefetch.proxyUsed;
} else {
// Fetch the document normally
- const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- });
+ const result = await fetchFileToBuffer(
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ );
response = result.response;
buffer = result.buffer;
@@ -141,14 +145,18 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
};
} finally {
// Clean up temporary file if it was created by prefetch
- if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
+ if (
+ tempFilePath &&
+ meta.documentPrefetch !== undefined &&
+ meta.documentPrefetch !== null
+ ) {
try {
await unlink(tempFilePath);
} catch (error) {
// Ignore errors when cleaning up temp files
- meta.logger?.warn("Failed to clean up temporary document file", {
- error,
- tempFilePath
+ meta.logger?.warn("Failed to clean up temporary document file", {
+ error,
+ tempFilePath,
});
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index fc6fd343d..e06fa94fc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -266,10 +266,14 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
proxyUsed: meta.pdfPrefetch.proxyUsed,
};
} else {
- const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- });
+ const file = await fetchFileToBuffer(
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ );
const ct = file.response.headers.get("Content-Type");
if (ct && !ct.includes("application/pdf")) {
@@ -302,10 +306,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const { response, tempFilePath } =
meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null
? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }
- : await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- });
+ : await downloadFile(
+ meta.id,
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ );
if ((response as any).headers) {
// if downloadFile was used
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
index 21b14338b..66f3d9eb2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -1,33 +1,87 @@
import path from "path";
import os from "os";
import { createWriteStream, promises as fs } from "node:fs";
-import { EngineError } from "../../error";
+import {
+ AddFeatureError,
+ DNSResolutionError,
+ EngineError,
+ SiteError,
+ SSLError,
+} from "../../error";
import { Writable } from "stream";
import { v4 as uuid } from "uuid";
import * as undici from "undici";
import { getSecureDispatcher } from "./safeFetch";
+const mapUndiciError = (url: string, skipTlsVerification: boolean, e: any) => {
+ const code = e?.code ?? e?.cause?.code ?? e?.errno ?? e?.name;
+ if (e?.name === "AbortError") {
+ return e;
+ }
+
+ switch (code) {
+ case "UND_ERR_CONNECT_TIMEOUT":
+ case "UND_ERR_HEADERS_TIMEOUT":
+ case "UND_ERR_BODY_TIMEOUT":
+ case "ETIMEDOUT":
+ return new SiteError("ERR_TIMED_OUT");
+
+ case "ECONNREFUSED":
+ case "EHOSTUNREACH":
+ case "ENETUNREACH":
+ return new SiteError("ERR_CONNECT_REFUSED");
+
+ case "ENOTFOUND":
+ case "EAI_AGAIN": {
+ let hostname = url;
+ try {
+ hostname = new URL(url).hostname;
+ } catch {}
+ return new DNSResolutionError(hostname);
+ }
+
+ case "ECONNRESET":
+ case "EPIPE":
+ case "ECONNABORTED":
+ return new SiteError("ERR_CONNECTION_RESET");
+
+ case "UNABLE_TO_VERIFY_LEAF_SIGNATURE":
+ case "DEPTH_ZERO_SELF_SIGNED_CERT":
+ case "ERR_TLS_CERT_ALTNAME_INVALID":
+ return new SSLError(skipTlsVerification);
+
+ default:
+ return e;
+ }
+};
+
export async function fetchFileToBuffer(
url: string,
+ skipTlsVerification: boolean = false,
init?: undici.RequestInit,
): Promise<{
response: undici.Response;
buffer: Buffer;
}> {
- const response = await undici.fetch(url, {
- ...init,
- redirect: "follow",
- dispatcher: getSecureDispatcher(),
- });
- return {
- response,
- buffer: Buffer.from(await response.arrayBuffer()),
- };
+ try {
+ const response = await undici.fetch(url, {
+ ...init,
+ redirect: "follow",
+ dispatcher: getSecureDispatcher(skipTlsVerification),
+ });
+ return {
+ response,
+ buffer: Buffer.from(await response.arrayBuffer()),
+ };
+ } catch (e) {
+ throw mapUndiciError(url, skipTlsVerification, e);
+ }
}
export async function downloadFile(
id: string,
url: string,
+ skipTlsVerification: boolean = false,
init?: undici.RequestInit,
): Promise<{
response: undici.Response;
@@ -37,18 +91,18 @@ export async function downloadFile(
const tempFileWrite = createWriteStream(tempFilePath);
// TODO: maybe we could use tlsclient for this? for proxying
- const response = await undici.fetch(url, {
- ...init,
- redirect: "follow",
- dispatcher: getSecureDispatcher(),
- });
+ try {
+ const response = await undici.fetch(url, {
+ ...init,
+ redirect: "follow",
+ dispatcher: getSecureDispatcher(skipTlsVerification),
+ });
- // This should never happen in the current state of JS/Undici (2024), but let's check anyways.
- if (response.body === null) {
- throw new EngineError("Response body was null", { cause: { response } });
- }
+ // This should never happen in the current state of JS/Undici (2024), but let's check anyways.
+ if (response.body === null) {
+ throw new EngineError("Response body was null", { cause: { response } });
+ }
- try {
await response.body
.pipeTo(Writable.toWeb(tempFileWrite), {
signal: init?.signal || undefined,
@@ -58,12 +112,14 @@ export async function downloadFile(
cause: { error },
});
});
+
+ return {
+ response,
+ tempFilePath,
+ };
+ } catch (e) {
+ throw mapUndiciError(url, skipTlsVerification, e);
} finally {
tempFileWrite.close();
}
-
- return {
- response,
- tempFilePath,
- };
}