Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api) (#3769)
Abimael Martell8d679cbcb68a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.tsindex fd04a6713..21f063aad 100644--- a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts@@ -1,13 +1,64 @@import {ALLOW_TEST_SUITE_WEBSITE,+ HAS_PROXY,TEST_PRODUCTION,TEST_SUITE_WEBSITE,testIf,} from "../lib";import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";+import https from "node:https";++const SELF_SIGNED_KEY = `-----BEGIN PRIVATE KEY-----+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDCBJPpxbzPPuW8+DxlXh4JXR56Qq7hT74yiD+PzZ2OznSnfpZOA2kmQ30ztuGNstr/oE5Ixwarcp+9P+0tFz321TeN/6PNmSb0MPay8Q9xjvylMSao2ydATTyKumoF0nqZfHlgQNu6uYGan7+/8qmZcVIJ09oJP29vFTxP7S6Nk40klHlQyBAwtT+IoRqWvBztnu+5Xuy8cIQP4er+WoKySMuuVsnU8kwwM+QbFKGipD1jnK1xUGaaUtNpStNqnUKwNtW4jsMRjR/FapPF+p8hHxOGrgHtZJ7lqZHi60TdrnaPQjvwe+drswJIM9cIPkyzyd0FBlg+CPGkmlUAC+4w0MjknjAgMBAAECggEAUyiXJWnlpYa1/UcTe5rPWQ2Pfz67AO75/jSFZbx41XGV+kxBrCp2FTp0HYhTYOK6TzqskzELQM0efoT0hHWM0fsSea6lNMCCUQ7WTNNhUTeMQ+fCJDnatwrj2ipQazJt7f+WHpVuGLiOPnIeXfPDb+uhBvTpocUAi697RwiCfimVEl+wOcETyKjG+AdkHn1WqXdiduIQsmm3f8H4rJcGZo2OpyU33s0ZK2ReZhIkOEoyzp++BX9cN8wnmiQc4M0DpcSOZYwn2OtLdsGirFasmhfDFMBgE+BS9W3knsMxd/zEtKTs+UxTK6bfdCNJkE8fR0H0ozViFvl61fcPuBvyDeTorwQKBgQD3oSbHo6MfZ53vNbBu+rfzoV35jnWUN34fkMo8YkGGIH1nPb7xlShg/zR6OL86wk6ADFO43UFEmbVDGxlCn+OscA04pfKP/Zvb9EDVr/j5Ix1LiFlATbjkPPqZj3IRdIih0ycjKFHR/OtbpPoINT+8CNs1/VYsgYh47t9fdApqdc4MQKBgQDIk4BGQfYc4Fzxpfhf7/IwA5AfTJip/H9f+XAxVFJhANymaUdDxar9D7VdoSQYOL2BD7KzT0JeeFwvwavnVc0jATAgmi81D+EaF+RX5DG5GoYSFD8aSnhu0dGF3QrPemaamlJdCYOd3of+b0wZhkCvVPcOy/bTdkOD9q+F3gZacWyUwKBgQCcbbjJtJ1/YT1rt0bVJCP3wg2db/g+Y0684RN0OQDjtKpPWA5z+DfNzmmgK+jhfY2JZkAdL+fjJhZTZfL3GZmMAKqHmq0e0jSEeJDGv70ozIGXQPEk1+SRGdRU3UD5tdv6HiFDHF1TgapMIlOwi7JZ+7SlE6znsBPZNbGvc82oWSgQKBgQDA+rVEQLNUrsCwYxoLuRiW1Efck2gPdZ31EMbx3Dq2jIlqIsAezogPSUPEicOOsRL6J+AZaUc1DywNjrPxh4f6Jnd6JsxOeOX7X+2F8OQDGQOp4mEr9FX0vwIzQb/cx4xA//+YXAci5osepF2lXK6x/wXMDd9PIF1eMMSOzFX2E/dmQKBgCAfdmgO9yU1K2esVb8B+vLPuFWNiQ1UvmDaV4DDc10JQH64R/yv7HsmRf5C+oPjUbgfXyn+SLdtaVKWtyDtk+8ajWWfZ2tqtXT31VdeJLjkABT+BU9H3u5YyrSPNIcGMgNSLjmb/DuWaclrIT6+hL+9g/pFxnn4mOVBYdX1GyDHhBD+-----END PRIVATE KEY-----`;++const SELF_SIGNED_CERT = `-----BEGIN CERTIFICATE-----+MIICyTCCAbGgAwIBAgIJAMikv3+5vuPnMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV+BAMMCWxvY2FsaG9zdDAeFw0yNjA2MjIxNzQ0NThaFw0zNjA2MTkxNzQ0NThaMBQx+EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC+ggEBAMIEk+nFvM8+5bwPGVeHgldHnpCruFPvjKIP4/NnY7OdKd+lk4DaSZDfTO24+Y2y2v+gTkjHBqtyn70/S0XPfbVN43/o82ZJvQw9rLxD3GO/KUxJqjbJ0BNPIq6ag+XSepl8eWBA27q5gZqfv/yqZlxUgnT2gk/b28VPE/tLo2TjSSUeVDIEDC1P4ihGpa+8HO2e77le7LxwhA/h6tagrJIy65WydTyTDAz5BsUoaKkPWOcrXFQZppS02lK02qd+QrA21biOwxGNH8Vqk8WnyEfE4auAe1knuWpkeLrRN2udo9CO/B752uzAkgz1wg+T+LPJ3QUGWD4I8aSaVQALjDQyOSeMCAwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxo+b3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCqlZaFVPZ69S25OVzEPiRClsEe+NcpBmGwwEl3Yn+hmMXspBaubv2cHWKL+KRm3C9FicD/FqwTxjVqBmizZRFUETS/r+vTBEKE2lwXUVUbMZtCJ+NPszXh00PFjHO/Z/1poG+ZiMEXWB+nYJRDxUIt5BmK9W+GRxVxKHLId8rqcGyQk/hdjhg8xd9jINKN6T378ZHv1t+Z1WAWLwA4oZeiWwD7kEF+3IDQNbcl65aq2Qw6IDw+6bnCLc54A3BiU7miHDlNiP9rgjqf3cYzutTiem8L7DbY+SDXcSDjxdKJmewIopBnycE2rAU4/O7AivhFA0iqQ/KAD27s6l/7rtO+LH6za+-----END CERTIFICATE-----`;describe("V2 Scrape skipTlsVerification Default", () => {let identity: Identity;+ let tlsServer: https.Server;+ let selfSignedUrl: string;beforeAll(async () => {identity = await idmux({@@ -15,31 +66,57 @@ describe("V2 Scrape skipTlsVerification Default", () => {concurrency: 100,credits: 1000000,});++ tlsServer = https.createServer(+ { key: SELF_SIGNED_KEY, cert: SELF_SIGNED_CERT },+ (_req, res) => {+ res.setHeader("Content-Type", "text/html; charset=utf-8");+ res.end("<main><h1>Self-signed TLS fixture</h1></main>");+ },+ );++ await new Promise<void>(resolve => {+ tlsServer.listen(0, "127.0.0.1", resolve);+ });+ const address = tlsServer.address();+ if (address === null || typeof address === "string") {+ throw new Error("Failed to start self-signed TLS fixture");+ }+ selfSignedUrl = `https://127.0.0.1:${address.port}/`;}, 10000);- test(+ afterAll(async () => {+ if (!tlsServer) {+ return;+ }+ await new Promise<void>((resolve, reject) => {+ tlsServer.close(err => (err ? reject(err) : resolve()));+ });+ });++ testIf(!HAS_PROXY)("should default skipTlsVerification to true in v2 API",async () => {const data = await scrape({- url: "https://expired.badssl.com/",+ url: selfSignedUrl,maxAge: 0,},identity,);expect(data).toBeDefined();- expect(data.markdown).toContain("badssl.com");+ expect(data.markdown).toContain("Self-signed TLS fixture");},scrapeTimeout,);- test(+ testIf(!HAS_PROXY)("should allow explicit skipTlsVerification: false override",async () => {const response = await scrapeRaw({- url: "https://expired.badssl.com/",+ url: selfSignedUrl,skipTlsVerification: false,maxAge: 0,},diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 79fc03bea..2d403fc2b 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -38,6 +38,29 @@ describe("V2 Types Validation", () => {expect(result.url).toBe("https://example.com");expect(result.origin).toBe("api");expect(result.formats).toEqual([{ type: "markdown" }]);+ expect(result.skipTlsVerification).toBe(true);+ });++ it("should preserve explicit skipTlsVerification false", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ skipTlsVerification: false,+ };++ const result = scrapeRequestSchema.parse(input);+ expect(result.skipTlsVerification).toBe(false);+ });++ it("should default skipTlsVerification to false when custom headers are used", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ headers: {+ "x-test": "true",+ },+ };++ const result = scrapeRequestSchema.parse(input);+ expect(result.skipTlsVerification).toBe(false);});it("should accept valid scrape request with format objects", () => {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 261e3f33c..f51a10a73 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -724,6 +724,18 @@ const waitForRefineOpts = {path: ["waitFor"],};+export const applyScrapeOptionsDefaults = <T extends ScrapeOptionsBase>(+ obj: T,+): T & { skipTlsVerification: boolean } => ({+ ...obj,+ skipTlsVerification:+ obj.skipTlsVerification ??+ ((obj.headers && Object.keys(obj.headers).length > 0) ||+ (obj.actions && obj.actions.length > 0)+ ? false+ : true),+});+// Base transform function that handles both nullable and non-nullable cases// Uses generic type to preserve all fields from extended schemasconst extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(@@ -731,7 +743,7 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(): T extends undefined ? undefined : T => {if (!obj) return obj as T extends undefined ? undefined : T;// Handle timeout- let result = { ...obj };+ let result = applyScrapeOptionsDefaults(obj);if (obj.formats.find(x => typeof x === "object" && x.type === "json") &&obj.timeout === 30000diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 796fe4c87..0c9fc5aac 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -5,6 +5,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { captureExceptionWithZdrCheck } from "../../services/sentry";import {+ applyScrapeOptionsDefaults,type Document,getPDFMaxPages,scrapeOptions,@@ -416,21 +417,13 @@ async function buildMetaObject(}}- const normalizedOptions = {- ...options,- skipTlsVerification:- options.skipTlsVerification ??- ((options.headers && Object.keys(options.headers).length > 0) ||- (options.actions && options.actions.length > 0)- ? false- : true),- };+ const effectiveOptions = applyScrapeOptionsDefaults(options);return {id,url,rewrittenUrl: rewriteUrl(url),- options: normalizedOptions,+ options: effectiveOptions,internalOptions,logger,abortHandle,@@ -447,7 +440,7 @@ async function buildMetaObject(}: undefined,),- featureFlags: buildFeatureFlags(url, normalizedOptions, internalOptions),+ featureFlags: buildFeatureFlags(url, effectiveOptions, internalOptions),mock:options.useMock !== undefined? await loadMock(options.useMock, _logger)