Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): change skipTlsVerification default for safe scrapes

Gergő Móricz483e90ad9424
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index c31517350..0dc9999c7 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -75,28 +75,40 @@ export const isAgentExtractModelValid = (x: string | undefined) =>
x?.toLowerCase() === agentExtractModelValue;
function normalizeSchemaForOpenAI(schema: any): any {
- if (!schema || typeof schema !== 'object') {
+ if (!schema || typeof schema !== "object") {
return schema;
}
-
+
const visited = new WeakSet();
-
+
function normalizeObject(obj: any): any {
- if (typeof obj !== 'object' || obj === null) return obj;
-
+ if (typeof obj !== "object" || obj === null) return obj;
+
if (visited.has(obj)) return obj;
visited.add(obj);
-
+
const normalized = { ...obj };
-
- if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("properties") &&
+ normalized.hasOwnProperty("additionalProperties")
+ ) {
delete normalized.additionalProperties;
}
-
- if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {
- if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {
- const validRequired = normalized.required.filter((field: string) =>
- normalized.properties.hasOwnProperty(field)
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("required") &&
+ normalized.hasOwnProperty("properties")
+ ) {
+ if (
+ Array.isArray(normalized.required) &&
+ typeof normalized.properties === "object" &&
+ normalized.properties !== null
+ ) {
+ const validRequired = normalized.required.filter((field: string) =>
+ normalized.properties.hasOwnProperty(field),
);
if (validRequired.length > 0) {
normalized.required = validRequired;
@@ -107,47 +119,53 @@ function normalizeSchemaForOpenAI(schema: any): any {
delete normalized.required;
}
}
-
+
for (const [key, value] of Object.entries(normalized)) {
- if (typeof value === 'object' && value !== null) {
+ if (typeof value === "object" && value !== null) {
normalized[key] = normalizeObject(value);
}
}
-
+
return normalized;
}
-
+
return normalizeObject(schema);
}
function validateSchemaForOpenAI(schema: any): boolean {
- if (!schema || typeof schema !== 'object') {
+ if (!schema || typeof schema !== "object") {
return true;
}
-
+
const visited = new WeakSet();
-
+
function hasInvalidStructure(obj: any): boolean {
- if (typeof obj !== 'object' || obj === null) return false;
-
+ if (typeof obj !== "object" || obj === null) return false;
+
if (visited.has(obj)) return false;
visited.add(obj);
-
- if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {
+
+ if (
+ obj.type === "object" &&
+ !obj.hasOwnProperty("properties") &&
+ !obj.hasOwnProperty("patternProperties") &&
+ obj.additionalProperties === true
+ ) {
return true;
}
-
+
for (const value of Object.values(obj)) {
- if (typeof value === 'object' && value !== null) {
+ if (typeof value === "object" && value !== null) {
if (hasInvalidStructure(value)) return true;
}
}
return false;
}
-
+
return !hasInvalidStructure(schema);
}
-const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
+const OPENAI_SCHEMA_ERROR_MESSAGE =
+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
export const agentOptionsExtract = z
.object({
@@ -175,13 +193,10 @@ export const extractOptions = z
message: "Invalid JSON schema.",
},
)
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -213,13 +228,10 @@ const extractOptionsWithAgent = z
message: "Invalid JSON schema.",
},
)
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -457,13 +469,10 @@ const baseScrapeOptions = z
message: "Invalid JSON schema.",
},
)
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -491,7 +500,7 @@ const baseScrapeOptions = z
languages: z.string().array().optional(),
})
.optional(),
- skipTlsVerification: z.boolean().default(false),
+ skipTlsVerification: z.boolean().optional(),
removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
useMock: z.string().optional(),
@@ -670,13 +679,10 @@ const extractV1Options = z
message: "Invalid JSON schema.",
},
)
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index fbc82e667..197b86b73 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -64,28 +64,40 @@ const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
function normalizeSchemaForOpenAI(schema: any): any {
- if (!schema || typeof schema !== 'object') {
+ if (!schema || typeof schema !== "object") {
return schema;
}
-
+
const visited = new WeakSet();
-
+
function normalizeObject(obj: any): any {
- if (typeof obj !== 'object' || obj === null) return obj;
-
+ if (typeof obj !== "object" || obj === null) return obj;
+
if (visited.has(obj)) return obj;
visited.add(obj);
-
+
const normalized = { ...obj };
-
- if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("properties") &&
+ normalized.hasOwnProperty("additionalProperties")
+ ) {
delete normalized.additionalProperties;
}
-
- if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {
- if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {
- const validRequired = normalized.required.filter((field: string) =>
- normalized.properties.hasOwnProperty(field)
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("required") &&
+ normalized.hasOwnProperty("properties")
+ ) {
+ if (
+ Array.isArray(normalized.required) &&
+ typeof normalized.properties === "object" &&
+ normalized.properties !== null
+ ) {
+ const validRequired = normalized.required.filter((field: string) =>
+ normalized.properties.hasOwnProperty(field),
);
if (validRequired.length > 0) {
normalized.required = validRequired;
@@ -96,48 +108,54 @@ function normalizeSchemaForOpenAI(schema: any): any {
delete normalized.required;
}
}
-
+
for (const [key, value] of Object.entries(normalized)) {
- if (typeof value === 'object' && value !== null) {
+ if (typeof value === "object" && value !== null) {
normalized[key] = normalizeObject(value);
}
}
-
+
return normalized;
}
-
+
return normalizeObject(schema);
}
function validateSchemaForOpenAI(schema: any): boolean {
- if (!schema || typeof schema !== 'object') {
+ if (!schema || typeof schema !== "object") {
return true;
}
-
+
const visited = new WeakSet();
-
+
function hasInvalidStructure(obj: any): boolean {
- if (typeof obj !== 'object' || obj === null) return false;
-
+ if (typeof obj !== "object" || obj === null) return false;
+
if (visited.has(obj)) return false;
visited.add(obj);
-
- if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {
+
+ if (
+ obj.type === "object" &&
+ !obj.hasOwnProperty("properties") &&
+ !obj.hasOwnProperty("patternProperties") &&
+ obj.additionalProperties === true
+ ) {
return true;
}
-
+
for (const value of Object.values(obj)) {
- if (typeof value === 'object' && value !== null) {
+ if (typeof value === "object" && value !== null) {
if (hasInvalidStructure(value)) return true;
}
}
return false;
}
-
+
return !hasInvalidStructure(schema);
}
-const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
+const OPENAI_SCHEMA_ERROR_MESSAGE =
+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
@@ -250,14 +268,13 @@ const actionsSchema = z
const jsonFormatWithOptions = z
.object({
type: z.literal("json"),
- schema: z.any().optional()
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ schema: z
+ .any()
+ .optional()
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
prompt: z.string().max(10000).optional(),
})
.strict();
@@ -268,14 +285,13 @@ const changeTrackingFormatWithOptions = z
.object({
type: z.literal("changeTracking"),
prompt: z.string().optional(),
- schema: z.any().optional()
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ schema: z
+ .any()
+ .optional()
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -463,7 +479,7 @@ const baseScrapeOptions = z
location: locationSchema,
- skipTlsVerification: z.boolean().default(true),
+ skipTlsVerification: z.boolean().optional(),
removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
useMock: z.string().optional(),
@@ -569,13 +585,10 @@ const extractOptions = z
message: "Invalid JSON schema.",
},
)
- .transform((val) => normalizeSchemaForOpenAI(val))
- .refine(
- (val) => validateSchemaForOpenAI(val),
- {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- },
- ),
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 7326d038a..633b9706f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -83,7 +83,7 @@ export type Meta = {
id: string;
url: string;
rewrittenUrl?: string;
- options: ScrapeOptions;
+ options: ScrapeOptions & { skipTlsVerification: boolean };
internalOptions: InternalOptions;
logger: Logger;
abort: AbortManager;
@@ -247,7 +247,15 @@ async function buildMetaObject(
id,
url,
rewrittenUrl: rewriteUrl(url),
- options,
+ options: {
+ ...options,
+ skipTlsVerification:
+ options.skipTlsVerification ??
+ ((options.headers && Object.keys(options.headers).length > 0) ||
+ (options.actions && options.actions.length > 0)
+ ? false
+ : true),
+ },
internalOptions,
logger,
abort: new AbortManager(