Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): change skipTlsVerification default for safe scrapes
Gergő Móricz483e90ad9424
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex c31517350..0dc9999c7 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -75,28 +75,40 @@ export const isAgentExtractModelValid = (x: string | undefined) =>x?.toLowerCase() === agentExtractModelValue;function normalizeSchemaForOpenAI(schema: any): any {- if (!schema || typeof schema !== 'object') {+ if (!schema || typeof schema !== "object") {return schema;}-+const visited = new WeakSet();-+function normalizeObject(obj: any): any {- if (typeof obj !== 'object' || obj === null) return obj;-+ if (typeof obj !== "object" || obj === null) return obj;+if (visited.has(obj)) return obj;visited.add(obj);-+const normalized = { ...obj };-- if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("properties") &&+ normalized.hasOwnProperty("additionalProperties")+ ) {delete normalized.additionalProperties;}-- if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {- if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {- const validRequired = normalized.required.filter((field: string) =>- normalized.properties.hasOwnProperty(field)++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("required") &&+ normalized.hasOwnProperty("properties")+ ) {+ if (+ Array.isArray(normalized.required) &&+ typeof normalized.properties === "object" &&+ normalized.properties !== null+ ) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field),);if (validRequired.length > 0) {normalized.required = validRequired;@@ -107,47 +119,53 @@ function normalizeSchemaForOpenAI(schema: any): any {delete normalized.required;}}-+for (const [key, value] of Object.entries(normalized)) {- if (typeof value === 'object' && value !== null) {+ if (typeof value === "object" && value !== null) {normalized[key] = normalizeObject(value);}}-+return normalized;}-+return normalizeObject(schema);}function validateSchemaForOpenAI(schema: any): boolean {- if (!schema || typeof schema !== 'object') {+ if (!schema || typeof schema !== "object") {return true;}-+const visited = new WeakSet();-+function hasInvalidStructure(obj: any): boolean {- if (typeof obj !== 'object' || obj === null) return false;-+ if (typeof obj !== "object" || obj === null) return false;+if (visited.has(obj)) return false;visited.add(obj);-- if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {++ if (+ obj.type === "object" &&+ !obj.hasOwnProperty("properties") &&+ !obj.hasOwnProperty("patternProperties") &&+ obj.additionalProperties === true+ ) {return true;}-+for (const value of Object.values(obj)) {- if (typeof value === 'object' && value !== null) {+ if (typeof value === "object" && value !== null) {if (hasInvalidStructure(value)) return true;}}return false;}-+return !hasInvalidStructure(schema);}-const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+const OPENAI_SCHEMA_ERROR_MESSAGE =+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";export const agentOptionsExtract = z.object({@@ -175,13 +193,10 @@ export const extractOptions = zmessage: "Invalid JSON schema.",},)- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -213,13 +228,10 @@ const extractOptionsWithAgent = zmessage: "Invalid JSON schema.",},)- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -457,13 +469,10 @@ const baseScrapeOptions = zmessage: "Invalid JSON schema.",},)- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -491,7 +500,7 @@ const baseScrapeOptions = zlanguages: z.string().array().optional(),}).optional(),- skipTlsVerification: z.boolean().default(false),+ skipTlsVerification: z.boolean().optional(),removeBase64Images: z.boolean().default(true),fastMode: z.boolean().default(false),useMock: z.string().optional(),@@ -670,13 +679,10 @@ const extractV1Options = zmessage: "Invalid JSON schema.",},)- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex fbc82e667..197b86b73 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -64,28 +64,40 @@ const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";function normalizeSchemaForOpenAI(schema: any): any {- if (!schema || typeof schema !== 'object') {+ if (!schema || typeof schema !== "object") {return schema;}-+const visited = new WeakSet();-+function normalizeObject(obj: any): any {- if (typeof obj !== 'object' || obj === null) return obj;-+ if (typeof obj !== "object" || obj === null) return obj;+if (visited.has(obj)) return obj;visited.add(obj);-+const normalized = { ...obj };-- if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("properties") &&+ normalized.hasOwnProperty("additionalProperties")+ ) {delete normalized.additionalProperties;}-- if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {- if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {- const validRequired = normalized.required.filter((field: string) =>- normalized.properties.hasOwnProperty(field)++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("required") &&+ normalized.hasOwnProperty("properties")+ ) {+ if (+ Array.isArray(normalized.required) &&+ typeof normalized.properties === "object" &&+ normalized.properties !== null+ ) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field),);if (validRequired.length > 0) {normalized.required = validRequired;@@ -96,48 +108,54 @@ function normalizeSchemaForOpenAI(schema: any): any {delete normalized.required;}}-+for (const [key, value] of Object.entries(normalized)) {- if (typeof value === 'object' && value !== null) {+ if (typeof value === "object" && value !== null) {normalized[key] = normalizeObject(value);}}-+return normalized;}-+return normalizeObject(schema);}function validateSchemaForOpenAI(schema: any): boolean {- if (!schema || typeof schema !== 'object') {+ if (!schema || typeof schema !== "object") {return true;}-+const visited = new WeakSet();-+function hasInvalidStructure(obj: any): boolean {- if (typeof obj !== 'object' || obj === null) return false;-+ if (typeof obj !== "object" || obj === null) return false;+if (visited.has(obj)) return false;visited.add(obj);-- if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {++ if (+ obj.type === "object" &&+ !obj.hasOwnProperty("properties") &&+ !obj.hasOwnProperty("patternProperties") &&+ obj.additionalProperties === true+ ) {return true;}-+for (const value of Object.values(obj)) {- if (typeof value === 'object' && value !== null) {+ if (typeof value === "object" && value !== null) {if (hasInvalidStructure(value)) return true;}}return false;}-+return !hasInvalidStructure(schema);}-const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+const OPENAI_SCHEMA_ERROR_MESSAGE =+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;@@ -250,14 +268,13 @@ const actionsSchema = zconst jsonFormatWithOptions = z.object({type: z.literal("json"),- schema: z.any().optional()- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ schema: z+ .any()+ .optional()+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),prompt: z.string().max(10000).optional(),}).strict();@@ -268,14 +285,13 @@ const changeTrackingFormatWithOptions = z.object({type: z.literal("changeTracking"),prompt: z.string().optional(),- schema: z.any().optional()- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ schema: z+ .any()+ .optional()+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -463,7 +479,7 @@ const baseScrapeOptions = zlocation: locationSchema,- skipTlsVerification: z.boolean().default(true),+ skipTlsVerification: z.boolean().optional(),removeBase64Images: z.boolean().default(true),fastMode: z.boolean().default(false),useMock: z.string().optional(),@@ -569,13 +585,10 @@ const extractOptions = zmessage: "Invalid JSON schema.",},)- .transform((val) => normalizeSchemaForOpenAI(val))- .refine(- (val) => validateSchemaForOpenAI(val),- {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- },- ),+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 7326d038a..633b9706f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -83,7 +83,7 @@ export type Meta = {id: string;url: string;rewrittenUrl?: string;- options: ScrapeOptions;+ options: ScrapeOptions & { skipTlsVerification: boolean };internalOptions: InternalOptions;logger: Logger;abort: AbortManager;@@ -247,7 +247,15 @@ async function buildMetaObject(id,url,rewrittenUrl: rewriteUrl(url),- options,+ options: {+ ...options,+ skipTlsVerification:+ options.skipTlsVerification ??+ ((options.headers && Object.keys(options.headers).length > 0) ||+ (options.actions && options.actions.length > 0)+ ? false+ : true),+ },internalOptions,logger,abort: new AbortManager(