Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolasa60f3ff645e5
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex e7618f726..f4e528cef 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -1,19 +1,14 @@import { Response } from "express";-import { RequestWithAuth } from "./types";+import {+ GenerateLLMsTextRequest,+ generateLLMsTextRequestSchema,+ RequestWithAuth,+} from "./types";import { getGenerateLlmsTxtQueue } from "../../services/queue-service";import * as Sentry from "@sentry/node";import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";import { z } from "zod";-export const generateLLMsTextRequestSchema = z.object({- url: z.string().url().describe('The URL to generate text from'),- maxUrls: z.number().min(1).max(100).default(10).describe('Maximum number of URLs to process'),- showFullText: z.boolean().default(false).describe('Whether to show the full LLMs-full.txt in the response'),- __experimental_stream: z.boolean().optional(),-});--export type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;-export type GenerateLLMsTextResponse = {success: boolean;id: string;@@ -66,14 +61,18 @@ export async function generateLLMsTextController(},},async (span) => {- await getGenerateLlmsTxtQueue().add(generationId, {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,+ await getGenerateLlmsTxtQueue().add(+ generationId,+ {+ ...jobData,+ sentry: {+ trace: Sentry.spanToTraceHeader(span),+ baggage: Sentry.spanToBaggageHeader(span),+ size,+ },},- }, { jobId: generationId });+ { jobId: generationId },+ );},);} else {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 9f0150b6d..580cf80ad 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -34,7 +34,10 @@ export const url = z.preprocess(.url().regex(/^https?:\/\//, "URL uses unsupported protocol").refine(- (x) => /\.[a-zA-Z\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(x),+ (x) =>+ /\.[a-zA-Z\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(+ x,+ ),"URL must have a valid top-level domain or be a valid path",).refine((x) => {@@ -69,7 +72,10 @@ export type ExtractOptions = z.infer<typeof extractOptions>;const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;-function calculateTotalWaitTime(actions: any[] = [], waitFor: number = 0): number {+function calculateTotalWaitTime(+ actions: any[] = [],+ waitFor: number = 0,+): number {const actionWaitTime = actions.reduce((acc, action) => {if (action.type === "wait") {if (action.milliseconds) {@@ -86,63 +92,64 @@ function calculateTotalWaitTime(actions: any[] = [], waitFor: number = 0): numbereturn waitFor + actionWaitTime;}-export const actionsSchema = z.array(- z.union([- z- .object({- type: z.literal("wait"),- milliseconds: z.number().int().positive().finite().optional(),+export const actionsSchema = z+ .array(+ z.union([+ z+ .object({+ type: z.literal("wait"),+ milliseconds: z.number().int().positive().finite().optional(),+ selector: z.string().optional(),+ })+ .refine(+ (data) =>+ (data.milliseconds !== undefined || data.selector !== undefined) &&+ !(data.milliseconds !== undefined && data.selector !== undefined),+ {+ message:+ "Either 'milliseconds' or 'selector' must be provided, but not both.",+ },+ ),+ z.object({+ type: z.literal("click"),+ selector: z.string(),+ }),+ z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ }),+ z.object({+ type: z.literal("write"),+ text: z.string(),+ }),+ z.object({+ type: z.literal("press"),+ key: z.string(),+ }),+ z.object({+ type: z.literal("scroll"),+ direction: z.enum(["up", "down"]).optional().default("down"),selector: z.string().optional(),- })- .refine(- (data) =>- (data.milliseconds !== undefined || data.selector !== undefined) &&- !(data.milliseconds !== undefined && data.selector !== undefined),- {- message:- "Either 'milliseconds' or 'selector' must be provided, but not both.",- },- ),- z.object({- type: z.literal("click"),- selector: z.string(),- }),- z.object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- }),- z.object({- type: z.literal("write"),- text: z.string(),- }),- z.object({- type: z.literal("press"),- key: z.string(),- }),- z.object({- type: z.literal("scroll"),- direction: z.enum(["up", "down"]).optional().default("down"),- selector: z.string().optional(),- }),- z.object({- type: z.literal("scrape"),- }),- z.object({- type: z.literal("executeJavascript"),- script: z.string(),- }),- ]),-).refine(- (actions) => actions.length <= MAX_ACTIONS,- {+ }),+ z.object({+ type: z.literal("scrape"),+ }),+ z.object({+ type: z.literal("executeJavascript"),+ script: z.string(),+ }),+ ]),+ )+ .refine((actions) => actions.length <= MAX_ACTIONS, {message: `Number of actions cannot exceed ${MAX_ACTIONS}`,- },-).refine(- (actions) => calculateTotalWaitTime(actions) <= ACTIONS_MAX_WAIT_TIME * 1000,- {- message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,- },-);+ })+ .refine(+ (actions) =>+ calculateTotalWaitTime(actions) <= ACTIONS_MAX_WAIT_TIME * 1000,+ {+ message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,+ },+ );const baseScrapeOptions = z.object({@@ -169,7 +176,14 @@ const baseScrapeOptions = zexcludeTags: z.string().array().optional(),onlyMainContent: z.boolean().default(true),timeout: z.number().int().positive().finite().safe().optional(),- waitFor: z.number().int().nonnegative().finite().safe().max(60000).default(0),+ waitFor: z+ .number()+ .int()+ .nonnegative()+ .finite()+ .safe()+ .max(60000)+ .default(0),// Deprecate this to jsonOptionsextract: extractOptions.optional(),// New@@ -184,7 +198,10 @@ const baseScrapeOptions = z.string().optional().refine(- (val) => !val || Object.keys(countries).includes(val.toUpperCase()) || val === "US-generic",+ (val) =>+ !val ||+ Object.keys(countries).includes(val.toUpperCase()) ||+ val === "US-generic",{message:"Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",@@ -267,18 +284,23 @@ const extractTransform = (obj) => {}return obj;-}+};-export const scrapeOptions = baseScrapeOptions.refine(- (obj) => {- if (!obj.actions) return true;- return calculateTotalWaitTime(obj.actions, obj.waitFor) <= ACTIONS_MAX_WAIT_TIME * 1000;- },- {- message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,- }-).refine(extractRefine, extractRefineOpts)-.transform(extractTransform);+export const scrapeOptions = baseScrapeOptions+ .refine(+ (obj) => {+ if (!obj.actions) return true;+ return (+ calculateTotalWaitTime(obj.actions, obj.waitFor) <=+ ACTIONS_MAX_WAIT_TIME * 1000+ );+ },+ {+ message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,+ },+ )+ .refine(extractRefine, extractRefineOpts)+ .transform(extractTransform);export type ScrapeOptions = z.infer<typeof baseScrapeOptions>;@@ -324,15 +346,26 @@ export const extractV1Options = z__experimental_showSources: z.boolean().default(false),showSources: z.boolean().default(false),__experimental_cacheKey: z.string().optional(),- __experimental_cacheMode: z.enum(["direct", "save", "load"]).default("direct").optional()+ __experimental_cacheMode: z+ .enum(["direct", "save", "load"])+ .default("direct")+ .optional(),}).strict(strictMessage).transform((obj) => ({...obj,allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,}))- .refine(x => x.scrapeOptions ? extractRefine(x.scrapeOptions) : true, extractRefineOpts)- .transform(x => ({ ...x, scrapeOptions: x.scrapeOptions ? extractTransform(x.scrapeOptions) : x.scrapeOptions }));+ .refine(+ (x) => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),+ extractRefineOpts,+ )+ .transform((x) => ({+ ...x,+ scrapeOptions: x.scrapeOptions+ ? extractTransform(x.scrapeOptions)+ : x.scrapeOptions,+ }));export type ExtractV1Options = z.infer<typeof extractV1Options>;export const extractRequestSchema = extractV1Options;@@ -366,7 +399,9 @@ export const webhookSchema = z.preprocess(url: z.string().url(),headers: z.record(z.string(), z.string()).default({}),metadata: z.record(z.string(), z.string()).default({}),- events: z.array(z.enum(["completed", "failed", "page", "started"])).default(["completed", "failed", "page", "started"]),+ events: z+ .array(z.enum(["completed", "failed", "page", "started"]))+ .default(["completed", "failed", "page", "started"]),}).strict(strictMessage),);@@ -435,8 +470,11 @@ export const crawlRequestSchema = crawlerOptionslimit: z.number().default(10000),}).strict(strictMessage)- .refine(x => extractRefine(x.scrapeOptions), extractRefineOpts)- .transform(x => ({ ...x, scrapeOptions: extractTransform(x.scrapeOptions) }));+ .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)+ .transform((x) => ({+ ...x,+ scrapeOptions: extractTransform(x.scrapeOptions),+ }));// export type CrawlRequest = {// url: string;@@ -902,7 +940,8 @@ export const searchRequestSchema = zlocation: z.string().optional(),origin: z.string().optional().default("api"),timeout: z.number().int().positive().finite().safe().default(60000),- scrapeOptions: baseScrapeOptions.extend({+ scrapeOptions: baseScrapeOptions+ .extend({formats: z.array(z.enum([@@ -922,8 +961,11 @@ export const searchRequestSchema = z.strict("Unrecognized key in body -- please review the v1 API documentation for request body changes",)- .refine(x => extractRefine(x.scrapeOptions), extractRefineOpts)- .transform(x => ({ ...x, scrapeOptions: extractTransform(x.scrapeOptions) }));+ .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)+ .transform((x) => ({+ ...x,+ scrapeOptions: extractTransform(x.scrapeOptions),+ }));export type SearchRequest = z.infer<typeof searchRequestSchema>;export type SearchRequestInput = z.input<typeof searchRequestSchema>;@@ -943,3 +985,22 @@ export type TokenUsage = {step?: string;model?: string;};++export const generateLLMsTextRequestSchema = z.object({+ url: url.describe("The URL to generate text from"),+ maxUrls: z+ .number()+ .min(1)+ .max(100)+ .default(10)+ .describe("Maximum number of URLs to process"),+ showFullText: z+ .boolean()+ .default(false)+ .describe("Whether to show the full LLMs-full.txt in the response"),+ __experimental_stream: z.boolean().optional(),+});++export type GenerateLLMsTextRequest = z.infer<+ typeof generateLLMsTextRequestSchema+>;