Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Allow international URLs to pass validation (#1717)
Micah Stairs9a5d40c3cf41
diff --git a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts b/apps/api/src/controllers/v1/__tests__/urlValidation.test.tsindex afa44e58d..9fe1f02fe 100644--- a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts+++ b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts@@ -76,4 +76,9 @@ describe("URL Schema Validation", () => {it("should reject malformed URLs containing multiple 'http://'", () => {expect(() => url.parse("http://ex ample.com/")).toThrow("Invalid URL");});++ it("should accept URLs with international domain names", () => {+ expect(() => url.parse("http://xn--1lqv92a901a.xn--ses554g/")).not.toThrow();+ });+});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ab4e39343..3b3a37d13 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -42,7 +42,7 @@ export const url = z.preprocess(if (!protocolIncluded(x as string)) {x = `http://${x}`;}-+// transforming the query parameters is breaking certain sites, so we're not doing it - mogery// try {// const urlObj = new URL(x as string);@@ -52,7 +52,7 @@ export const url = z.preprocess(// }// } catch (e) {// }-+return x;},z@@ -61,7 +61,7 @@ export const url = z.preprocess(.regex(/^https?:\/\//, "URL uses unsupported protocol").refine((x) =>- /\.[a-zA-Z\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(+ /\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(x,),"URL must have a valid top-level domain or be a valid path",@@ -74,7 +74,7 @@ export const url = z.preprocess(return false;}}, "Invalid URL")- // .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),+ // .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);const strictMessage =@@ -823,11 +823,11 @@ export type ErrorResponse = {export type ScrapeResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document;- scrape_id?: string;- };+ success: true;+ warning?: string;+ data: Document;+ scrape_id?: string;+ };export interface ScrapeResponseRequestTest {statusCode: number;@@ -878,27 +878,27 @@ export interface ExtractResponseRequestTest {export type CrawlResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- };+ success: true;+ id: string;+ url: string;+ };export type BatchScrapeResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- invalidURLs?: string[];- };+ success: true;+ id: string;+ url: string;+ invalidURLs?: string[];+ };export type MapResponse =| ErrorResponse| {- success: true;- links: string[];- scrape_id?: string;- };+ success: true;+ links: string[];+ scrape_id?: string;+ };export type CrawlStatusParams = {jobId: string;@@ -911,47 +911,47 @@ export type ConcurrencyCheckParams = {export type ConcurrencyCheckResponse =| ErrorResponse| {- success: true;- concurrency: number;- maxConcurrency: number;- };+ success: true;+ concurrency: number;+ maxConcurrency: number;+ };export type CrawlStatusResponse =| ErrorResponse| {- success: true;- status: "scraping" | "completed" | "failed" | "cancelled";- completed: number;- total: number;- creditsUsed: number;- expiresAt: string;- next?: string;- data: Document[];- };+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: string;+ next?: string;+ data: Document[];+ };export type OngoingCrawlsResponse =| ErrorResponse| {- success: true;- crawls: {- id: string;- teamId: string;- url: string;- options: CrawlerOptions;- }[];+ success: true;+ crawls: {+ id: string;+ teamId: string;+ url: string;+ options: CrawlerOptions;+ }[];};-+export type CrawlErrorsResponse =| ErrorResponse| {- errors: {- id: string;- timestamp?: string;- url: string;- error: string;- }[];- robotsBlocked: string[];- };+ errors: {+ id: string;+ timestamp?: string;+ url: string;+ error: string;+ }[];+ robotsBlocked: string[];+ };type AuthObject = {team_id: string;@@ -1146,7 +1146,7 @@ export function fromLegacyScrapeOptions(? ("screenshot@fullPage" as const): null,extractorOptions !== undefined &&- extractorOptions.mode.includes("llm-extraction")+ extractorOptions.mode.includes("llm-extraction")? ("extract" as const): null,"links",@@ -1170,12 +1170,12 @@ export function fromLegacyScrapeOptions(removeBase64Images: pageOptions.removeBase64Images,extract:extractorOptions !== undefined &&- extractorOptions.mode.includes("llm-extraction")+ extractorOptions.mode.includes("llm-extraction")? {- systemPrompt: extractorOptions.extractionPrompt,- prompt: extractorOptions.userPrompt,- schema: extractorOptions.extractionSchema,- }+ systemPrompt: extractorOptions.extractionPrompt,+ prompt: extractorOptions.userPrompt,+ schema: extractorOptions.extractionSchema,+ }: undefined,mobile: pageOptions.mobile,fastMode: pageOptions.useFastMode,@@ -1290,10 +1290,10 @@ export type SearchRequestInput = z.input<typeof searchRequestSchema>;export type SearchResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document[];- };+ success: true;+ warning?: string;+ data: Document[];+ };export type TokenUsage = {promptTokens: number;