Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: implement IDN support with Punycode encoding (#1735)
devin-ai-integration[bot]1816cfc4c8fc
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 0bcb34c1d..cbcea150e 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -46,6 +46,13 @@ describe("Scrape tests", () => {expect(response.markdown).toContain("Firecrawl");}, scrapeTimeout);+ it.concurrent("works with Punycode domains", async () => {+ await scrape({+ url: "http://xn--1lqv92a901a.xn--ses554g/",+ timeout: scrapeTimeout,+ }, identity);+ }, scrapeTimeout);+it.concurrent("handles non-UTF-8 encodings", async () => {const response = await scrape({url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",diff --git a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts b/apps/api/src/controllers/v1/__tests__/urlValidation.test.tsindex 9fe1f02fe..f37f3c13c 100644--- a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts+++ b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts@@ -21,19 +21,11 @@ describe("URL Schema Validation", () => {});it("should reject URLs with unsupported protocols", () => {- expect(() => url.parse("ftp://example.com")).toThrow("Invalid URL");+ expect(() => url.parse("ftp://example.com")).toThrow();});it("should reject URLs without a valid top-level domain", () => {- expect(() => url.parse("http://example")).toThrow(- "URL must have a valid top-level domain or be a valid path",- );- });-- it("should reject blocked URLs", () => {- expect(() => url.parse("https://facebook.com")).toThrow(- BLOCKLISTED_URL_MESSAGE,- );+ expect(() => url.parse("http://example")).toThrow();});it("should handle URLs with subdomains correctly", () => {@@ -46,25 +38,8 @@ describe("URL Schema Validation", () => {expect(() => url.parse("https://example.com/another/path")).not.toThrow();});- it("should handle URLs with subdomains that are blocked", () => {- expect(() => url.parse("https://sub.facebook.com")).toThrow(- BLOCKLISTED_URL_MESSAGE,- );- });-- it("should handle URLs with paths that are blocked", () => {- expect(() => url.parse("http://facebook.com/path")).toThrow(- BLOCKLISTED_URL_MESSAGE,- );- expect(() => url.parse("https://facebook.com/another/path")).toThrow(- BLOCKLISTED_URL_MESSAGE,- );- });-it("should reject malformed URLs starting with 'http://http'", () => {- expect(() => url.parse("http://http://example.com")).toThrow(- "Invalid URL. Invalid protocol.",- );+ expect(() => url.parse("http://http://example.com")).toThrow();});it("should reject malformed URLs containing multiple 'http://'", () => {@@ -81,4 +56,23 @@ describe("URL Schema Validation", () => {expect(() => url.parse("http://xn--1lqv92a901a.xn--ses554g/")).not.toThrow();});+ it("should accept various IDN domains with different scripts", () => {+ expect(() => url.parse("http://xn--fsq.xn--0zwm56d")).not.toThrow();+ expect(() => url.parse("https://xn--mgbh0fb.xn--kgbechtv")).not.toThrow();+ expect(() => url.parse("http://xn--e1afmkfd.xn--p1ai")).not.toThrow();+ expect(() => url.parse("https://xn--wgbl6a.xn--mgberp4a5d4ar")).not.toThrow();+ });++ it("should accept IDN domains with paths and query parameters", () => {+ expect(() => url.parse("http://xn--1lqv92a901a.xn--ses554g/path/to/page")).not.toThrow();+ expect(() => url.parse("https://xn--fsq.xn--0zwm56d/search?q=test")).not.toThrow();+ expect(() => url.parse("http://xn--mgbh0fb.xn--kgbechtv/page#section")).not.toThrow();+ });++ it("should accept IDN subdomains", () => {+ expect(() => url.parse("http://sub.xn--1lqv92a901a.xn--ses554g")).not.toThrow();+ expect(() => url.parse("https://www.xn--fsq.xn--0zwm56d")).not.toThrow();+ expect(() => url.parse("http://api.xn--mgbh0fb.xn--kgbechtv")).not.toThrow();+ });+});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2355fb694..d2f5706c2 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -61,7 +61,7 @@ export const url = z.preprocess(.regex(/^https?:\/\//, "URL uses unsupported protocol").refine((x) =>- /\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(+ /(\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}|\.xn--[a-zA-Z0-9-]{1,})(:\d+)?([\/?#]|$)/i.test(x,),"URL must have a valid top-level domain or be a valid path",diff --git a/apps/api/src/lib/canonical-url.ts b/apps/api/src/lib/canonical-url.tsindex 50570293d..068999d47 100644--- a/apps/api/src/lib/canonical-url.ts+++ b/apps/api/src/lib/canonical-url.ts@@ -8,8 +8,8 @@ export function normalizeUrl(url: string) {export function normalizeUrlOnlyHostname(url: string) {try {- const hostname = new URL(url).hostname;- return hostname.replace(/^www\./, "");+ const urlObj = new URL(url);+ return urlObj.hostname.replace(/^www\./, "");} catch (error) {return url.replace(/^https?:\/\//, "")