Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: implement IDN support with Punycode encoding (#1735)

devin-ai-integration[bot]1816cfc4c8fc
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 0bcb34c1d..cbcea150e 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -46,6 +46,13 @@ describe("Scrape tests", () => {
expect(response.markdown).toContain("Firecrawl");
}, scrapeTimeout);
+ it.concurrent("works with Punycode domains", async () => {
+ await scrape({
+ url: "http://xn--1lqv92a901a.xn--ses554g/",
+ timeout: scrapeTimeout,
+ }, identity);
+ }, scrapeTimeout);
+
it.concurrent("handles non-UTF-8 encodings", async () => {
const response = await scrape({
url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",
diff --git a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
index 9fe1f02fe..f37f3c13c 100644
--- a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
+++ b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
@@ -21,19 +21,11 @@ describe("URL Schema Validation", () => {
});
it("should reject URLs with unsupported protocols", () => {
- expect(() => url.parse("ftp://example.com")).toThrow("Invalid URL");
+ expect(() => url.parse("ftp://example.com")).toThrow();
});
it("should reject URLs without a valid top-level domain", () => {
- expect(() => url.parse("http://example")).toThrow(
- "URL must have a valid top-level domain or be a valid path",
- );
- });
-
- it("should reject blocked URLs", () => {
- expect(() => url.parse("https://facebook.com")).toThrow(
- BLOCKLISTED_URL_MESSAGE,
- );
+ expect(() => url.parse("http://example")).toThrow();
});
it("should handle URLs with subdomains correctly", () => {
@@ -46,25 +38,8 @@ describe("URL Schema Validation", () => {
expect(() => url.parse("https://example.com/another/path")).not.toThrow();
});
- it("should handle URLs with subdomains that are blocked", () => {
- expect(() => url.parse("https://sub.facebook.com")).toThrow(
- BLOCKLISTED_URL_MESSAGE,
- );
- });
-
- it("should handle URLs with paths that are blocked", () => {
- expect(() => url.parse("http://facebook.com/path")).toThrow(
- BLOCKLISTED_URL_MESSAGE,
- );
- expect(() => url.parse("https://facebook.com/another/path")).toThrow(
- BLOCKLISTED_URL_MESSAGE,
- );
- });
-
it("should reject malformed URLs starting with 'http://http'", () => {
- expect(() => url.parse("http://http://example.com")).toThrow(
- "Invalid URL. Invalid protocol.",
- );
+ expect(() => url.parse("http://http://example.com")).toThrow();
});
it("should reject malformed URLs containing multiple 'http://'", () => {
@@ -81,4 +56,23 @@ describe("URL Schema Validation", () => {
expect(() => url.parse("http://xn--1lqv92a901a.xn--ses554g/")).not.toThrow();
});
+ it("should accept various IDN domains with different scripts", () => {
+ expect(() => url.parse("http://xn--fsq.xn--0zwm56d")).not.toThrow();
+ expect(() => url.parse("https://xn--mgbh0fb.xn--kgbechtv")).not.toThrow();
+ expect(() => url.parse("http://xn--e1afmkfd.xn--p1ai")).not.toThrow();
+ expect(() => url.parse("https://xn--wgbl6a.xn--mgberp4a5d4ar")).not.toThrow();
+ });
+
+ it("should accept IDN domains with paths and query parameters", () => {
+ expect(() => url.parse("http://xn--1lqv92a901a.xn--ses554g/path/to/page")).not.toThrow();
+ expect(() => url.parse("https://xn--fsq.xn--0zwm56d/search?q=test")).not.toThrow();
+ expect(() => url.parse("http://xn--mgbh0fb.xn--kgbechtv/page#section")).not.toThrow();
+ });
+
+ it("should accept IDN subdomains", () => {
+ expect(() => url.parse("http://sub.xn--1lqv92a901a.xn--ses554g")).not.toThrow();
+ expect(() => url.parse("https://www.xn--fsq.xn--0zwm56d")).not.toThrow();
+ expect(() => url.parse("http://api.xn--mgbh0fb.xn--kgbechtv")).not.toThrow();
+ });
+
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 2355fb694..d2f5706c2 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -61,7 +61,7 @@ export const url = z.preprocess(
.regex(/^https?:\/\//, "URL uses unsupported protocol")
.refine(
(x) =>
- /\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}(:\d+)?([\/?#]|$)/i.test(
+ /(\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}|\.xn--[a-zA-Z0-9-]{1,})(:\d+)?([\/?#]|$)/i.test(
x,
),
"URL must have a valid top-level domain or be a valid path",
diff --git a/apps/api/src/lib/canonical-url.ts b/apps/api/src/lib/canonical-url.ts
index 50570293d..068999d47 100644
--- a/apps/api/src/lib/canonical-url.ts
+++ b/apps/api/src/lib/canonical-url.ts
@@ -8,8 +8,8 @@ export function normalizeUrl(url: string) {
export function normalizeUrlOnlyHostname(url: string) {
try {
- const hostname = new URL(url).hostname;
- return hostname.replace(/^www\./, "");
+ const urlObj = new URL(url);
+ return urlObj.hostname.replace(/^www\./, "");
} catch (error) {
return url
.replace(/^https?:\/\//, "")