Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
FIR-1951: Add automatic URL encoding in preprocessing for special characters in query parameters (#1547)
devin-ai-integration[bot]8eeb3c5cd4ce
diff --git a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts b/apps/api/src/__tests__/snips/url-validation/special-characters.test.tsnew file mode 100644index 000000000..c91ce0078--- /dev/null+++ b/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts@@ -0,0 +1,34 @@+import { url } from "../../../controllers/v1/types";+import { describe, it, expect } from "@jest/globals";++describe("URL Schema Validation with Special Characters", () => {+ it("should handle URLs with special characters in query parameters", () => {+ const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";++ expect(() => url.parse(testUrl)).not.toThrow();++ const parsedUrl = url.parse(testUrl);+ expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");+ });++ it("should preserve URL structure when encoding special characters", () => {+ const testUrl = "https://example.com/path?param1=value1¶m2=value~with|special¶m3=normal";++ expect(() => url.parse(testUrl)).not.toThrow();++ const parsedUrl = url.parse(testUrl);+ expect(parsedUrl).toContain("example.com/path?");+ expect(parsedUrl).toContain("param1=value1");+ expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");+ expect(parsedUrl).toContain("param3=normal");+ });++ it("should handle URLs with already encoded special characters", () => {+ const testUrl = "https://example.com/path?param=value%7Eencoded";++ expect(() => url.parse(testUrl)).not.toThrow();++ const parsedUrl = url.parse(testUrl);+ expect(parsedUrl).toContain("param=value%7Eencoded");+ });+});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 0ae2acc35..1a39037a3 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -24,8 +24,18 @@ export type Format =export const url = z.preprocess((x) => {if (!protocolIncluded(x as string)) {- return `http://${x}`;+ x = `http://${x}`;}++ try {+ const urlObj = new URL(x as string);+ if (urlObj.search) {+ const searchParams = new URLSearchParams(urlObj.search.substring(1));+ return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;+ }+ } catch (e) {+ }+return x;},z