Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

FIR-1951: Add automatic URL encoding in preprocessing for special characters in query parameters (#1547)

devin-ai-integration[bot]8eeb3c5cd4ce
diff --git a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts b/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts
new file mode 100644
index 000000000..c91ce0078
--- /dev/null
+++ b/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts
@@ -0,0 +1,34 @@
+import { url } from "../../../controllers/v1/types";
+import { describe, it, expect } from "@jest/globals";
+
+describe("URL Schema Validation with Special Characters", () => {
+ it("should handle URLs with special characters in query parameters", () => {
+ const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";
+
+ expect(() => url.parse(testUrl)).not.toThrow();
+
+ const parsedUrl = url.parse(testUrl);
+ expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");
+ });
+
+ it("should preserve URL structure when encoding special characters", () => {
+ const testUrl = "https://example.com/path?param1=value1&param2=value~with|special&param3=normal";
+
+ expect(() => url.parse(testUrl)).not.toThrow();
+
+ const parsedUrl = url.parse(testUrl);
+ expect(parsedUrl).toContain("example.com/path?");
+ expect(parsedUrl).toContain("param1=value1");
+ expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");
+ expect(parsedUrl).toContain("param3=normal");
+ });
+
+ it("should handle URLs with already encoded special characters", () => {
+ const testUrl = "https://example.com/path?param=value%7Eencoded";
+
+ expect(() => url.parse(testUrl)).not.toThrow();
+
+ const parsedUrl = url.parse(testUrl);
+ expect(parsedUrl).toContain("param=value%7Eencoded");
+ });
+});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 0ae2acc35..1a39037a3 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -24,8 +24,18 @@ export type Format =
export const url = z.preprocess(
(x) => {
if (!protocolIncluded(x as string)) {
- return `http://${x}`;
+ x = `http://${x}`;
}
+
+ try {
+ const urlObj = new URL(x as string);
+ if (urlObj.search) {
+ const searchParams = new URLSearchParams(urlObj.search.substring(1));
+ return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;
+ }
+ } catch (e) {
+ }
+
return x;
},
z