Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix confusing error when missing content type json (#4207)
rowinsg9554ad079840
diff --git a/apps/api/src/controllers/__tests__/url-schema.test.ts b/apps/api/src/controllers/__tests__/url-schema.test.tsnew file mode 100644index 000000000..83be4f93e--- /dev/null+++ b/apps/api/src/controllers/__tests__/url-schema.test.ts@@ -0,0 +1,63 @@+import { describe, it, expect } from "vitest";+import { z } from "zod";+import { url as urlSchemaV1 } from "../v1/types";+import { URL as urlSchemaV2 } from "../v2/types";++/**+ * Pins the ordering these schemas depend on: the string check must run before+ * the scheme is prepended.+ *+ * Were it the other way round, a missing url would be interpolated into+ * "http://undefined" β structurally a valid URL, so the only check left to fail+ * would be the top-level-domain refinement, and the caller would be told their+ * url had a bad TLD when they never sent one. Reordering reintroduces that, so+ * these tests hold the order in place.+ */+describe.each([+ ["v1", urlSchemaV1],+ ["v2", urlSchemaV2],+])("%s url schema", (_version, schema) => {+ const parse = (value: unknown) =>+ z.object({ url: schema as z.ZodType }).safeParse(value as object);++ it("reports a missing url as a type error, not a TLD complaint", () => {+ const result = parse({});++ expect(result.success).toBe(false);+ const issue = result.error!.issues[0];+ expect(issue.code).toBe("invalid_type");+ expect(issue.path).toEqual(["url"]);+ expect(issue.message).not.toContain("top-level domain");+ });++ it.each([[null], [123], [{}], [[]]])(+ "does not turn %s into a plausible URL",+ value => {+ const result = parse({ url: value });++ expect(result.success).toBe(false);+ expect(result.error!.issues[0].message).not.toContain("top-level domain");+ },+ );++ it("still prepends a scheme to a bare hostname", () => {+ const result = parse({ url: "example.com/path" });++ expect(result.success).toBe(true);+ expect(result.data!.url).toBe("http://example.com/path");+ });++ it("leaves an explicit scheme alone", () => {+ const result = parse({ url: "https://example.com/path" });++ expect(result.success).toBe(true);+ expect(result.data!.url).toBe("https://example.com/path");+ });++ it("still rejects a host with no usable TLD", () => {+ const result = parse({ url: "http://nodots" });++ expect(result.success).toBe(false);+ expect(result.error!.issues[0].message).toContain("top-level domain");+ });+});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 027f982e5..23fc0bd9e 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -38,8 +38,15 @@ type Format =| "product"| "menu";-export const url = z.preprocess(- x => {+export const url = z+ .string()+ // .overwrite must stay after .string(). It interpolates x into a template, so+ // a non-string url would become the string "http://undefined" β a valid URL+ // whose only failure is the TLD check below, which would blame a field the+ // caller never sent. Letting .string() reject it first gives the real error.+ // .overwrite rather than .transform, so this stays a ZodString and .url() /+ // .regex() can still chain onto it.+ .overwrite(x => {if (!protocolIncluded(x as string)) {x = `http://${x}`;}@@ -55,35 +62,33 @@ export const url = z.preprocess(// }return x;- },- z- .url()- .regex(/^https?:\/\//i, "URL uses unsupported protocol")- .refine(x => {- if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {- if (- /^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(- x as string,- )- ) {- return true;- }- }- // Same TLD-shaped check as before, plus IP literals β which the old- // inline regex accepted only when the last octet had 2+ digits, so- // 8.8.8.8 was rejected while 169.254.169.254 passed.- return hasReachableHost(x as string);- }, "URL must have a valid top-level domain or be an IP address")- .refine(x => {- try {- checkUrl(x as string);+ })+ .url()+ .regex(/^https?:\/\//i, "URL uses unsupported protocol")+ .refine(x => {+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {+ if (+ /^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(+ x as string,+ )+ ) {return true;- } catch (_) {- return false;}- }, "Invalid URL"),- // .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE),-);+ }+ // Same TLD-shaped check as before, plus IP literals β which the old+ // inline regex accepted only when the last octet had 2+ digits, so+ // 8.8.8.8 was rejected while 169.254.169.254 passed.+ return hasReachableHost(x as string);+ }, "URL must have a valid top-level domain or be an IP address")+ .refine(x => {+ try {+ checkUrl(x as string);+ return true;+ } catch (_) {+ return false;+ }+ }, "Invalid URL");+// .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE)const agentExtractModelValue = "fire-1";export const isAgentExtractModelValid = (x: string | undefined) =>diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex a21a0d87f..466fb2c2a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -33,8 +33,15 @@ import { threatProtectionOverrideSchema } from "../../lib/threat-protection/confimport { auditMetadataSchema } from "../../lib/siem-logging/types";// Base URL schema with common validation logic-export const URL = z.preprocess(- x => {+export const URL = z+ .string()+ // .overwrite must stay after .string(). It interpolates x into a template, so+ // a non-string url would become the string "http://undefined" β a valid URL+ // whose only failure is the TLD check below, which would blame a field the+ // caller never sent. Letting .string() reject it first gives the real error.+ // .overwrite rather than .transform, so this stays a ZodString and .url() /+ // .regex() can still chain onto it.+ .overwrite(x => {if (!protocolIncluded(x as string)) {x = `http://${x}`;}@@ -50,35 +57,33 @@ export const URL = z.preprocess(// }return x;- },- z- .url()- .regex(/^https?:\/\//i, "URL uses unsupported protocol")- .refine(x => {- if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {- if (- /^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(- x as string,- )- ) {- return true;- }- }- // Same TLD-shaped check as before, plus IP literals β which the old- // inline regex accepted only when the last octet had 2+ digits, so- // 8.8.8.8 was rejected while 169.254.169.254 passed.- return hasReachableHost(x as string);- }, "URL must have a valid top-level domain or be an IP address")- .refine(x => {- try {- checkUrl(x as string);+ })+ .url()+ .regex(/^https?:\/\//i, "URL uses unsupported protocol")+ .refine(x => {+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {+ if (+ /^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(+ x as string,+ )+ ) {return true;- } catch (_) {- return false;}- }, "Invalid URL"),- // .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE),-);+ }+ // Same TLD-shaped check as before, plus IP literals β which the old+ // inline regex accepted only when the last octet had 2+ digits, so+ // 8.8.8.8 was rejected while 169.254.169.254 passed.+ return hasReachableHost(x as string);+ }, "URL must have a valid top-level domain or be an IP address")+ .refine(x => {+ try {+ checkUrl(x as string);+ return true;+ } catch (_) {+ return false;+ }+ }, "Invalid URL");+// .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE)const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";