Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(branding): stop discarding LLM branding results on schema mismatches (#4808)
Gergő Móricz828a794d90e1
diff --git a/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.ts b/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.tsnew file mode 100644index 000000000..179eaded5--- /dev/null+++ b/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.ts@@ -0,0 +1,130 @@+import { describe, it, expect } from "vitest";+import { unwrapSchemaShapedAnswer } from "../../../lib/branding/llm";+import { getBrandingEnhancementSchema } from "../../../lib/branding/schema";++const answer = {+ buttonClassification: {+ primaryButtonIndex: 0,+ primaryButtonReasoning: "Vibrant brand color and action-oriented text.",+ secondaryButtonIndex: 2,+ secondaryButtonReasoning: "Different background color.",+ confidence: 0.9,+ },+ colorRoles: {+ primaryColor: "#5551FF",+ accentColor: "#5551FF",+ backgroundColor: "#FFFFFF",+ textPrimary: "#000000",+ confidence: 0.9,+ },+ personality: {+ tone: "modern",+ energy: "high",+ targetAudience: "developers",+ },+ designSystem: { framework: "tailwind", componentLibrary: "" },+ cleanedFonts: [{ family: "Inter", role: "body" }],+ logoSelection: {+ selectedLogoIndex: 0,+ selectedLogoReasoning: "Header logo linking to the homepage.",+ confidence: 0.9,+ },+};++describe("branding LLM schema", () => {+ it("accepts answers without personality or designSystem", () => {+ // What gpt-4o-mini returns for pages without buttons or logo candidates.+ const result = getBrandingEnhancementSchema(false).safeParse({+ colorRoles: answer.colorRoles,+ cleanedFonts: answer.cleanedFonts,+ });++ expect(result.success).toBe(true);+ expect(result.data?.buttonClassification.primaryButtonIndex).toBe(-1);+ });+});++describe("unwrapSchemaShapedAnswer", () => {+ it("unwraps an answer wrapped as {type, properties}", () => {+ const text = JSON.stringify({ type: "response", properties: answer });++ const repaired = JSON.parse(unwrapSchemaShapedAnswer(text)!);++ expect(repaired).toEqual(answer);+ expect(getBrandingEnhancementSchema(true).safeParse(repaired).success).toBe(+ true,+ );+ });++ it("recovers the answer from a schema echoed back with examples", () => {+ const leaf = (type: string, example: unknown) => ({+ type,+ description: "REQUIRED: YOU MUST RETURN THIS FIELD.",+ example,+ });+ const group = (values: Record<string, unknown>) => ({+ type: "object",+ properties: Object.fromEntries(+ Object.entries(values).map(([key, value]) => [+ key,+ leaf(typeof value === "number" ? "number" : "string", value),+ ]),+ ),+ });+ const text = JSON.stringify({+ type: "object",+ properties: {+ buttonClassification: group(answer.buttonClassification),+ colorRoles: group(answer.colorRoles),+ personality: group(answer.personality),+ designSystem: group(answer.designSystem),+ cleanedFonts: {+ type: "array",+ items: {+ type: "object",+ properties: {+ family: { type: "string" },+ role: { type: "string" },+ },+ },+ example: answer.cleanedFonts,+ },+ logoSelection: group(answer.logoSelection),+ },+ });++ const repaired = JSON.parse(unwrapSchemaShapedAnswer(text)!);++ expect(repaired).toEqual(answer);+ expect(getBrandingEnhancementSchema(true).safeParse(repaired).success).toBe(+ true,+ );+ });++ it("returns null when an echoed field carries no answer", () => {+ const text = JSON.stringify({+ type: "object",+ properties: {+ colorRoles: {+ type: "object",+ properties: {+ primaryColor: { type: "string", example: "#5551FF" },+ confidence: { type: "number" },+ },+ },+ cleanedFonts: {+ type: "array",+ items: { type: "object", properties: {} },+ },+ },+ });++ expect(unwrapSchemaShapedAnswer(text)).toBeNull();+ });++ it("returns null for text that is not a schema-shaped answer", () => {+ expect(unwrapSchemaShapedAnswer("not json")).toBeNull();+ expect(unwrapSchemaShapedAnswer(JSON.stringify(answer))).toBeNull();+ expect(unwrapSchemaShapedAnswer("[]")).toBeNull();+ });+});diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex bdd96c58b..7af7fb6bd 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -6,6 +6,77 @@ import { buildBrandingPrompt } from "./prompt";import { BrandingLLMInput } from "./types";import { getModel } from "../generic-ai";+const JSON_SCHEMA_TYPES = new Set([+ "string",+ "number",+ "integer",+ "boolean",+ "object",+ "array",+ "null",+]);++function isRecord(x: unknown): x is Record<string, unknown> {+ return typeof x === "object" && x !== null && !Array.isArray(x);+}++// A node of an echoed schema, e.g. {"type": "number", "example": 2}. None of+// the branding answer's own objects has a "type" key.+function isSchemaNode(x: unknown): x is Record<string, unknown> {+ return (+ isRecord(x) && typeof x.type === "string" && JSON_SCHEMA_TYPES.has(x.type)+ );+}++// Marks a schema node that carries no answer (no "example", no properties).+const UNRESOLVED = Symbol("unresolved");++function valueFromSchemaShape(node: unknown): unknown {+ if (!isSchemaNode(node)) return node;+ if ("example" in node) return node.example;+ if (isRecord(node.properties)) return propertiesToValue(node.properties);+ return UNRESOLVED;+}++function propertiesToValue(+ properties: Record<string, unknown>,+): Record<string, unknown> | typeof UNRESOLVED {+ const value: Record<string, unknown> = {};+ for (const [key, node] of Object.entries(properties)) {+ const resolved = valueFromSchemaShape(node);+ if (resolved === UNRESOLVED) return UNRESOLVED;+ value[key] = resolved;+ }+ return value;+}++/**+ * gpt-4o in non-strict mode sometimes answers in the shape of a JSON schema+ * instead of an instance of it: the answer wrapped as+ * {"type": "response", "properties": {...}}, or the schema echoed back with+ * each answer in an "example" field. Both carry the whole answer, so unwrap+ * them instead of failing the call. Returns null for anything else, including+ * an echo with any field left without an answer, so the SDK reports the+ * original error rather than one about half-repaired text.+ */+export function unwrapSchemaShapedAnswer(text: string): string | null {+ let parsed: unknown;+ try {+ parsed = JSON.parse(text);+ } catch {+ return null;+ }+ if (+ !isRecord(parsed) ||+ typeof parsed.type !== "string" ||+ !isRecord(parsed.properties)+ ) {+ return null;+ }+ const value = propertiesToValue(parsed.properties);+ return value === UNRESOLVED ? null : JSON.stringify(value);+}+function isDebugBrandingEnabled(input: BrandingLLMInput): boolean {return (config.DEBUG_BRANDING === true || input.teamFlags?.debugBranding === true@@ -118,6 +189,9 @@ export async function enhanceBrandingWithLLM(},],temperature: 0.1,+ // Only called once the response failed to parse or validate.+ experimental_repairText: async ({ text }) =>+ unwrapSchemaShapedAnswer(text),experimental_telemetry: {isEnabled: true,// The input carries the page screenshot / raw page content; too largediff --git a/apps/api/src/lib/branding/schema.ts b/apps/api/src/lib/branding/schema.tsindex fd36b1222..743556ba6 100644--- a/apps/api/src/lib/branding/schema.ts+++ b/apps/api/src/lib/branding/schema.ts@@ -56,38 +56,43 @@ const baseBrandingEnhancementSchema = z.object({confidence: z.number().min(0).max(1),}),- // Brand personality- personality: z.object({- tone: z- .enum([- "professional",- "playful",- "modern",- "traditional",- "minimalist",- "bold",- ])- .describe("Overall brand tone"),- energy: z.enum(["low", "medium", "high"]).describe("Visual energy level"),- targetAudience: z.string().describe("Perceived target audience"),- }),+ // Brand personality. Optional: gpt-4o-mini (non-strict mode) often leaves+ // it out, and failing the whole call over it discarded the colors and fonts.+ personality: z+ .object({+ tone: z+ .enum([+ "professional",+ "playful",+ "modern",+ "traditional",+ "minimalist",+ "bold",+ ])+ .describe("Overall brand tone"),+ energy: z.enum(["low", "medium", "high"]).describe("Visual energy level"),+ targetAudience: z.string().describe("Perceived target audience"),+ })+ .optional(),- // Design system insights- designSystem: z.object({- framework: z- .enum([- "tailwind",- "bootstrap",- "material",- "chakra",- "custom",- "unknown",- ])- .describe("Detected CSS framework"),- componentLibrary: z- .string()- .describe("Detected component library (e.g., radix-ui, shadcn)"),- }),+ // Design system insights. Optional for the same reason as personality.+ designSystem: z+ .object({+ framework: z+ .enum([+ "tailwind",+ "bootstrap",+ "material",+ "chakra",+ "custom",+ "unknown",+ ])+ .describe("Detected CSS framework"),+ componentLibrary: z+ .string()+ .describe("Detected component library (e.g., radix-ui, shadcn)"),+ })+ .optional(),// Font cleaning - LLM cleans and filters font namescleanedFonts: z