Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(branding): stop discarding LLM branding results on schema mismatches (#4808)

Gergő Móricz828a794d90e1
diff --git a/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.ts b/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.ts
new file mode 100644
index 000000000..179eaded5
--- /dev/null
+++ b/apps/api/src/__tests__/lib/branding/llm-schema-shaped-answer.test.ts
@@ -0,0 +1,130 @@
+import { describe, it, expect } from "vitest";
+import { unwrapSchemaShapedAnswer } from "../../../lib/branding/llm";
+import { getBrandingEnhancementSchema } from "../../../lib/branding/schema";
+
+const answer = {
+ buttonClassification: {
+ primaryButtonIndex: 0,
+ primaryButtonReasoning: "Vibrant brand color and action-oriented text.",
+ secondaryButtonIndex: 2,
+ secondaryButtonReasoning: "Different background color.",
+ confidence: 0.9,
+ },
+ colorRoles: {
+ primaryColor: "#5551FF",
+ accentColor: "#5551FF",
+ backgroundColor: "#FFFFFF",
+ textPrimary: "#000000",
+ confidence: 0.9,
+ },
+ personality: {
+ tone: "modern",
+ energy: "high",
+ targetAudience: "developers",
+ },
+ designSystem: { framework: "tailwind", componentLibrary: "" },
+ cleanedFonts: [{ family: "Inter", role: "body" }],
+ logoSelection: {
+ selectedLogoIndex: 0,
+ selectedLogoReasoning: "Header logo linking to the homepage.",
+ confidence: 0.9,
+ },
+};
+
+describe("branding LLM schema", () => {
+ it("accepts answers without personality or designSystem", () => {
+ // What gpt-4o-mini returns for pages without buttons or logo candidates.
+ const result = getBrandingEnhancementSchema(false).safeParse({
+ colorRoles: answer.colorRoles,
+ cleanedFonts: answer.cleanedFonts,
+ });
+
+ expect(result.success).toBe(true);
+ expect(result.data?.buttonClassification.primaryButtonIndex).toBe(-1);
+ });
+});
+
+describe("unwrapSchemaShapedAnswer", () => {
+ it("unwraps an answer wrapped as {type, properties}", () => {
+ const text = JSON.stringify({ type: "response", properties: answer });
+
+ const repaired = JSON.parse(unwrapSchemaShapedAnswer(text)!);
+
+ expect(repaired).toEqual(answer);
+ expect(getBrandingEnhancementSchema(true).safeParse(repaired).success).toBe(
+ true,
+ );
+ });
+
+ it("recovers the answer from a schema echoed back with examples", () => {
+ const leaf = (type: string, example: unknown) => ({
+ type,
+ description: "REQUIRED: YOU MUST RETURN THIS FIELD.",
+ example,
+ });
+ const group = (values: Record<string, unknown>) => ({
+ type: "object",
+ properties: Object.fromEntries(
+ Object.entries(values).map(([key, value]) => [
+ key,
+ leaf(typeof value === "number" ? "number" : "string", value),
+ ]),
+ ),
+ });
+ const text = JSON.stringify({
+ type: "object",
+ properties: {
+ buttonClassification: group(answer.buttonClassification),
+ colorRoles: group(answer.colorRoles),
+ personality: group(answer.personality),
+ designSystem: group(answer.designSystem),
+ cleanedFonts: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ family: { type: "string" },
+ role: { type: "string" },
+ },
+ },
+ example: answer.cleanedFonts,
+ },
+ logoSelection: group(answer.logoSelection),
+ },
+ });
+
+ const repaired = JSON.parse(unwrapSchemaShapedAnswer(text)!);
+
+ expect(repaired).toEqual(answer);
+ expect(getBrandingEnhancementSchema(true).safeParse(repaired).success).toBe(
+ true,
+ );
+ });
+
+ it("returns null when an echoed field carries no answer", () => {
+ const text = JSON.stringify({
+ type: "object",
+ properties: {
+ colorRoles: {
+ type: "object",
+ properties: {
+ primaryColor: { type: "string", example: "#5551FF" },
+ confidence: { type: "number" },
+ },
+ },
+ cleanedFonts: {
+ type: "array",
+ items: { type: "object", properties: {} },
+ },
+ },
+ });
+
+ expect(unwrapSchemaShapedAnswer(text)).toBeNull();
+ });
+
+ it("returns null for text that is not a schema-shaped answer", () => {
+ expect(unwrapSchemaShapedAnswer("not json")).toBeNull();
+ expect(unwrapSchemaShapedAnswer(JSON.stringify(answer))).toBeNull();
+ expect(unwrapSchemaShapedAnswer("[]")).toBeNull();
+ });
+});
diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.ts
index bdd96c58b..7af7fb6bd 100644
--- a/apps/api/src/lib/branding/llm.ts
+++ b/apps/api/src/lib/branding/llm.ts
@@ -6,6 +6,77 @@ import { buildBrandingPrompt } from "./prompt";
import { BrandingLLMInput } from "./types";
import { getModel } from "../generic-ai";
+const JSON_SCHEMA_TYPES = new Set([
+ "string",
+ "number",
+ "integer",
+ "boolean",
+ "object",
+ "array",
+ "null",
+]);
+
+function isRecord(x: unknown): x is Record<string, unknown> {
+ return typeof x === "object" && x !== null && !Array.isArray(x);
+}
+
+// A node of an echoed schema, e.g. {"type": "number", "example": 2}. None of
+// the branding answer's own objects has a "type" key.
+function isSchemaNode(x: unknown): x is Record<string, unknown> {
+ return (
+ isRecord(x) && typeof x.type === "string" && JSON_SCHEMA_TYPES.has(x.type)
+ );
+}
+
+// Marks a schema node that carries no answer (no "example", no properties).
+const UNRESOLVED = Symbol("unresolved");
+
+function valueFromSchemaShape(node: unknown): unknown {
+ if (!isSchemaNode(node)) return node;
+ if ("example" in node) return node.example;
+ if (isRecord(node.properties)) return propertiesToValue(node.properties);
+ return UNRESOLVED;
+}
+
+function propertiesToValue(
+ properties: Record<string, unknown>,
+): Record<string, unknown> | typeof UNRESOLVED {
+ const value: Record<string, unknown> = {};
+ for (const [key, node] of Object.entries(properties)) {
+ const resolved = valueFromSchemaShape(node);
+ if (resolved === UNRESOLVED) return UNRESOLVED;
+ value[key] = resolved;
+ }
+ return value;
+}
+
+/**
+ * gpt-4o in non-strict mode sometimes answers in the shape of a JSON schema
+ * instead of an instance of it: the answer wrapped as
+ * {"type": "response", "properties": {...}}, or the schema echoed back with
+ * each answer in an "example" field. Both carry the whole answer, so unwrap
+ * them instead of failing the call. Returns null for anything else, including
+ * an echo with any field left without an answer, so the SDK reports the
+ * original error rather than one about half-repaired text.
+ */
+export function unwrapSchemaShapedAnswer(text: string): string | null {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ return null;
+ }
+ if (
+ !isRecord(parsed) ||
+ typeof parsed.type !== "string" ||
+ !isRecord(parsed.properties)
+ ) {
+ return null;
+ }
+ const value = propertiesToValue(parsed.properties);
+ return value === UNRESOLVED ? null : JSON.stringify(value);
+}
+
function isDebugBrandingEnabled(input: BrandingLLMInput): boolean {
return (
config.DEBUG_BRANDING === true || input.teamFlags?.debugBranding === true
@@ -118,6 +189,9 @@ export async function enhanceBrandingWithLLM(
},
],
temperature: 0.1,
+ // Only called once the response failed to parse or validate.
+ experimental_repairText: async ({ text }) =>
+ unwrapSchemaShapedAnswer(text),
experimental_telemetry: {
isEnabled: true,
// The input carries the page screenshot / raw page content; too large
diff --git a/apps/api/src/lib/branding/schema.ts b/apps/api/src/lib/branding/schema.ts
index fd36b1222..743556ba6 100644
--- a/apps/api/src/lib/branding/schema.ts
+++ b/apps/api/src/lib/branding/schema.ts
@@ -56,38 +56,43 @@ const baseBrandingEnhancementSchema = z.object({
confidence: z.number().min(0).max(1),
}),
- // Brand personality
- personality: z.object({
- tone: z
- .enum([
- "professional",
- "playful",
- "modern",
- "traditional",
- "minimalist",
- "bold",
- ])
- .describe("Overall brand tone"),
- energy: z.enum(["low", "medium", "high"]).describe("Visual energy level"),
- targetAudience: z.string().describe("Perceived target audience"),
- }),
+ // Brand personality. Optional: gpt-4o-mini (non-strict mode) often leaves
+ // it out, and failing the whole call over it discarded the colors and fonts.
+ personality: z
+ .object({
+ tone: z
+ .enum([
+ "professional",
+ "playful",
+ "modern",
+ "traditional",
+ "minimalist",
+ "bold",
+ ])
+ .describe("Overall brand tone"),
+ energy: z.enum(["low", "medium", "high"]).describe("Visual energy level"),
+ targetAudience: z.string().describe("Perceived target audience"),
+ })
+ .optional(),
- // Design system insights
- designSystem: z.object({
- framework: z
- .enum([
- "tailwind",
- "bootstrap",
- "material",
- "chakra",
- "custom",
- "unknown",
- ])
- .describe("Detected CSS framework"),
- componentLibrary: z
- .string()
- .describe("Detected component library (e.g., radix-ui, shadcn)"),
- }),
+ // Design system insights. Optional for the same reason as personality.
+ designSystem: z
+ .object({
+ framework: z
+ .enum([
+ "tailwind",
+ "bootstrap",
+ "material",
+ "chakra",
+ "custom",
+ "unknown",
+ ])
+ .describe("Detected CSS framework"),
+ componentLibrary: z
+ .string()
+ .describe("Detected component library (e.g., radix-ui, shadcn)"),
+ })
+ .optional(),
// Font cleaning - LLM cleans and filters font names
cleanedFonts: z