:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): surface the real metrics error from nullable union rejections (#128664)
ruby childse07a2a60a57d
diff --git a/services/mcp/src/tools/exec.ts b/services/mcp/src/tools/exec.tsindex b6451e3a3d41..d78b0f465c96 100644--- a/services/mcp/src/tools/exec.ts+++ b/services/mcp/src/tools/exec.ts@@ -1266,8 +1266,16 @@ function bestUnionBranch(branches: readonly (readonly z.core.$ZodIssue[])[]): reselector === undefined? populated.filter((branch) => pinnedValues(branch).size === 0): populated.filter((branch) => !rejectedValueKeys(branch).has(selector))+ const candidates = named.length > 0 ? named : populated+ // A nullable field — `union([array(metric), null])` on the experiment metrics+ // — rejects a malformed array with the null branch's single complaint,+ // "expected null, received array", as the shortest list, burying the real+ // error inside the array branch. A branch with an issue below the root+ // engaged with the value's contents, so when one exists, a branch that only+ // rejects the whole value's type restates what the caller already knows.+ const engaged = candidates.filter((branch) => !rejectsWholeValueType(branch))let best: readonly z.core.$ZodIssue[] | undefined- for (const branch of named.length > 0 ? named : populated) {+ for (const branch of engaged.length > 0 ? engaged : candidates) {if (best === undefined || branch.length < best.length) {best = branch}@@ -1275,6 +1283,13 @@ function bestUnionBranch(branches: readonly (readonly z.core.$ZodIssue[])[]): rereturn best}+/** True when the branch's every issue says the whole value is the wrong type,+ * with nothing about its contents — the shape of a `null` branch rejecting an+ * array, or a scalar branch rejecting an object. */+function rejectsWholeValueType(branch: readonly z.core.$ZodIssue[]): boolean {+ return branch.every((issue) => issue.code === 'invalid_type' && issue.path.length === 0)+}+/** The accepted values, when every branch of a union rejects the same enum* value because the options are split across several enums. */function unionValueOptions(branches: readonly (readonly z.core.$ZodIssue[])[]): string[] | undefined {diff --git a/services/mcp/tests/unit/exec.test.ts b/services/mcp/tests/unit/exec.test.tsindex 730a6543bdf2..5018ed625fa9 100644--- a/services/mcp/tests/unit/exec.test.ts+++ b/services/mcp/tests/unit/exec.test.ts@@ -3182,5 +3182,70 @@ describe('exec tool', () => {expect(message).toContain('parameter "series.0.nodes.0.properties.0.value"')})})++ // Production traces show experiment-update rejections where a metrics+ // payload failing deep validation read "parameter 'metrics_secondary':+ // Invalid input: expected null, received array" — the null branch of the+ // nullable metrics union, not the real error inside the array branch.+ describe('a nullable field whose non-null branch failed inside', () => {+ const formatFor = (input: unknown): string => {+ const tool = GENERATED_TOOL_MAP['experiment-update']!()+ const result = tool.schema.safeParse(input, { reportInput: true })+ expect(result.success).toBe(false)+ return formatInputValidationError('experiment-update', result.error!, input, tool.schema)+ }++ // Two invalid entries, not one: a single bad metric yields one array-branch+ // issue, which TIES the null branch's single complaint, and the pre-fix+ // formatter already broke that tie in the array branch's favor. Two bad+ // metrics make the array branch "noisier" than the null branch, which is+ // the real production shape the old fewest-issues heuristic got wrong.+ it.each(['metrics', 'metrics_secondary'] as const)(+ 'names the field inside the %s array, not the null branch',+ (field) => {+ const message = formatFor({+ id: 1,+ [field]: [+ { kind: 'ExperimentMetric', metric_type: 'bogus' },+ { kind: 'ExperimentMetric', metric_type: 'bogus' },+ ],+ })++ expect(message).toContain(`${field}.0.metric_type`)+ expect(message).not.toContain('expected null')+ }+ )++ it('still accepts an explicit null', () => {+ const tool = GENERATED_TOOL_MAP['experiment-update']!()+ const result = tool.schema.safeParse({ id: 1, metrics_secondary: null }, { reportInput: true })+ expect(result.success, JSON.stringify(result.error?.issues)).toBe(true)+ })++ it('keeps the honest type error when the value matches no branch at all', () => {+ const message = formatFor({ id: 1, metrics_secondary: 'garbage' })++ expect(message).toContain('metrics_secondary')+ expect(message).toContain('expected array')+ })++ // The synthetic shape pins the mechanism down without the generated+ // schema's size: the null branch's single root complaint must lose to+ // the branch that engaged with the array's contents, even when the+ // engaged branch carries MORE issues (two bad entries) than null's one.+ it('prefers the engaged branch on a minimal union([array, null])', () => {+ const schema = z.object({+ metrics: z.union([z.array(z.object({ metric_type: z.enum(['mean', 'funnel']) })), z.null()]),+ })+ const input = { metrics: [{ metric_type: 'bogus' }, { metric_type: 'bogus' }] }+ const result = schema.safeParse(input, { reportInput: true })+ expect(result.success).toBe(false)++ const message = formatInputValidationError('some-tool', result.error!, input, schema)++ expect(message).toContain('metrics.0.metric_type')+ expect(message).not.toContain('expected null')+ })+ })})})