Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): surface the real metrics error from nullable union rejections (#128664)

ruby childse07a2a60a57d
diff --git a/services/mcp/src/tools/exec.ts b/services/mcp/src/tools/exec.ts
index b6451e3a3d41..d78b0f465c96 100644
--- a/services/mcp/src/tools/exec.ts
+++ b/services/mcp/src/tools/exec.ts
@@ -1266,8 +1266,16 @@ function bestUnionBranch(branches: readonly (readonly z.core.$ZodIssue[])[]): re
selector === undefined
? populated.filter((branch) => pinnedValues(branch).size === 0)
: populated.filter((branch) => !rejectedValueKeys(branch).has(selector))
+ const candidates = named.length > 0 ? named : populated
+ // A nullable field — `union([array(metric), null])` on the experiment metrics
+ // — rejects a malformed array with the null branch's single complaint,
+ // "expected null, received array", as the shortest list, burying the real
+ // error inside the array branch. A branch with an issue below the root
+ // engaged with the value's contents, so when one exists, a branch that only
+ // rejects the whole value's type restates what the caller already knows.
+ const engaged = candidates.filter((branch) => !rejectsWholeValueType(branch))
let best: readonly z.core.$ZodIssue[] | undefined
- for (const branch of named.length > 0 ? named : populated) {
+ for (const branch of engaged.length > 0 ? engaged : candidates) {
if (best === undefined || branch.length < best.length) {
best = branch
}
@@ -1275,6 +1283,13 @@ function bestUnionBranch(branches: readonly (readonly z.core.$ZodIssue[])[]): re
return best
}
+/** True when the branch's every issue says the whole value is the wrong type,
+ * with nothing about its contents — the shape of a `null` branch rejecting an
+ * array, or a scalar branch rejecting an object. */
+function rejectsWholeValueType(branch: readonly z.core.$ZodIssue[]): boolean {
+ return branch.every((issue) => issue.code === 'invalid_type' && issue.path.length === 0)
+}
+
/** The accepted values, when every branch of a union rejects the same enum
* value because the options are split across several enums. */
function unionValueOptions(branches: readonly (readonly z.core.$ZodIssue[])[]): string[] | undefined {
diff --git a/services/mcp/tests/unit/exec.test.ts b/services/mcp/tests/unit/exec.test.ts
index 730a6543bdf2..5018ed625fa9 100644
--- a/services/mcp/tests/unit/exec.test.ts
+++ b/services/mcp/tests/unit/exec.test.ts
@@ -3182,5 +3182,70 @@ describe('exec tool', () => {
expect(message).toContain('parameter "series.0.nodes.0.properties.0.value"')
})
})
+
+ // Production traces show experiment-update rejections where a metrics
+ // payload failing deep validation read "parameter 'metrics_secondary':
+ // Invalid input: expected null, received array" — the null branch of the
+ // nullable metrics union, not the real error inside the array branch.
+ describe('a nullable field whose non-null branch failed inside', () => {
+ const formatFor = (input: unknown): string => {
+ const tool = GENERATED_TOOL_MAP['experiment-update']!()
+ const result = tool.schema.safeParse(input, { reportInput: true })
+ expect(result.success).toBe(false)
+ return formatInputValidationError('experiment-update', result.error!, input, tool.schema)
+ }
+
+ // Two invalid entries, not one: a single bad metric yields one array-branch
+ // issue, which TIES the null branch's single complaint, and the pre-fix
+ // formatter already broke that tie in the array branch's favor. Two bad
+ // metrics make the array branch "noisier" than the null branch, which is
+ // the real production shape the old fewest-issues heuristic got wrong.
+ it.each(['metrics', 'metrics_secondary'] as const)(
+ 'names the field inside the %s array, not the null branch',
+ (field) => {
+ const message = formatFor({
+ id: 1,
+ [field]: [
+ { kind: 'ExperimentMetric', metric_type: 'bogus' },
+ { kind: 'ExperimentMetric', metric_type: 'bogus' },
+ ],
+ })
+
+ expect(message).toContain(`${field}.0.metric_type`)
+ expect(message).not.toContain('expected null')
+ }
+ )
+
+ it('still accepts an explicit null', () => {
+ const tool = GENERATED_TOOL_MAP['experiment-update']!()
+ const result = tool.schema.safeParse({ id: 1, metrics_secondary: null }, { reportInput: true })
+ expect(result.success, JSON.stringify(result.error?.issues)).toBe(true)
+ })
+
+ it('keeps the honest type error when the value matches no branch at all', () => {
+ const message = formatFor({ id: 1, metrics_secondary: 'garbage' })
+
+ expect(message).toContain('metrics_secondary')
+ expect(message).toContain('expected array')
+ })
+
+ // The synthetic shape pins the mechanism down without the generated
+ // schema's size: the null branch's single root complaint must lose to
+ // the branch that engaged with the array's contents, even when the
+ // engaged branch carries MORE issues (two bad entries) than null's one.
+ it('prefers the engaged branch on a minimal union([array, null])', () => {
+ const schema = z.object({
+ metrics: z.union([z.array(z.object({ metric_type: z.enum(['mean', 'funnel']) })), z.null()]),
+ })
+ const input = { metrics: [{ metric_type: 'bogus' }, { metric_type: 'bogus' }] }
+ const result = schema.safeParse(input, { reportInput: true })
+ expect(result.success).toBe(false)
+
+ const message = formatInputValidationError('some-tool', result.error!, input, schema)
+
+ expect(message).toContain('metrics.0.metric_type')
+ expect(message).not.toContain('expected null')
+ })
+ })
})
})