Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): say backfill tools need an evaluation id (#129083)

Bernat Torres7e642b8cd48f
diff --git a/products/ai_observability/mcp/tools.yaml b/products/ai_observability/mcp/tools.yaml
index 44ebda4ec7dd..d516dae5b5a3 100644
--- a/products/ai_observability/mcp/tools.yaml
+++ b/products/ai_observability/mcp/tools.yaml
@@ -435,6 +435,7 @@ tools:
results are kept. A cancelled backfill cannot be resumed; start a new one over the remaining range instead.
param_overrides:
evaluation_id:
+ description: UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.
aliases:
- evaluationId
id:
@@ -462,6 +463,7 @@ tools:
with llma-evaluation-backfill-get. Results are '$ai_evaluation' events.
param_overrides:
evaluation_id:
+ description: UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.
aliases:
- evaluationId
llma-evaluation-backfill-estimate:
@@ -488,6 +490,7 @@ tools:
filter needs a value, so pass conditions when the evaluation has none or is still a draft.
param_overrides:
evaluation_id:
+ description: UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.
aliases:
- evaluationId
llma-evaluation-backfill-get:
@@ -507,6 +510,7 @@ tools:
window still have no result; null means it was not measured. total_count is the count taken at the start.
param_overrides:
evaluation_id:
+ description: UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.
aliases:
- evaluationId
id:
@@ -525,10 +529,13 @@ tools:
list: true
title: List evaluation backfills
description: >
- List the backfills of one evaluation, newest first, with status, window, and counts. At most one is running
- at a time. See llma-evaluation-backfill-get for what each count means.
+ List the backfills of one evaluation, newest first, with status, window, and counts. evaluation_id is
+ required: there is no project-wide list, so to check every evaluation, get their UUIDs from
+ llma-evaluation-list and call this once per evaluation. At most one is running at a time. See
+ llma-evaluation-backfill-get for what each count means.
param_overrides:
evaluation_id:
+ description: UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.
aliases:
- evaluationId
llma-evaluation-config-get:
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index d7e16497372d..dfd4c40525ce 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -9043,7 +9043,7 @@
}
},
"llma-evaluation-backfill-list": {
- "description": "List the backfills of one evaluation, newest first, with status, window, and counts. At most one is running at a time. See llma-evaluation-backfill-get for what each count means.",
+ "description": "List the backfills of one evaluation, newest first, with status, window, and counts. evaluation_id is required: there is no project-wide list, so to check every evaluation, get their UUIDs from llma-evaluation-list and call this once per evaluation. At most one is running at a time. See llma-evaluation-backfill-get for what each count means.",
"category": "AI observability",
"feature": "llm_analytics",
"summary": "List evaluation backfills",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 089457aad4a1..6f7b3a08312b 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -9309,7 +9309,7 @@
}
},
"llma-evaluation-backfill-list": {
- "description": "List the backfills of one evaluation, newest first, with status, window, and counts. At most one is running at a time. See llma-evaluation-backfill-get for what each count means.",
+ "description": "List the backfills of one evaluation, newest first, with status, window, and counts. evaluation_id is required: there is no project-wide list, so to check every evaluation, get their UUIDs from llma-evaluation-list and call this once per evaluation. At most one is running at a time. See llma-evaluation-backfill-get for what each count means.",
"category": "AI observability",
"feature": "llm_analytics",
"summary": "List evaluation backfills",
diff --git a/services/mcp/src/tools/generated/ai_observability.ts b/services/mcp/src/tools/generated/ai_observability.ts
index 263b47bb4c05..ea3b6f69a3e6 100644
--- a/services/mcp/src/tools/generated/ai_observability.ts
+++ b/services/mcp/src/tools/generated/ai_observability.ts
@@ -729,7 +729,11 @@ const LlmaEvaluationBackfillCancelSchema = () => {
const EvaluationsBackfillsCancelCreateParams = orvalSchemas.EvaluationsBackfillsCancelCreateParams()
return z.preprocess(
normalizeParamAliases({ evaluation_id: ['evaluationId'], id: ['backfillId', 'backfill_id'] }),
- EvaluationsBackfillsCancelCreateParams.omit({ project_id: true })
+ EvaluationsBackfillsCancelCreateParams.omit({ project_id: true }).extend({
+ evaluation_id: EvaluationsBackfillsCancelCreateParams.shape['evaluation_id'].describe(
+ 'UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.'
+ ),
+ })
)
}
@@ -754,7 +758,13 @@ const LlmaEvaluationBackfillCreateSchema = () => {
const EvaluationsBackfillsCreateParams = orvalSchemas.EvaluationsBackfillsCreateParams()
return z.preprocess(
normalizeParamAliases({ evaluation_id: ['evaluationId'] }),
- EvaluationsBackfillsCreateParams.omit({ project_id: true }).extend(EvaluationsBackfillsCreateBody.shape)
+ EvaluationsBackfillsCreateParams.omit({ project_id: true })
+ .extend(EvaluationsBackfillsCreateBody.shape)
+ .extend({
+ evaluation_id: EvaluationsBackfillsCreateParams.shape['evaluation_id'].describe(
+ 'UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.'
+ ),
+ })
)
}
@@ -793,9 +803,13 @@ const LlmaEvaluationBackfillEstimateSchema = () => {
const EvaluationsBackfillsEstimateCreateParams = orvalSchemas.EvaluationsBackfillsEstimateCreateParams()
return z.preprocess(
normalizeParamAliases({ evaluation_id: ['evaluationId'] }),
- EvaluationsBackfillsEstimateCreateParams.omit({ project_id: true }).extend(
- EvaluationsBackfillsEstimateCreateBody.shape
- )
+ EvaluationsBackfillsEstimateCreateParams.omit({ project_id: true })
+ .extend(EvaluationsBackfillsEstimateCreateBody.shape)
+ .extend({
+ evaluation_id: EvaluationsBackfillsEstimateCreateParams.shape['evaluation_id'].describe(
+ 'UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.'
+ ),
+ })
)
}
@@ -833,7 +847,11 @@ const LlmaEvaluationBackfillGetSchema = () => {
const EvaluationsBackfillsRetrieveParams = orvalSchemas.EvaluationsBackfillsRetrieveParams()
return z.preprocess(
normalizeParamAliases({ evaluation_id: ['evaluationId'], id: ['backfillId', 'backfill_id'] }),
- EvaluationsBackfillsRetrieveParams.omit({ project_id: true })
+ EvaluationsBackfillsRetrieveParams.omit({ project_id: true }).extend({
+ evaluation_id: EvaluationsBackfillsRetrieveParams.shape['evaluation_id'].describe(
+ 'UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.'
+ ),
+ })
)
}
@@ -858,7 +876,13 @@ const LlmaEvaluationBackfillListSchema = () => {
const EvaluationsBackfillsListQueryParams = orvalSchemas.EvaluationsBackfillsListQueryParams()
return z.preprocess(
normalizeParamAliases({ evaluation_id: ['evaluationId'] }),
- EvaluationsBackfillsListParams.omit({ project_id: true }).extend(EvaluationsBackfillsListQueryParams.shape)
+ EvaluationsBackfillsListParams.omit({ project_id: true })
+ .extend(EvaluationsBackfillsListQueryParams.shape)
+ .extend({
+ evaluation_id: EvaluationsBackfillsListParams.shape['evaluation_id'].describe(
+ 'UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.'
+ ),
+ })
)
}
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-cancel.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-cancel.json
index c0fbe90309d8..3f16f051383f 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-cancel.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-cancel.json
@@ -2,6 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"evaluation_id": {
+ "description": "UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.",
"type": "string"
},
"id": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-create.json
index 6dc520153129..efd81a708448 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-create.json
@@ -36,6 +36,7 @@
"type": "array"
},
"evaluation_id": {
+ "description": "UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.",
"type": "string"
},
"rerun_existing": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-estimate.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-estimate.json
index 6dc520153129..efd81a708448 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-estimate.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-estimate.json
@@ -36,6 +36,7 @@
"type": "array"
},
"evaluation_id": {
+ "description": "UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.",
"type": "string"
},
"rerun_existing": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-get.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-get.json
index c0fbe90309d8..3f16f051383f 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-get.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-get.json
@@ -2,6 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"evaluation_id": {
+ "description": "UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.",
"type": "string"
},
"id": {
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-list.json
index b87c1dfd44cc..e4c89c768a2b 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-backfill-list.json
@@ -2,6 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"evaluation_id": {
+ "description": "UUID of the evaluation that owns the backfill. Get it from llma-evaluation-list.",
"type": "string"
},
"limit": {