Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mcp): post a weekly mcp tool health report per owning team (#123266)

Richard Solomoubd89935b96fd
diff --git a/.github/scripts/weekly-flaky-report.mjs b/.github/scripts/weekly-flaky-report.mjs
index 2c40ac0042e4..43cfbdae8e74 100644
--- a/.github/scripts/weekly-flaky-report.mjs
+++ b/.github/scripts/weekly-flaky-report.mjs
@@ -16,7 +16,9 @@ import { pathToFileURL } from 'node:url'
import {
AUTH_HEADERS,
+ buildTeamDigests,
cell,
+ deliverTeamDigests,
DRY_RUN,
editWorkflowBlock,
GITHUB_REPOSITORY,
@@ -33,7 +35,7 @@ import {
shortName,
SLACK_BOT_TOKEN,
SLACK_CHANNEL,
- slackPermalink,
+ teamLabel,
} from './weekly-report-common.mjs'
// Off, each team's slice stays in the digest thread, labeled with the channel it would go to.
@@ -471,22 +473,6 @@ function tableRows(items, ownerFor, extrasFor, statusFor = quarantineStatusFor)
})
}
-// The per-team slices carry the same rows as the channel digest. Takes [{owner, slack, row}]
-// and groups by owner.
-function buildTeamDigests(entries) {
- const byOwner = new Map()
- for (const { owner, slack, row } of entries) {
- if (owner === 'unowned' || !slack) {
- continue
- }
- if (!byOwner.has(owner)) {
- byOwner.set(owner, { owner, channel: slack, rows: [] })
- }
- byOwner.get(owner).rows.push(row)
- }
- return [...byOwner.values()].sort((a, b) => b.rows.length - a.rows.length)
-}
-
function flakyTable(rows) {
return {
type: 'table',
@@ -516,8 +502,6 @@ function flakyTable(rows) {
}
}
-const teamLabel = (owner) => owner.replace(/^team-/, '')
-
function titledTable(title, rows) {
return [{ type: 'section', text: { type: 'mrkdwn', text: title } }, flakyTable(rows)]
}
@@ -532,10 +516,7 @@ function buildThreadSliceBlocks({ owner, channel, rows }, reason) {
}
function buildBlocks(now, rows) {
- const blocks = titledTable(
- reportTitle(now, `CI, last ${REPORT_WINDOW_DAYS} days, up to ${TOP_N} per runner`),
- rows
- )
+ const blocks = titledTable(reportTitle(now, `CI, last ${REPORT_WINDOW_DAYS} days, up to ${TOP_N} per runner`), rows)
const editBlock = editWorkflowBlock()
return editBlock ? [...blocks, editBlock] : blocks
}
@@ -551,79 +532,13 @@ function buildTeamBlocks(now, { owner, rows }, digestUrl) {
]
}
-function buildTeamIndexBlocks(posted) {
- return [
- {
- type: 'table',
- column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'left' }],
- rows: [
- [cell('team'), cell('tests'), cell('post')],
- ...posted.map(({ owner, channel, rows, url }) => [
- cell(teamLabel(owner)),
- cell(String(rows.length)),
- url ? linkedCell([{ url, text: channel }]) : cell(channel),
- ]),
- ],
- },
- ]
-}
-
-const SLACK = {
- post: postToSlack,
- permalink: slackPermalink,
- // chat.postMessage allows about one message per second per channel.
- pause: () => new Promise((resolve) => setTimeout(resolve, 1100)),
-}
-
-async function postToTeamChannel(team, { now, digestUrl, slack }) {
- try {
- const post = await slack.post(buildTeamBlocks(now, team, digestUrl), 'Weekly flaky test report', {
- channel: team.channel,
- })
- return { ...team, url: await slack.permalink(post).catch(() => null) }
- } catch (err) {
- console.warn(`team digest for ${team.owner} failed: ${err.message}`)
- return null
- }
-}
-
-// Sends each team its slice, then indexes those posts in the digest thread. A slice that cannot
-// go to its channel stays in the thread, so no team's rows are lost. `withheld` names a reason to
-// keep every slice in the thread.
-async function deliverTeamDigests(teamDigests, { now, digest, withheld = null, slack = SLACK }) {
- const digestUrl = withheld ? null : await slack.permalink(digest).catch(() => null)
- const posted = []
- for (const [index, team] of teamDigests.entries()) {
- if (index > 0) {
- await slack.pause()
- }
- const sent = withheld ? null : await postToTeamChannel(team, { now, digestUrl, slack })
- if (sent) {
- posted.push(sent)
- continue
- }
- // A failed slice must not sink the slices behind it; the digest itself already landed.
- try {
- await slack.post(
- buildThreadSliceBlocks(team, withheld || 'not delivered to'),
- `Flaky tests owned by ${team.owner}`,
- { threadTs: digest.ts }
- )
- } catch (err) {
- console.warn(`thread slice for ${team.owner} failed: ${err.message}`)
- }
- }
- if (posted.length > 0) {
- // The team posts already landed. A failed run would invite a re-dispatch that repeats them.
- try {
- await slack.post(buildTeamIndexBlocks(posted), 'Team posts for the weekly flaky test report', {
- threadTs: digest.ts,
- })
- } catch (err) {
- console.warn(`team post index failed: ${err.message}`)
- }
+function teamReport(now) {
+ return {
+ title: 'Weekly flaky test report',
+ rowLabel: 'tests',
+ teamBlocks: (team, digestUrl) => buildTeamBlocks(now, team, digestUrl),
+ threadSliceBlocks: buildThreadSliceBlocks,
}
- return posted
}
// A report built without Trunk state keeps unproven failures, which a team channel must not receive.
@@ -678,7 +593,7 @@ async function main() {
}
const digest = await postToSlack(blocks, 'Weekly flaky test report')
console.info(`Posted weekly flaky report to ${SLACK_CHANNEL}.`)
- const posted = await deliverTeamDigests(teamDigests, { now, digest, withheld })
+ const posted = await deliverTeamDigests(teamDigests, { digest, report: teamReport(now), withheld })
console.info(`Posted ${posted.length}/${teamDigests.length} team digest(s) to team channels.`)
}
@@ -691,10 +606,9 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
export {
buildBlocks,
- buildTeamDigests,
buildThreadSliceBlocks,
- deliverTeamDigests,
teamPostsWithheld,
+ teamReport,
buildRunnerReports,
CLUSTER_MIN_TESTS,
enrich,
diff --git a/.github/scripts/weekly-flaky-report.test.mjs b/.github/scripts/weekly-flaky-report.test.mjs
index 99950827d42f..339f32ab5acb 100644
--- a/.github/scripts/weekly-flaky-report.test.mjs
+++ b/.github/scripts/weekly-flaky-report.test.mjs
@@ -3,10 +3,9 @@ import { describe, it } from 'node:test'
import {
buildBlocks,
- buildTeamDigests,
buildThreadSliceBlocks,
- deliverTeamDigests,
teamPostsWithheld,
+ teamReport,
buildRunnerReports,
CLUSTER_MIN_TESTS,
enrich,
@@ -21,7 +20,7 @@ import {
sharedTrunkLookup,
tableRows,
} from './weekly-flaky-report.mjs'
-import { repoPathResolver, trackedTestPaths } from './weekly-report-common.mjs'
+import { buildTeamDigests, deliverTeamDigests, repoPathResolver, trackedTestPaths } from './weekly-report-common.mjs'
const onMasterResolver = (path) => [path]
@@ -643,8 +642,8 @@ describe('weekly flaky report', () => {
}
const posted = await deliverTeamDigests(teams, {
- now: new Date('2026-10-05T13:10:00Z'),
digest: { ts: '0', channel: 'C_DIGEST' },
+ report: teamReport(new Date('2026-10-05T13:10:00Z')),
slack,
})
diff --git a/.github/scripts/weekly-mcp-tool-health-report.mjs b/.github/scripts/weekly-mcp-tool-health-report.mjs
new file mode 100644
index 000000000000..699be3b2d4f7
--- /dev/null
+++ b/.github/scripts/weekly-mcp-tool-health-report.mjs
@@ -0,0 +1,315 @@
+// Weekly MCP tool health report, posted to the MCP team's channel on Monday.
+//
+// Sibling of weekly-flaky-report.mjs: one HogQL read of the `$mcp_tool_call` events the MCP
+// server already captures, ranked by failed calls, with owner attribution through
+// packages/owners-yaml. Each tool maps to the YAML file that defines it, so the team that owns
+// the definition sees its own failing tools without the MCP team relaying them.
+//
+// GHA cron ──> HogQL over $mcp_tool_call ──> owners.yaml ──> Slack
+
+import { execFileSync } from 'node:child_process'
+import { readFileSync } from 'node:fs'
+import { pathToFileURL } from 'node:url'
+
+import {
+ API_KEY,
+ buildTeamDigests,
+ cell,
+ deliverTeamDigests,
+ DRY_RUN,
+ editWorkflowBlock,
+ hogql,
+ postToSlack,
+ PROJECT_ID,
+ resolveOwners,
+ SLACK_BOT_TOKEN,
+ SLACK_CHANNEL,
+ teamLabel,
+} from './weekly-report-common.mjs'
+
+const TEAM_CHANNEL_POSTS = process.env.MCP_TOOL_HEALTH_TEAM_CHANNELS === 'true'
+const FEEDBACK_CHANNEL = '#team-context-and-mcp'
+// The name a team uses under `notifications:` in owners.yaml to redirect or silence this report.
+const OWNERS_PRODUCER = 'mcp_tool_health'
+
+const REPORT_WINDOW_DAYS = 7
+const TOP_N = 25
+// A tool qualifies only past all three floors, so a handful of calls from one account never
+// reads as a broken tool.
+const MIN_ERRORS = 50
+const MIN_ERROR_RATE_PCT = 10
+const MIN_USERS = 5
+
+const TOOL_DEFINITION_GLOBS = ['products/*/mcp/*.yaml', 'services/mcp/definitions/*.yaml']
+// Tools with no YAML entry are handwritten in the MCP server. The registry maps each one to its
+// source file, whose path services/mcp/src/owners.yaml can assign more narrowly than the directory.
+const HANDWRITTEN_TOOL_PATH = 'services/mcp/src/tools'
+const HANDWRITTEN_TOOL_REGISTRY = `${HANDWRITTEN_TOOL_PATH}/index.ts`
+
+// Exec-mode calls name the real tool in `$mcp_exec_tool_call_name`. The bare `exec` dispatcher is
+// excluded because its own rejections belong to the MCP server, not to any product tool. Error
+// messages carry project ids and UUIDs in their API paths, so masking them groups one failure as
+// one message.
+const TOOL_HEALTH_QUERY = `
+ SELECT tool, calls, errors, users, top_error
+ FROM (
+ SELECT
+ coalesce(nullIf(toString(properties.$mcp_exec_tool_call_name), ''), toString(properties.$mcp_tool_name)) AS tool,
+ count() AS calls,
+ countIf(toBool(properties.$mcp_is_error)) AS errors,
+ uniqIf(person_id, toBool(properties.$mcp_is_error)) AS users,
+ arrayElement(topK(1)(if(toBool(properties.$mcp_is_error), replaceRegexpAll(substring(toString(properties.$mcp_error_message), 1, 160), '/[0-9a-f]{8}-[0-9a-f-]+|/[0-9]+', '/#'), NULL)), 1) AS top_error
+ FROM events
+ WHERE event = '$mcp_tool_call'
+ AND timestamp >= now() - INTERVAL ${REPORT_WINDOW_DAYS} DAY
+ GROUP BY tool
+ )
+ WHERE tool NOT IN ('', 'exec')
+ AND errors >= {min_errors}
+ AND errors * 100 >= calls * {min_error_rate_pct}
+ AND users >= {min_users}
+ ORDER BY errors DESC
+ LIMIT ${TOP_N}`
+
+const TOOL_HEALTH_VALUES = {
+ min_errors: MIN_ERRORS,
+ min_error_rate_pct: MIN_ERROR_RATE_PCT,
+ min_users: MIN_USERS,
+}
+
+async function fetchFailingTools(runHogql = hogql) {
+ const result = await runHogql(TOOL_HEALTH_QUERY, TOOL_HEALTH_VALUES)
+ return (result.results || []).map(([tool, calls, errors, users, topError]) => ({
+ tool,
+ calls: Number(calls),
+ errors: Number(errors),
+ users: Number(users),
+ topError: topError || '',
+ }))
+}
+
+// Tool names are the keys directly under `tools:` and `wrappers:` in each definition file. A line
+// scan keeps the script free of a YAML dependency, and the generator enforces this layout. Codegen
+// registers a `confirmed_action` tool only as its `-prepare` and `-execute` pair, so analytics never
+// sees the base name.
+function parseToolNames(yamlText) {
+ const names = []
+ let section = null
+ for (const line of yamlText.split('\n')) {
+ const topLevel = /^([A-Za-z_]+):/.exec(line)
+ if (topLevel) {
+ section = topLevel[1]
+ continue
+ }
+ const toolKey = /^ {4}([a-z0-9][a-z0-9-]*):\s*$/.exec(line)
+ if (toolKey && (section === 'tools' || section === 'wrappers')) {
+ names.push(toolKey[1])
+ }
+ if (/^ {8}confirmed_action:/.test(line) && section === 'tools' && names.length > 0) {
+ const base = names.pop()
+ names.push(`${base}-prepare`, `${base}-execute`)
+ }
+ }
+ return names
+}
+
+// Reads `'tool-name': factory,` entries of the registry and the `./path` each factory is imported
+// from. Entries it cannot follow (a computed key, a spread, an inline factory) fall back to the
+// directory.
+function parseHandwrittenTools(registrySource) {
+ const modules = new Map()
+ for (const [, defaultImport, namedImports, path] of registrySource.matchAll(
+ /^import (?:(\w+)|\{([^}]*)\}) from '\.\/([^']+)'/gm
+ )) {
+ for (const name of defaultImport ? [defaultImport] : namedImports.split(',')) {
+ modules.set(name.trim(), `${HANDWRITTEN_TOOL_PATH}/${path}`)
+ }
+ }
+ const tools = []
+ for (const [, tool, factory] of registrySource.matchAll(/^ {4}'([a-z0-9-]+)': (\w+),$/gm)) {
+ if (modules.has(factory)) {
+ tools.push([tool, modules.get(factory)])
+ }
+ }
+ return tools
+}
+
+function toolDefinitionIndex({
+ listFiles = trackedDefinitionFiles,
+ readFile = (path) => readFileSync(path, 'utf8'),
+} = {}) {
+ const index = new Map()
+ for (const path of listFiles()) {
+ for (const name of parseToolNames(readFile(path))) {
+ if (!index.has(name)) {
+ index.set(name, path)
+ }
+ }
+ }
+ for (const [name, path] of parseHandwrittenTools(readFile(HANDWRITTEN_TOOL_REGISTRY))) {
+ if (!index.has(name)) {
+ index.set(name, path)
+ }
+ }
+ return index
+}
+
+function trackedDefinitionFiles() {
+ return execFileSync('git', ['ls-files', ...TOOL_DEFINITION_GLOBS], { encoding: 'utf8' })
+ .split('\n')
+ .filter(Boolean)
+}
+
+function definitionPathFor(tool, index) {
+ return index.get(tool) || HANDWRITTEN_TOOL_PATH
+}
+
+function formatRate(errors, calls) {
+ return `${Math.round((errors * 100) / calls)}%`
+}
+
+function tableRows(items, ownerFor) {
+ return items.map((item) => [
+ cell(item.tool),
+ cell(teamLabel(ownerFor(item).owner)),
+ cell(String(item.errors)),
+ cell(formatRate(item.errors, item.calls)),
+ cell(String(item.users)),
+ cell(item.topError || 'no message captured'),
+ ])
+}
+
+function toolTable(rows) {
+ return {
+ type: 'table',
+ column_settings: [
+ { align: 'left' },
+ { align: 'left' },
+ { align: 'right' },
+ { align: 'right' },
+ { align: 'right' },
+ { align: 'left' },
+ ],
+ rows: [
+ [
+ cell('tool'),
+ cell('owner'),
+ cell('failed calls'),
+ cell('error rate'),
+ cell('users hit'),
+ cell('top error'),
+ ],
+ ...rows,
+ ],
+ }
+}
+
+function reportTitle(now, scope) {
+ return `*Weekly MCP tool health - ${now.toISOString().slice(0, 10)}* _(${scope})_`
+}
+
+const FLOOR_NOTE = `A tool is listed when it failed at least ${MIN_ERRORS} calls, on at least ${MIN_ERROR_RATE_PCT}% of its calls, for at least ${MIN_USERS} users, in the last ${REPORT_WINDOW_DAYS} days.`
+
+function buildBlocks(now, rows) {
+ const blocks = [
+ {
+ type: 'section',
+ text: { type: 'mrkdwn', text: reportTitle(now, `all teams, last ${REPORT_WINDOW_DAYS} days`) },
+ },
+ toolTable(rows),
+ { type: 'context', elements: [{ type: 'mrkdwn', text: FLOOR_NOTE }] },
+ ]
+ const editBlock = editWorkflowBlock()
+ return editBlock ? [...blocks, editBlock] : blocks
+}
+
+function buildTeamBlocks(now, { owner, rows }, digestUrl) {
+ const footer = [
+ ...(digestUrl ? [`<${digestUrl}|Report for all teams>`] : []),
+ `${FLOOR_NOTE} Wrong owner or a tool that should be off? Tell ${FEEDBACK_CHANNEL}.`,
+ ]
+ return [
+ { type: 'section', text: { type: 'mrkdwn', text: reportTitle(now, `owned by ${teamLabel(owner)}`) } },
+ toolTable(rows),
+ { type: 'context', elements: [{ type: 'mrkdwn', text: footer.join(' · ') }] },
+ ]
+}
+
+// A slice kept in the digest thread. `reason` says why it is not in the team's channel.
+function buildThreadSliceBlocks({ owner, channel, rows }, reason) {
+ return [
+ { type: 'section', text: { type: 'mrkdwn', text: `*${teamLabel(owner)}* _(${reason} ${channel})_` } },
+ toolTable(rows),
+ ]
+}
+
+function teamReport(now) {
+ return {
+ title: 'Weekly MCP tool health report',
+ rowLabel: 'tools',
+ teamBlocks: (team, digestUrl) => buildTeamBlocks(now, team, digestUrl),
+ threadSliceBlocks: buildThreadSliceBlocks,
+ }
+}
+
+// Off, each slice goes to the digest thread, labeled with the channel it would reach, so the
+// routing can be checked before any team channel receives a post.
+function teamPostsWithheld(enabled = TEAM_CHANNEL_POSTS) {
+ return enabled ? null : 'shadow: would post to'
+}
+
+async function main() {
+ if (!PROJECT_ID || !API_KEY) {
+ console.warn('POSTHOG_PROJECT_ID / POSTHOG_API_KEY not set - skipping report. Wire them to enable.')
+ return
+ }
+ const now = new Date()
+ const failing = await fetchFailingTools()
+ if (failing.length === 0) {
+ console.info('No MCP tool above the failure floor this week - nothing to post.')
+ return
+ }
+ const index = toolDefinitionIndex()
+ const items = failing.map((item) => ({ ...item, selector: definitionPathFor(item.tool, index) }))
+ const ownerFor = resolveOwners(items, (path) => [path], OWNERS_PRODUCER)
+ const rows = tableRows(items, ownerFor)
+ const entries = items.map((item, i) => ({ ...ownerFor(item), row: rows[i] }))
+ const blocks = buildBlocks(now, rows)
+ const teamDigests = buildTeamDigests(entries)
+ if (DRY_RUN) {
+ // Actions logs on this public repo are readable by anyone, so a dry run prints only the routing.
+ // Shadow mode previews the full report, with its counts and errors, in the Slack digest thread.
+ for (const item of items) {
+ const { owner, slack } = ownerFor(item)
+ console.info(`${item.tool} -> ${owner} (${slack ?? 'digest only'})`)
+ }
+ return
+ }
+ if (!SLACK_BOT_TOKEN) {
+ throw new Error('SLACK_BOT_TOKEN not set on a non-dry run - refusing to silently skip.')
+ }
+ const digest = await postToSlack(blocks, 'Weekly MCP tool health report')
+ console.info(`Posted weekly MCP tool health report to ${SLACK_CHANNEL}.`)
+ const posted = await deliverTeamDigests(teamDigests, {
+ digest,
+ report: teamReport(now),
+ withheld: teamPostsWithheld(),
+ })
+ console.info(`Posted ${posted.length}/${teamDigests.length} team slice(s) to team channels.`)
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ main().catch((err) => {
+ console.error(err)
+ process.exit(1)
+ })
+}
+
+export {
+ definitionPathFor,
+ fetchFailingTools,
+ HANDWRITTEN_TOOL_PATH,
+ teamPostsWithheld,
+ teamReport,
+ toolDefinitionIndex,
+}
diff --git a/.github/scripts/weekly-mcp-tool-health-report.test.mjs b/.github/scripts/weekly-mcp-tool-health-report.test.mjs
new file mode 100644
index 000000000000..ae0e373ced60
--- /dev/null
+++ b/.github/scripts/weekly-mcp-tool-health-report.test.mjs
@@ -0,0 +1,143 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+
+import {
+ definitionPathFor,
+ fetchFailingTools,
+ HANDWRITTEN_TOOL_PATH,
+ teamPostsWithheld,
+ teamReport,
+ toolDefinitionIndex,
+} from './weekly-mcp-tool-health-report.mjs'
+import { deliverTeamDigests, resolveOwners } from './weekly-report-common.mjs'
+
+const DEFINITION = `category: Surveys
+ui_apps:
+ survey-list:
+ type: list
+tools:
+ survey-create:
+ operation: surveys_create
+ param_overrides:
+ name:
+ aliases:
+ survey-get:
+ operation: surveys_retrieve
+ survey-delete:
+ operation: surveys_destroy
+ confirmed_action:
+ action_label: Delete survey
+wrappers:
+ query-survey:
+ schema_ref: SurveyQuery
+`
+
+const REGISTRY = `import getLLMCosts from './aiObservability/getLLMCosts'
+import {
+ executeSql,
+ readDataSchema,
+} from './posthogAiTools'
+
+export const TOOL_MAP = {
+ 'get-llm-total-costs-for-project': getLLMCosts,
+ 'read-data-schema': readDataSchema,
+ [EXECUTE_SQL_TOOL_NAME]: executeSql,
+ 'posthog-connection-call': () => createConnectionCallTool(resolveToolBase),
+}
+`
+
+const TEAM = { owner: 'team-surveys', channel: '#team-surveys', rows: [['row']] }
+
+// A slice's heading, a team post's digest link, or the index's header row.
+function lastNote(blocks) {
+ const notes = blocks
+ .filter((block) => block.type !== 'table')
+ .map((block) => (block.text ?? block.elements[0]).text)
+ return notes.length > 0 ? notes.at(-1).split(' · ')[0] : blocks[0].rows[0].map((c) => c.text).join(' | ')
+}
+
+function recordingSlack() {
+ const posts = []
+ return {
+ posts,
+ slack: {
+ post: async (blocks, text, options) => {
+ posts.push({ ...options, note: lastNote(blocks) })
+ return { ts: '2.0', channel: options.channel || 'C_DIGEST' }
+ },
+ permalink: async ({ channel, ts }) => `https://slack.test/${channel}/${ts}`,
+ pause: async () => {},
+ },
+ }
+}
+
+describe('weekly MCP tool health report', () => {
+ it('maps tools, wrappers and handwritten tools to their source file and nothing else', () => {
+ const index = toolDefinitionIndex({
+ listFiles: () => ['products/surveys/mcp/tools.yaml'],
+ readFile: (path) => (path.endsWith('index.ts') ? REGISTRY : DEFINITION),
+ })
+ assert.deepEqual(
+ [...index.entries()],
+ [
+ ['survey-create', 'products/surveys/mcp/tools.yaml'],
+ ['survey-get', 'products/surveys/mcp/tools.yaml'],
+ ['survey-delete-prepare', 'products/surveys/mcp/tools.yaml'],
+ ['survey-delete-execute', 'products/surveys/mcp/tools.yaml'],
+ ['query-survey', 'products/surveys/mcp/tools.yaml'],
+ ['get-llm-total-costs-for-project', 'services/mcp/src/tools/aiObservability/getLLMCosts'],
+ ['read-data-schema', 'services/mcp/src/tools/posthogAiTools'],
+ ]
+ )
+ assert.equal(definitionPathFor('execute-sql', index), HANDWRITTEN_TOOL_PATH)
+ })
+
+ it('resolves owners for the report producer through owners.yaml', () => {
+ const items = [{ selector: 'products/cdp/mcp/cdp_functions.yaml' }, { selector: HANDWRITTEN_TOOL_PATH }]
+ const ownerFor = resolveOwners(items, (path) => [path], 'mcp_tool_health')
+ for (const item of items) {
+ assert.notEqual(ownerFor(item).owner, 'unowned')
+ }
+ })
+
+ it('reads query rows into tool records', async () => {
+ const tools = await fetchFailingTools(async () => ({
+ results: [['survey-get', '120', '30', '7', null]],
+ }))
+ assert.deepEqual(tools, [{ tool: 'survey-get', calls: 120, errors: 30, users: 7, topError: '' }])
+ })
+
+ for (const { label, enabled, digestChannel = '#team-context-and-mcp', expected } of [
+ {
+ label: 'keeps every slice in the thread while team posts are off',
+ enabled: false,
+ expected: [{ threadTs: '1.0', note: '*surveys* _(shadow: would post to #team-surveys)_' }],
+ },
+ {
+ label: 'posts a slice to its team channel and indexes it when team posts are on',
+ enabled: true,
+ expected: [
+ { channel: '#team-surveys', note: '<https://slack.test/C_DIGEST/1.0|Report for all teams>' },
+ { threadTs: '1.0', note: 'team | tools | post' },
+ ],
+ },
+ {
+ label: 'keeps the slice of the digest channel team in the thread',
+ enabled: true,
+ digestChannel: '#team-surveys',
+ expected: [{ threadTs: '1.0', note: '*surveys* _(report already in #team-surveys)_' }],
+ },
+ ]) {
+ it(label, async () => {
+ const { posts, slack } = recordingSlack()
+ await deliverTeamDigests([TEAM], {
+ digest: { ts: '1.0', channel: 'C_DIGEST' },
+ report: teamReport(new Date('2026-10-26T00:00:00Z')),
+ withheld: teamPostsWithheld(enabled),
+ digestChannel,
+ slack,
+ })
+ assert.deepEqual(posts, expected)
+ })
+ }
+})
diff --git a/.github/scripts/weekly-report-common.mjs b/.github/scripts/weekly-report-common.mjs
index ff2e45113bee..b5f06a793f9e 100644
--- a/.github/scripts/weekly-report-common.mjs
+++ b/.github/scripts/weekly-report-common.mjs
@@ -1,7 +1,8 @@
-// Shared machinery for the weekly CI report scripts (weekly-flaky-report.mjs,
-// weekly-slow-tests-report.mjs): PostHog API plumbing, HogQL, repo path and owner
-// resolution, Slack table cells, and the Slack post. Everything here reads the same
-// environment both workflows set, so each report owns only its signal and its table.
+// Shared machinery for the weekly report scripts (weekly-flaky-report.mjs,
+// weekly-slow-tests-report.mjs, weekly-mcp-tool-health-report.mjs): PostHog API plumbing,
+// HogQL, repo path and owner resolution, Slack table cells, the Slack post, and the per-team
+// delivery. Everything here reads the same environment every workflow sets, so each report owns
+// only its signal and its table.
import { execFileSync } from 'node:child_process'
@@ -201,3 +202,101 @@ export async function slackPermalink({ channel, ts }) {
}
return data.permalink
}
+
+export const teamLabel = (owner) => owner.replace(/^team-/, '')
+
+// The per-team slices carry the same rows as the channel digest. Takes [{owner, slack, row}]
+// and groups by owner.
+export function buildTeamDigests(entries) {
+ const byOwner = new Map()
+ for (const { owner, slack, row } of entries) {
+ if (owner === 'unowned' || !slack) {
+ continue
+ }
+ if (!byOwner.has(owner)) {
+ byOwner.set(owner, { owner, channel: slack, rows: [] })
+ }
+ byOwner.get(owner).rows.push(row)
+ }
+ return [...byOwner.values()].sort((a, b) => b.rows.length - a.rows.length)
+}
+
+function buildTeamIndexBlocks(posted, rowLabel) {
+ return [
+ {
+ type: 'table',
+ column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'left' }],
+ rows: [
+ [cell('team'), cell(rowLabel), cell('post')],
+ ...posted.map(({ owner, channel, rows, url }) => [
+ cell(teamLabel(owner)),
+ cell(String(rows.length)),
+ url ? linkedCell([{ url, text: channel }]) : cell(channel),
+ ]),
+ ],
+ },
+ ]
+}
+
+const SLACK = {
+ post: postToSlack,
+ permalink: slackPermalink,
+ // chat.postMessage allows about one message per second per channel.
+ pause: () => new Promise((resolve) => setTimeout(resolve, 1100)),
+}
+
+async function postToTeamChannel(team, { report, digestUrl, slack }) {
+ try {
+ const post = await slack.post(report.teamBlocks(team, digestUrl), report.title, { channel: team.channel })
+ return { ...team, url: await slack.permalink(post).catch(() => null) }
+ } catch (err) {
+ console.warn(`team digest for ${team.owner} failed: ${err.message}`)
+ return null
+ }
+}
+
+// Sends each team its slice, then indexes those posts in the digest thread. A slice that cannot
+// go to its channel stays in the thread, so no team's rows are lost. `withheld` names a reason to
+// keep every slice in the thread. A team whose channel is `digestChannel` keeps its slice in the
+// thread too, so that channel does not get the digest twice. `report` supplies the report's
+// `title`, the `rowLabel` for the index, and its `teamBlocks(team, digestUrl)` and
+// `threadSliceBlocks(team, reason)` builders.
+export async function deliverTeamDigests(
+ teamDigests,
+ { digest, report, withheld = null, digestChannel = SLACK_CHANNEL, slack = SLACK }
+) {
+ const digestUrl = withheld ? null : await slack.permalink(digest).catch(() => null)
+ const posted = []
+ for (const [index, team] of teamDigests.entries()) {
+ if (index > 0) {
+ await slack.pause()
+ }
+ const kept = withheld || (team.channel === digestChannel ? 'report already in' : null)
+ const sent = kept ? null : await postToTeamChannel(team, { report, digestUrl, slack })
+ if (sent) {
+ posted.push(sent)
+ continue
+ }
+ // A failed slice must not sink the slices behind it; the digest itself already landed.
+ try {
+ await slack.post(
+ report.threadSliceBlocks(team, kept || 'not delivered to'),
+ `${report.title}: ${teamLabel(team.owner)}`,
+ { threadTs: digest.ts }
+ )
+ } catch (err) {
+ console.warn(`thread slice for ${team.owner} failed: ${err.message}`)
+ }
+ }
+ if (posted.length > 0) {
+ // The team posts already landed. A failed run would invite a re-dispatch that repeats them.
+ try {
+ await slack.post(buildTeamIndexBlocks(posted, report.rowLabel), `${report.title}: team posts`, {
+ threadTs: digest.ts,
+ })
+ } catch (err) {
+ console.warn(`team post index failed: ${err.message}`)
+ }
+ }
+ return posted
+}
diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml
index 8a120f1c85c5..02629e6ba339 100644
--- a/.github/workflows/ci-scripts.yml
+++ b/.github/workflows/ci-scripts.yml
@@ -74,7 +74,8 @@ jobs:
/.nvmrc
/.oxlintrc.json
/.oxfmtrc.json
- /owners.yaml
+ **/owners.yaml
+ **/product.yaml
/pnpm-workspace.yaml
/uv.lock
/tach.toml
diff --git a/.github/workflows/weekly-mcp-tool-health-report.yml b/.github/workflows/weekly-mcp-tool-health-report.yml
new file mode 100644
index 000000000000..f7ab74dc21a4
--- /dev/null
+++ b/.github/workflows/weekly-mcp-tool-health-report.yml
@@ -0,0 +1,73 @@
+name: Weekly MCP tool health report
+
+# Posts the week's failing MCP tools to #team-context-and-mcp every Monday morning. Sibling of
+# weekly-flaky-report.yml: the script reads the `$mcp_tool_call` events the MCP server captures
+# and maps each tool to the team that owns its definition through the packages/owners-yaml
+# resolver, which is why the sparse checkout includes the ownership and tool definition files.
+# Each owning team's rows go to that team's channel when vars.MCP_TOOL_HEALTH_TEAM_CHANNELS is
+# 'true', and stay in the digest thread otherwise.
+#
+# Requires (no-ops with a warning until set, so it never spams failures):
+# - vars.ENG_ANALYTICS_PROJECT_ID: the PostHog project that records MCP tool calls.
+# - secrets.ENG_ANALYTICS_POSTHOG_API_KEY: a personal API key with query:read for that project.
+# - secrets.SLACK_DEVEX_BOT_TOKEN: the existing DevEx Slack bot.
+
+on:
+ schedule:
+ - cron: '50 13 * * 1' # Mondays 13:50 UTC, staggered after the slow tests report
+ workflow_dispatch:
+ inputs:
+ dry_run:
+ description: 'Print each failing tool with its owner and channel instead of posting to Slack'
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+jobs:
+ report:
+ runs-on: ubuntu-24.04
+ # Worst case is 3 query attempts x 150s abort plus 2 x 30s backoffs (~8.5min), plus checkout
+ # and owner resolution.
+ timeout-minutes: 12
+ # Never run from forks (no secret access anyway), only the canonical repo.
+ # Manual dispatch is master-only so branch-controlled code can't run with the secrets.
+ if: >-
+ github.repository == 'PostHog/posthog' &&
+ (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/master')
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ sparse-checkout: |
+ .github/scripts/weekly-mcp-tool-health-report.mjs
+ .github/scripts/weekly-report-common.mjs
+ .nvmrc
+ packages/owners-yaml
+ owners.yaml
+ **/owners.yaml
+ **/product.yaml
+ products/*/mcp/*.yaml
+ services/mcp/definitions/*.yaml
+ services/mcp/src/tools/index.ts
+ sparse-checkout-cone-mode: false
+
+ - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+ with:
+ node-version-file: .nvmrc
+
+ - name: Install owners resolver dependency
+ run: python3 -m pip install --quiet pyyaml
+
+ - name: Build and post weekly MCP tool health report
+ # POSTHOG_HOST rides the script's default.
+ env:
+ POSTHOG_PROJECT_ID: ${{ vars.ENG_ANALYTICS_PROJECT_ID }}
+ POSTHOG_API_KEY: ${{ secrets.ENG_ANALYTICS_POSTHOG_API_KEY }}
+ SLACK_CHANNEL: '#team-context-and-mcp'
+ # 'true' posts each team's rows to that team's channel. Any other value keeps the
+ # rows in the digest thread, labeled with the channel they would reach.
+ MCP_TOOL_HEALTH_TEAM_CHANNELS: ${{ vars.MCP_TOOL_HEALTH_TEAM_CHANNELS }}
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_DEVEX_BOT_TOKEN }}
+ DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+ run: node .github/scripts/weekly-mcp-tool-health-report.mjs
diff --git a/owners.yaml b/owners.yaml
index 543aabec3803..82a07ccfcc83 100644
--- a/owners.yaml
+++ b/owners.yaml
@@ -3,7 +3,7 @@ owners: []
# Team slugs below are GitHub teams in this organization.
github_org: PostHog
# Automation that a team can redirect or silence under `notifications:`.
-producers: [stamphog, visual_review, flaky_report]
+producers: [stamphog, visual_review, flaky_report, mcp_tool_health]
# services/mcp generate-tools reads every YAML file in a product's mcp/ directory.
reserved_dirs: ['products/**/mcp/**']
# A product manifest already lists its owners, so it counts as an ownership file.