:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(mcp): post a weekly mcp tool health report per owning team (#123266)
Richard Solomoubd89935b96fd
diff --git a/.github/scripts/weekly-flaky-report.mjs b/.github/scripts/weekly-flaky-report.mjsindex 2c40ac0042e4..43cfbdae8e74 100644--- a/.github/scripts/weekly-flaky-report.mjs+++ b/.github/scripts/weekly-flaky-report.mjs@@ -16,7 +16,9 @@ import { pathToFileURL } from 'node:url'import {AUTH_HEADERS,+ buildTeamDigests,cell,+ deliverTeamDigests,DRY_RUN,editWorkflowBlock,GITHUB_REPOSITORY,@@ -33,7 +35,7 @@ import {shortName,SLACK_BOT_TOKEN,SLACK_CHANNEL,- slackPermalink,+ teamLabel,} from './weekly-report-common.mjs'// Off, each team's slice stays in the digest thread, labeled with the channel it would go to.@@ -471,22 +473,6 @@ function tableRows(items, ownerFor, extrasFor, statusFor = quarantineStatusFor)})}-// The per-team slices carry the same rows as the channel digest. Takes [{owner, slack, row}]-// and groups by owner.-function buildTeamDigests(entries) {- const byOwner = new Map()- for (const { owner, slack, row } of entries) {- if (owner === 'unowned' || !slack) {- continue- }- if (!byOwner.has(owner)) {- byOwner.set(owner, { owner, channel: slack, rows: [] })- }- byOwner.get(owner).rows.push(row)- }- return [...byOwner.values()].sort((a, b) => b.rows.length - a.rows.length)-}-function flakyTable(rows) {return {type: 'table',@@ -516,8 +502,6 @@ function flakyTable(rows) {}}-const teamLabel = (owner) => owner.replace(/^team-/, '')-function titledTable(title, rows) {return [{ type: 'section', text: { type: 'mrkdwn', text: title } }, flakyTable(rows)]}@@ -532,10 +516,7 @@ function buildThreadSliceBlocks({ owner, channel, rows }, reason) {}function buildBlocks(now, rows) {- const blocks = titledTable(- reportTitle(now, `CI, last ${REPORT_WINDOW_DAYS} days, up to ${TOP_N} per runner`),- rows- )+ const blocks = titledTable(reportTitle(now, `CI, last ${REPORT_WINDOW_DAYS} days, up to ${TOP_N} per runner`), rows)const editBlock = editWorkflowBlock()return editBlock ? [...blocks, editBlock] : blocks}@@ -551,79 +532,13 @@ function buildTeamBlocks(now, { owner, rows }, digestUrl) {]}-function buildTeamIndexBlocks(posted) {- return [- {- type: 'table',- column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'left' }],- rows: [- [cell('team'), cell('tests'), cell('post')],- ...posted.map(({ owner, channel, rows, url }) => [- cell(teamLabel(owner)),- cell(String(rows.length)),- url ? linkedCell([{ url, text: channel }]) : cell(channel),- ]),- ],- },- ]-}--const SLACK = {- post: postToSlack,- permalink: slackPermalink,- // chat.postMessage allows about one message per second per channel.- pause: () => new Promise((resolve) => setTimeout(resolve, 1100)),-}--async function postToTeamChannel(team, { now, digestUrl, slack }) {- try {- const post = await slack.post(buildTeamBlocks(now, team, digestUrl), 'Weekly flaky test report', {- channel: team.channel,- })- return { ...team, url: await slack.permalink(post).catch(() => null) }- } catch (err) {- console.warn(`team digest for ${team.owner} failed: ${err.message}`)- return null- }-}--// Sends each team its slice, then indexes those posts in the digest thread. A slice that cannot-// go to its channel stays in the thread, so no team's rows are lost. `withheld` names a reason to-// keep every slice in the thread.-async function deliverTeamDigests(teamDigests, { now, digest, withheld = null, slack = SLACK }) {- const digestUrl = withheld ? null : await slack.permalink(digest).catch(() => null)- const posted = []- for (const [index, team] of teamDigests.entries()) {- if (index > 0) {- await slack.pause()- }- const sent = withheld ? null : await postToTeamChannel(team, { now, digestUrl, slack })- if (sent) {- posted.push(sent)- continue- }- // A failed slice must not sink the slices behind it; the digest itself already landed.- try {- await slack.post(- buildThreadSliceBlocks(team, withheld || 'not delivered to'),- `Flaky tests owned by ${team.owner}`,- { threadTs: digest.ts }- )- } catch (err) {- console.warn(`thread slice for ${team.owner} failed: ${err.message}`)- }- }- if (posted.length > 0) {- // The team posts already landed. A failed run would invite a re-dispatch that repeats them.- try {- await slack.post(buildTeamIndexBlocks(posted), 'Team posts for the weekly flaky test report', {- threadTs: digest.ts,- })- } catch (err) {- console.warn(`team post index failed: ${err.message}`)- }+function teamReport(now) {+ return {+ title: 'Weekly flaky test report',+ rowLabel: 'tests',+ teamBlocks: (team, digestUrl) => buildTeamBlocks(now, team, digestUrl),+ threadSliceBlocks: buildThreadSliceBlocks,}- return posted}// A report built without Trunk state keeps unproven failures, which a team channel must not receive.@@ -678,7 +593,7 @@ async function main() {}const digest = await postToSlack(blocks, 'Weekly flaky test report')console.info(`Posted weekly flaky report to ${SLACK_CHANNEL}.`)- const posted = await deliverTeamDigests(teamDigests, { now, digest, withheld })+ const posted = await deliverTeamDigests(teamDigests, { digest, report: teamReport(now), withheld })console.info(`Posted ${posted.length}/${teamDigests.length} team digest(s) to team channels.`)}@@ -691,10 +606,9 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)export {buildBlocks,- buildTeamDigests,buildThreadSliceBlocks,- deliverTeamDigests,teamPostsWithheld,+ teamReport,buildRunnerReports,CLUSTER_MIN_TESTS,enrich,diff --git a/.github/scripts/weekly-flaky-report.test.mjs b/.github/scripts/weekly-flaky-report.test.mjsindex 99950827d42f..339f32ab5acb 100644--- a/.github/scripts/weekly-flaky-report.test.mjs+++ b/.github/scripts/weekly-flaky-report.test.mjs@@ -3,10 +3,9 @@ import { describe, it } from 'node:test'import {buildBlocks,- buildTeamDigests,buildThreadSliceBlocks,- deliverTeamDigests,teamPostsWithheld,+ teamReport,buildRunnerReports,CLUSTER_MIN_TESTS,enrich,@@ -21,7 +20,7 @@ import {sharedTrunkLookup,tableRows,} from './weekly-flaky-report.mjs'-import { repoPathResolver, trackedTestPaths } from './weekly-report-common.mjs'+import { buildTeamDigests, deliverTeamDigests, repoPathResolver, trackedTestPaths } from './weekly-report-common.mjs'const onMasterResolver = (path) => [path]@@ -643,8 +642,8 @@ describe('weekly flaky report', () => {}const posted = await deliverTeamDigests(teams, {- now: new Date('2026-10-05T13:10:00Z'),digest: { ts: '0', channel: 'C_DIGEST' },+ report: teamReport(new Date('2026-10-05T13:10:00Z')),slack,})diff --git a/.github/scripts/weekly-mcp-tool-health-report.mjs b/.github/scripts/weekly-mcp-tool-health-report.mjsnew file mode 100644index 000000000000..699be3b2d4f7--- /dev/null+++ b/.github/scripts/weekly-mcp-tool-health-report.mjs@@ -0,0 +1,315 @@+// Weekly MCP tool health report, posted to the MCP team's channel on Monday.+//+// Sibling of weekly-flaky-report.mjs: one HogQL read of the `$mcp_tool_call` events the MCP+// server already captures, ranked by failed calls, with owner attribution through+// packages/owners-yaml. Each tool maps to the YAML file that defines it, so the team that owns+// the definition sees its own failing tools without the MCP team relaying them.+//+// GHA cron ──> HogQL over $mcp_tool_call ──> owners.yaml ──> Slack++import { execFileSync } from 'node:child_process'+import { readFileSync } from 'node:fs'+import { pathToFileURL } from 'node:url'++import {+ API_KEY,+ buildTeamDigests,+ cell,+ deliverTeamDigests,+ DRY_RUN,+ editWorkflowBlock,+ hogql,+ postToSlack,+ PROJECT_ID,+ resolveOwners,+ SLACK_BOT_TOKEN,+ SLACK_CHANNEL,+ teamLabel,+} from './weekly-report-common.mjs'++const TEAM_CHANNEL_POSTS = process.env.MCP_TOOL_HEALTH_TEAM_CHANNELS === 'true'+const FEEDBACK_CHANNEL = '#team-context-and-mcp'+// The name a team uses under `notifications:` in owners.yaml to redirect or silence this report.+const OWNERS_PRODUCER = 'mcp_tool_health'++const REPORT_WINDOW_DAYS = 7+const TOP_N = 25+// A tool qualifies only past all three floors, so a handful of calls from one account never+// reads as a broken tool.+const MIN_ERRORS = 50+const MIN_ERROR_RATE_PCT = 10+const MIN_USERS = 5++const TOOL_DEFINITION_GLOBS = ['products/*/mcp/*.yaml', 'services/mcp/definitions/*.yaml']+// Tools with no YAML entry are handwritten in the MCP server. The registry maps each one to its+// source file, whose path services/mcp/src/owners.yaml can assign more narrowly than the directory.+const HANDWRITTEN_TOOL_PATH = 'services/mcp/src/tools'+const HANDWRITTEN_TOOL_REGISTRY = `${HANDWRITTEN_TOOL_PATH}/index.ts`++// Exec-mode calls name the real tool in `$mcp_exec_tool_call_name`. The bare `exec` dispatcher is+// excluded because its own rejections belong to the MCP server, not to any product tool. Error+// messages carry project ids and UUIDs in their API paths, so masking them groups one failure as+// one message.+const TOOL_HEALTH_QUERY = `+ SELECT tool, calls, errors, users, top_error+ FROM (+ SELECT+ coalesce(nullIf(toString(properties.$mcp_exec_tool_call_name), ''), toString(properties.$mcp_tool_name)) AS tool,+ count() AS calls,+ countIf(toBool(properties.$mcp_is_error)) AS errors,+ uniqIf(person_id, toBool(properties.$mcp_is_error)) AS users,+ arrayElement(topK(1)(if(toBool(properties.$mcp_is_error), replaceRegexpAll(substring(toString(properties.$mcp_error_message), 1, 160), '/[0-9a-f]{8}-[0-9a-f-]+|/[0-9]+', '/#'), NULL)), 1) AS top_error+ FROM events+ WHERE event = '$mcp_tool_call'+ AND timestamp >= now() - INTERVAL ${REPORT_WINDOW_DAYS} DAY+ GROUP BY tool+ )+ WHERE tool NOT IN ('', 'exec')+ AND errors >= {min_errors}+ AND errors * 100 >= calls * {min_error_rate_pct}+ AND users >= {min_users}+ ORDER BY errors DESC+ LIMIT ${TOP_N}`++const TOOL_HEALTH_VALUES = {+ min_errors: MIN_ERRORS,+ min_error_rate_pct: MIN_ERROR_RATE_PCT,+ min_users: MIN_USERS,+}++async function fetchFailingTools(runHogql = hogql) {+ const result = await runHogql(TOOL_HEALTH_QUERY, TOOL_HEALTH_VALUES)+ return (result.results || []).map(([tool, calls, errors, users, topError]) => ({+ tool,+ calls: Number(calls),+ errors: Number(errors),+ users: Number(users),+ topError: topError || '',+ }))+}++// Tool names are the keys directly under `tools:` and `wrappers:` in each definition file. A line+// scan keeps the script free of a YAML dependency, and the generator enforces this layout. Codegen+// registers a `confirmed_action` tool only as its `-prepare` and `-execute` pair, so analytics never+// sees the base name.+function parseToolNames(yamlText) {+ const names = []+ let section = null+ for (const line of yamlText.split('\n')) {+ const topLevel = /^([A-Za-z_]+):/.exec(line)+ if (topLevel) {+ section = topLevel[1]+ continue+ }+ const toolKey = /^ {4}([a-z0-9][a-z0-9-]*):\s*$/.exec(line)+ if (toolKey && (section === 'tools' || section === 'wrappers')) {+ names.push(toolKey[1])+ }+ if (/^ {8}confirmed_action:/.test(line) && section === 'tools' && names.length > 0) {+ const base = names.pop()+ names.push(`${base}-prepare`, `${base}-execute`)+ }+ }+ return names+}++// Reads `'tool-name': factory,` entries of the registry and the `./path` each factory is imported+// from. Entries it cannot follow (a computed key, a spread, an inline factory) fall back to the+// directory.+function parseHandwrittenTools(registrySource) {+ const modules = new Map()+ for (const [, defaultImport, namedImports, path] of registrySource.matchAll(+ /^import (?:(\w+)|\{([^}]*)\}) from '\.\/([^']+)'/gm+ )) {+ for (const name of defaultImport ? [defaultImport] : namedImports.split(',')) {+ modules.set(name.trim(), `${HANDWRITTEN_TOOL_PATH}/${path}`)+ }+ }+ const tools = []+ for (const [, tool, factory] of registrySource.matchAll(/^ {4}'([a-z0-9-]+)': (\w+),$/gm)) {+ if (modules.has(factory)) {+ tools.push([tool, modules.get(factory)])+ }+ }+ return tools+}++function toolDefinitionIndex({+ listFiles = trackedDefinitionFiles,+ readFile = (path) => readFileSync(path, 'utf8'),+} = {}) {+ const index = new Map()+ for (const path of listFiles()) {+ for (const name of parseToolNames(readFile(path))) {+ if (!index.has(name)) {+ index.set(name, path)+ }+ }+ }+ for (const [name, path] of parseHandwrittenTools(readFile(HANDWRITTEN_TOOL_REGISTRY))) {+ if (!index.has(name)) {+ index.set(name, path)+ }+ }+ return index+}++function trackedDefinitionFiles() {+ return execFileSync('git', ['ls-files', ...TOOL_DEFINITION_GLOBS], { encoding: 'utf8' })+ .split('\n')+ .filter(Boolean)+}++function definitionPathFor(tool, index) {+ return index.get(tool) || HANDWRITTEN_TOOL_PATH+}++function formatRate(errors, calls) {+ return `${Math.round((errors * 100) / calls)}%`+}++function tableRows(items, ownerFor) {+ return items.map((item) => [+ cell(item.tool),+ cell(teamLabel(ownerFor(item).owner)),+ cell(String(item.errors)),+ cell(formatRate(item.errors, item.calls)),+ cell(String(item.users)),+ cell(item.topError || 'no message captured'),+ ])+}++function toolTable(rows) {+ return {+ type: 'table',+ column_settings: [+ { align: 'left' },+ { align: 'left' },+ { align: 'right' },+ { align: 'right' },+ { align: 'right' },+ { align: 'left' },+ ],+ rows: [+ [+ cell('tool'),+ cell('owner'),+ cell('failed calls'),+ cell('error rate'),+ cell('users hit'),+ cell('top error'),+ ],+ ...rows,+ ],+ }+}++function reportTitle(now, scope) {+ return `*Weekly MCP tool health - ${now.toISOString().slice(0, 10)}* _(${scope})_`+}++const FLOOR_NOTE = `A tool is listed when it failed at least ${MIN_ERRORS} calls, on at least ${MIN_ERROR_RATE_PCT}% of its calls, for at least ${MIN_USERS} users, in the last ${REPORT_WINDOW_DAYS} days.`++function buildBlocks(now, rows) {+ const blocks = [+ {+ type: 'section',+ text: { type: 'mrkdwn', text: reportTitle(now, `all teams, last ${REPORT_WINDOW_DAYS} days`) },+ },+ toolTable(rows),+ { type: 'context', elements: [{ type: 'mrkdwn', text: FLOOR_NOTE }] },+ ]+ const editBlock = editWorkflowBlock()+ return editBlock ? [...blocks, editBlock] : blocks+}++function buildTeamBlocks(now, { owner, rows }, digestUrl) {+ const footer = [+ ...(digestUrl ? [`<${digestUrl}|Report for all teams>`] : []),+ `${FLOOR_NOTE} Wrong owner or a tool that should be off? Tell ${FEEDBACK_CHANNEL}.`,+ ]+ return [+ { type: 'section', text: { type: 'mrkdwn', text: reportTitle(now, `owned by ${teamLabel(owner)}`) } },+ toolTable(rows),+ { type: 'context', elements: [{ type: 'mrkdwn', text: footer.join(' · ') }] },+ ]+}++// A slice kept in the digest thread. `reason` says why it is not in the team's channel.+function buildThreadSliceBlocks({ owner, channel, rows }, reason) {+ return [+ { type: 'section', text: { type: 'mrkdwn', text: `*${teamLabel(owner)}* _(${reason} ${channel})_` } },+ toolTable(rows),+ ]+}++function teamReport(now) {+ return {+ title: 'Weekly MCP tool health report',+ rowLabel: 'tools',+ teamBlocks: (team, digestUrl) => buildTeamBlocks(now, team, digestUrl),+ threadSliceBlocks: buildThreadSliceBlocks,+ }+}++// Off, each slice goes to the digest thread, labeled with the channel it would reach, so the+// routing can be checked before any team channel receives a post.+function teamPostsWithheld(enabled = TEAM_CHANNEL_POSTS) {+ return enabled ? null : 'shadow: would post to'+}++async function main() {+ if (!PROJECT_ID || !API_KEY) {+ console.warn('POSTHOG_PROJECT_ID / POSTHOG_API_KEY not set - skipping report. Wire them to enable.')+ return+ }+ const now = new Date()+ const failing = await fetchFailingTools()+ if (failing.length === 0) {+ console.info('No MCP tool above the failure floor this week - nothing to post.')+ return+ }+ const index = toolDefinitionIndex()+ const items = failing.map((item) => ({ ...item, selector: definitionPathFor(item.tool, index) }))+ const ownerFor = resolveOwners(items, (path) => [path], OWNERS_PRODUCER)+ const rows = tableRows(items, ownerFor)+ const entries = items.map((item, i) => ({ ...ownerFor(item), row: rows[i] }))+ const blocks = buildBlocks(now, rows)+ const teamDigests = buildTeamDigests(entries)+ if (DRY_RUN) {+ // Actions logs on this public repo are readable by anyone, so a dry run prints only the routing.+ // Shadow mode previews the full report, with its counts and errors, in the Slack digest thread.+ for (const item of items) {+ const { owner, slack } = ownerFor(item)+ console.info(`${item.tool} -> ${owner} (${slack ?? 'digest only'})`)+ }+ return+ }+ if (!SLACK_BOT_TOKEN) {+ throw new Error('SLACK_BOT_TOKEN not set on a non-dry run - refusing to silently skip.')+ }+ const digest = await postToSlack(blocks, 'Weekly MCP tool health report')+ console.info(`Posted weekly MCP tool health report to ${SLACK_CHANNEL}.`)+ const posted = await deliverTeamDigests(teamDigests, {+ digest,+ report: teamReport(now),+ withheld: teamPostsWithheld(),+ })+ console.info(`Posted ${posted.length}/${teamDigests.length} team slice(s) to team channels.`)+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ main().catch((err) => {+ console.error(err)+ process.exit(1)+ })+}++export {+ definitionPathFor,+ fetchFailingTools,+ HANDWRITTEN_TOOL_PATH,+ teamPostsWithheld,+ teamReport,+ toolDefinitionIndex,+}diff --git a/.github/scripts/weekly-mcp-tool-health-report.test.mjs b/.github/scripts/weekly-mcp-tool-health-report.test.mjsnew file mode 100644index 000000000000..ae0e373ced60--- /dev/null+++ b/.github/scripts/weekly-mcp-tool-health-report.test.mjs@@ -0,0 +1,143 @@+import assert from 'node:assert/strict'+import { describe, it } from 'node:test'++import {+ definitionPathFor,+ fetchFailingTools,+ HANDWRITTEN_TOOL_PATH,+ teamPostsWithheld,+ teamReport,+ toolDefinitionIndex,+} from './weekly-mcp-tool-health-report.mjs'+import { deliverTeamDigests, resolveOwners } from './weekly-report-common.mjs'++const DEFINITION = `category: Surveys+ui_apps:+ survey-list:+ type: list+tools:+ survey-create:+ operation: surveys_create+ param_overrides:+ name:+ aliases:+ survey-get:+ operation: surveys_retrieve+ survey-delete:+ operation: surveys_destroy+ confirmed_action:+ action_label: Delete survey+wrappers:+ query-survey:+ schema_ref: SurveyQuery+`++const REGISTRY = `import getLLMCosts from './aiObservability/getLLMCosts'+import {+ executeSql,+ readDataSchema,+} from './posthogAiTools'++export const TOOL_MAP = {+ 'get-llm-total-costs-for-project': getLLMCosts,+ 'read-data-schema': readDataSchema,+ [EXECUTE_SQL_TOOL_NAME]: executeSql,+ 'posthog-connection-call': () => createConnectionCallTool(resolveToolBase),+}+`++const TEAM = { owner: 'team-surveys', channel: '#team-surveys', rows: [['row']] }++// A slice's heading, a team post's digest link, or the index's header row.+function lastNote(blocks) {+ const notes = blocks+ .filter((block) => block.type !== 'table')+ .map((block) => (block.text ?? block.elements[0]).text)+ return notes.length > 0 ? notes.at(-1).split(' · ')[0] : blocks[0].rows[0].map((c) => c.text).join(' | ')+}++function recordingSlack() {+ const posts = []+ return {+ posts,+ slack: {+ post: async (blocks, text, options) => {+ posts.push({ ...options, note: lastNote(blocks) })+ return { ts: '2.0', channel: options.channel || 'C_DIGEST' }+ },+ permalink: async ({ channel, ts }) => `https://slack.test/${channel}/${ts}`,+ pause: async () => {},+ },+ }+}++describe('weekly MCP tool health report', () => {+ it('maps tools, wrappers and handwritten tools to their source file and nothing else', () => {+ const index = toolDefinitionIndex({+ listFiles: () => ['products/surveys/mcp/tools.yaml'],+ readFile: (path) => (path.endsWith('index.ts') ? REGISTRY : DEFINITION),+ })+ assert.deepEqual(+ [...index.entries()],+ [+ ['survey-create', 'products/surveys/mcp/tools.yaml'],+ ['survey-get', 'products/surveys/mcp/tools.yaml'],+ ['survey-delete-prepare', 'products/surveys/mcp/tools.yaml'],+ ['survey-delete-execute', 'products/surveys/mcp/tools.yaml'],+ ['query-survey', 'products/surveys/mcp/tools.yaml'],+ ['get-llm-total-costs-for-project', 'services/mcp/src/tools/aiObservability/getLLMCosts'],+ ['read-data-schema', 'services/mcp/src/tools/posthogAiTools'],+ ]+ )+ assert.equal(definitionPathFor('execute-sql', index), HANDWRITTEN_TOOL_PATH)+ })++ it('resolves owners for the report producer through owners.yaml', () => {+ const items = [{ selector: 'products/cdp/mcp/cdp_functions.yaml' }, { selector: HANDWRITTEN_TOOL_PATH }]+ const ownerFor = resolveOwners(items, (path) => [path], 'mcp_tool_health')+ for (const item of items) {+ assert.notEqual(ownerFor(item).owner, 'unowned')+ }+ })++ it('reads query rows into tool records', async () => {+ const tools = await fetchFailingTools(async () => ({+ results: [['survey-get', '120', '30', '7', null]],+ }))+ assert.deepEqual(tools, [{ tool: 'survey-get', calls: 120, errors: 30, users: 7, topError: '' }])+ })++ for (const { label, enabled, digestChannel = '#team-context-and-mcp', expected } of [+ {+ label: 'keeps every slice in the thread while team posts are off',+ enabled: false,+ expected: [{ threadTs: '1.0', note: '*surveys* _(shadow: would post to #team-surveys)_' }],+ },+ {+ label: 'posts a slice to its team channel and indexes it when team posts are on',+ enabled: true,+ expected: [+ { channel: '#team-surveys', note: '<https://slack.test/C_DIGEST/1.0|Report for all teams>' },+ { threadTs: '1.0', note: 'team | tools | post' },+ ],+ },+ {+ label: 'keeps the slice of the digest channel team in the thread',+ enabled: true,+ digestChannel: '#team-surveys',+ expected: [{ threadTs: '1.0', note: '*surveys* _(report already in #team-surveys)_' }],+ },+ ]) {+ it(label, async () => {+ const { posts, slack } = recordingSlack()+ await deliverTeamDigests([TEAM], {+ digest: { ts: '1.0', channel: 'C_DIGEST' },+ report: teamReport(new Date('2026-10-26T00:00:00Z')),+ withheld: teamPostsWithheld(enabled),+ digestChannel,+ slack,+ })+ assert.deepEqual(posts, expected)+ })+ }+})diff --git a/.github/scripts/weekly-report-common.mjs b/.github/scripts/weekly-report-common.mjsindex ff2e45113bee..b5f06a793f9e 100644--- a/.github/scripts/weekly-report-common.mjs+++ b/.github/scripts/weekly-report-common.mjs@@ -1,7 +1,8 @@-// Shared machinery for the weekly CI report scripts (weekly-flaky-report.mjs,-// weekly-slow-tests-report.mjs): PostHog API plumbing, HogQL, repo path and owner-// resolution, Slack table cells, and the Slack post. Everything here reads the same-// environment both workflows set, so each report owns only its signal and its table.+// Shared machinery for the weekly report scripts (weekly-flaky-report.mjs,+// weekly-slow-tests-report.mjs, weekly-mcp-tool-health-report.mjs): PostHog API plumbing,+// HogQL, repo path and owner resolution, Slack table cells, the Slack post, and the per-team+// delivery. Everything here reads the same environment every workflow sets, so each report owns+// only its signal and its table.import { execFileSync } from 'node:child_process'@@ -201,3 +202,101 @@ export async function slackPermalink({ channel, ts }) {}return data.permalink}++export const teamLabel = (owner) => owner.replace(/^team-/, '')++// The per-team slices carry the same rows as the channel digest. Takes [{owner, slack, row}]+// and groups by owner.+export function buildTeamDigests(entries) {+ const byOwner = new Map()+ for (const { owner, slack, row } of entries) {+ if (owner === 'unowned' || !slack) {+ continue+ }+ if (!byOwner.has(owner)) {+ byOwner.set(owner, { owner, channel: slack, rows: [] })+ }+ byOwner.get(owner).rows.push(row)+ }+ return [...byOwner.values()].sort((a, b) => b.rows.length - a.rows.length)+}++function buildTeamIndexBlocks(posted, rowLabel) {+ return [+ {+ type: 'table',+ column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'left' }],+ rows: [+ [cell('team'), cell(rowLabel), cell('post')],+ ...posted.map(({ owner, channel, rows, url }) => [+ cell(teamLabel(owner)),+ cell(String(rows.length)),+ url ? linkedCell([{ url, text: channel }]) : cell(channel),+ ]),+ ],+ },+ ]+}++const SLACK = {+ post: postToSlack,+ permalink: slackPermalink,+ // chat.postMessage allows about one message per second per channel.+ pause: () => new Promise((resolve) => setTimeout(resolve, 1100)),+}++async function postToTeamChannel(team, { report, digestUrl, slack }) {+ try {+ const post = await slack.post(report.teamBlocks(team, digestUrl), report.title, { channel: team.channel })+ return { ...team, url: await slack.permalink(post).catch(() => null) }+ } catch (err) {+ console.warn(`team digest for ${team.owner} failed: ${err.message}`)+ return null+ }+}++// Sends each team its slice, then indexes those posts in the digest thread. A slice that cannot+// go to its channel stays in the thread, so no team's rows are lost. `withheld` names a reason to+// keep every slice in the thread. A team whose channel is `digestChannel` keeps its slice in the+// thread too, so that channel does not get the digest twice. `report` supplies the report's+// `title`, the `rowLabel` for the index, and its `teamBlocks(team, digestUrl)` and+// `threadSliceBlocks(team, reason)` builders.+export async function deliverTeamDigests(+ teamDigests,+ { digest, report, withheld = null, digestChannel = SLACK_CHANNEL, slack = SLACK }+) {+ const digestUrl = withheld ? null : await slack.permalink(digest).catch(() => null)+ const posted = []+ for (const [index, team] of teamDigests.entries()) {+ if (index > 0) {+ await slack.pause()+ }+ const kept = withheld || (team.channel === digestChannel ? 'report already in' : null)+ const sent = kept ? null : await postToTeamChannel(team, { report, digestUrl, slack })+ if (sent) {+ posted.push(sent)+ continue+ }+ // A failed slice must not sink the slices behind it; the digest itself already landed.+ try {+ await slack.post(+ report.threadSliceBlocks(team, kept || 'not delivered to'),+ `${report.title}: ${teamLabel(team.owner)}`,+ { threadTs: digest.ts }+ )+ } catch (err) {+ console.warn(`thread slice for ${team.owner} failed: ${err.message}`)+ }+ }+ if (posted.length > 0) {+ // The team posts already landed. A failed run would invite a re-dispatch that repeats them.+ try {+ await slack.post(buildTeamIndexBlocks(posted, report.rowLabel), `${report.title}: team posts`, {+ threadTs: digest.ts,+ })+ } catch (err) {+ console.warn(`team post index failed: ${err.message}`)+ }+ }+ return posted+}diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.ymlindex 8a120f1c85c5..02629e6ba339 100644--- a/.github/workflows/ci-scripts.yml+++ b/.github/workflows/ci-scripts.yml@@ -74,7 +74,8 @@ jobs:/.nvmrc/.oxlintrc.json/.oxfmtrc.json- /owners.yaml+ **/owners.yaml+ **/product.yaml/pnpm-workspace.yaml/uv.lock/tach.tomldiff --git a/.github/workflows/weekly-mcp-tool-health-report.yml b/.github/workflows/weekly-mcp-tool-health-report.ymlnew file mode 100644index 000000000000..f7ab74dc21a4--- /dev/null+++ b/.github/workflows/weekly-mcp-tool-health-report.yml@@ -0,0 +1,73 @@+name: Weekly MCP tool health report++# Posts the week's failing MCP tools to #team-context-and-mcp every Monday morning. Sibling of+# weekly-flaky-report.yml: the script reads the `$mcp_tool_call` events the MCP server captures+# and maps each tool to the team that owns its definition through the packages/owners-yaml+# resolver, which is why the sparse checkout includes the ownership and tool definition files.+# Each owning team's rows go to that team's channel when vars.MCP_TOOL_HEALTH_TEAM_CHANNELS is+# 'true', and stay in the digest thread otherwise.+#+# Requires (no-ops with a warning until set, so it never spams failures):+# - vars.ENG_ANALYTICS_PROJECT_ID: the PostHog project that records MCP tool calls.+# - secrets.ENG_ANALYTICS_POSTHOG_API_KEY: a personal API key with query:read for that project.+# - secrets.SLACK_DEVEX_BOT_TOKEN: the existing DevEx Slack bot.++on:+ schedule:+ - cron: '50 13 * * 1' # Mondays 13:50 UTC, staggered after the slow tests report+ workflow_dispatch:+ inputs:+ dry_run:+ description: 'Print each failing tool with its owner and channel instead of posting to Slack'+ type: boolean+ default: true++permissions:+ contents: read++jobs:+ report:+ runs-on: ubuntu-24.04+ # Worst case is 3 query attempts x 150s abort plus 2 x 30s backoffs (~8.5min), plus checkout+ # and owner resolution.+ timeout-minutes: 12+ # Never run from forks (no secret access anyway), only the canonical repo.+ # Manual dispatch is master-only so branch-controlled code can't run with the secrets.+ if: >-+ github.repository == 'PostHog/posthog' &&+ (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/master')+ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ sparse-checkout: |+ .github/scripts/weekly-mcp-tool-health-report.mjs+ .github/scripts/weekly-report-common.mjs+ .nvmrc+ packages/owners-yaml+ owners.yaml+ **/owners.yaml+ **/product.yaml+ products/*/mcp/*.yaml+ services/mcp/definitions/*.yaml+ services/mcp/src/tools/index.ts+ sparse-checkout-cone-mode: false++ - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0+ with:+ node-version-file: .nvmrc++ - name: Install owners resolver dependency+ run: python3 -m pip install --quiet pyyaml++ - name: Build and post weekly MCP tool health report+ # POSTHOG_HOST rides the script's default.+ env:+ POSTHOG_PROJECT_ID: ${{ vars.ENG_ANALYTICS_PROJECT_ID }}+ POSTHOG_API_KEY: ${{ secrets.ENG_ANALYTICS_POSTHOG_API_KEY }}+ SLACK_CHANNEL: '#team-context-and-mcp'+ # 'true' posts each team's rows to that team's channel. Any other value keeps the+ # rows in the digest thread, labeled with the channel they would reach.+ MCP_TOOL_HEALTH_TEAM_CHANNELS: ${{ vars.MCP_TOOL_HEALTH_TEAM_CHANNELS }}+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_DEVEX_BOT_TOKEN }}+ DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}+ run: node .github/scripts/weekly-mcp-tool-health-report.mjsdiff --git a/owners.yaml b/owners.yamlindex 543aabec3803..82a07ccfcc83 100644--- a/owners.yaml+++ b/owners.yaml@@ -3,7 +3,7 @@ owners: []# Team slugs below are GitHub teams in this organization.github_org: PostHog# Automation that a team can redirect or silence under `notifications:`.-producers: [stamphog, visual_review, flaky_report]+producers: [stamphog, visual_review, flaky_report, mcp_tool_health]# services/mcp generate-tools reads every YAML file in a product's mcp/ directory.reserved_dirs: ['products/**/mcp/**']# A product manifest already lists its owners, so it counts as an ownership file.