:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { describe, expect, it } from 'vitest'
import { MCP_INSTRUCTIONS_CHAR_BUDGET } from '@/lib/constants'
import { buildToolDomainsCompact, type QueryToolInfo } from '@/lib/instructions'
import { InstructionsFormatter, type InstructionsContext } from '@/lib/instructions-formatter'
// `tools` mirrors production: the full tool set, query-* included (the domain
// extractor collapses them into the single `query` domain). `queryTools` below
// is the parallel catalog projection with hints.
const realisticTools = [
{ name: 'dashboard-create', category: 'Dashboards' },
{ name: 'dashboard-get', category: 'Dashboards' },
{ name: 'feature-flag-create', category: 'Feature flags' },
{ name: 'feature-flag-get-all', category: 'Feature flags' },
{ name: 'execute-sql', category: 'SQL' },
{ name: 'project-get', category: 'Core' },
{ name: 'business-knowledge-documents-search', category: 'Business knowledge' },
{ name: 'docs-search', category: 'Docs' },
{ name: 'query-trends', category: 'Query wrappers' },
{ name: 'query-funnel', category: 'Query wrappers' },
]
const realisticQueryTools: QueryToolInfo[] = [
{ name: 'query-trends', title: 'Trends', systemPromptHint: 'time series' },
{ name: 'query-funnel', title: 'Funnel', systemPromptHint: 'conversion rate' },
]
const ENV_POINTER = 'Call `project-get` without an ID to read the active project'
const GROUP_TYPES_POINTER = 'run `execute-sql` on `system.group_type_mappings`'
const fullCtx: InstructionsContext = {
guidelines: 'some guidelines',
tools: realisticTools,
queryTools: realisticQueryTools,
renderUiEnabled: true,
}
describe('InstructionsFormatter', () => {
describe('buildToolsInstructions', () => {
it('resolves every placeholder when fully populated', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions(fullCtx)
expect(result.indexOf('If the target project is ambiguous')).toBe(0)
expect(result).toContain(ENV_POINTER)
expect(result).toContain('- dashboard')
expect(result).toContain('- feature-flag')
expect(result).toContain('- execute-sql')
expect(result).toContain('- `query-funnel` — conversion rate')
expect(result).toContain('- `query-trends` — time series')
expect(result).not.toMatch(/\{tool_domains\}|\{query_tools\}|\{guidelines\}/)
})
it('renders the basic functionality, retrieving data, and examples sections', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions(fullCtx)
expect(result).toContain('### Basic functionality')
expect(result).toContain('### PostHog knowledge sources')
expect(result.indexOf('`business-knowledge-documents-search`')).toBeLessThan(
result.indexOf('`docs-search`')
)
expect(result).toContain('### Retrieving data')
expect(result).toContain('### Examples')
})
it('does not leak any CLI-only sections (tools mode is not exec mode)', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions(fullCtx)
expect(result).not.toContain('SCHEMA DRILL-DOWN RULE')
expect(result).not.toContain('Using the `posthog` tool')
expect(result).not.toContain('CLI-style command string')
})
it('omits placeholders cleanly when context fields are undefined', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions({ guidelines: 'rules' })
expect(result).not.toContain('{guidelines}')
expect(result).not.toContain('{tool_domains}')
expect(result).not.toContain('{query_tools}')
})
it('omits business knowledge guidance when search is unavailable', () => {
const formatter = new InstructionsFormatter()
expect(formatter.buildToolsInstructions({ guidelines: 'rules' })).not.toContain(
'### PostHog knowledge sources'
)
})
it('includes Inkeep guidance when only docs search is available', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions({
guidelines: 'rules',
tools: [{ name: 'docs-search', category: 'Docs' }],
})
expect(result).toContain('### PostHog knowledge sources')
expect(result).not.toContain('`business-knowledge-documents-search`')
expect(result).toContain('- PostHog features, SDKs, setup, or pricing: call the `docs-search` tool.')
})
it('omits knowledge guidance when only business knowledge search is available', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildToolsInstructions({
guidelines: 'rules',
tools: [{ name: 'business-knowledge-documents-search', category: 'Business knowledge' }],
})
expect(result).not.toContain('### PostHog knowledge sources')
})
it('always includes the agent-feedback section', () => {
const formatter = new InstructionsFormatter()
expect(formatter.buildToolsInstructions(fullCtx)).toContain('### Sharing feedback on PostHog')
expect(formatter.buildToolsInstructions({ guidelines: 'rules' })).toContain(
'### Sharing feedback on PostHog'
)
})
})
describe('buildExecInstructions', () => {
it('renders the compact instructions section with placeholders resolved', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecInstructions(fullCtx)
// query-* tools surface as the single `query` domain, not a separate catalog line
expect(result).toContain(
'business-knowledge-documents|dashboard|docs-search|execute-sql|feature-flag|project|query'
)
expect(result).not.toContain('### PostHog knowledge sources')
expect(result).not.toContain('query-*:')
// Env context is not here — it rides the exec command description, which has no
// truncation cap, leaving this payload's whole budget to the domain index.
expect(result).not.toContain(ENV_POINTER)
expect(result).not.toMatch(/\{tool_domains\}|\{query_tools\}|\{guidelines\}/)
})
// Synthetic fixture: 60 uniform `domain-N-get` names collapse to short domains and
// cost ~1/3 of what the live catalog does, which is how a 2,923-char payload shipped
// past a passing 2048 assertion. Kept for the placeholder wiring it exercises; the
// assertion that actually guards the cap runs the real catalog, in
// `instructions-formatter-snapshot.test.ts`.
it('stays under the character budget at synthetic tool counts', () => {
const manyTools = Array.from({ length: 60 }, (_, i) => ({
name: `domain-${i}-get`,
category: `Category ${i % 6}`,
}))
const manyQueryTools: QueryToolInfo[] = Array.from({ length: 12 }, (_, i) => ({
name: `query-tool-${i}`,
title: `Query tool ${i}`,
systemPromptHint: `short hint for query tool ${i}`,
}))
const formatter = new InstructionsFormatter()
const result = formatter.buildExecInstructions({
guidelines: 'guidelines',
tools: manyTools,
queryTools: manyQueryTools,
})
expect(result.length).toBeLessThanOrEqual(MCP_INSTRUCTIONS_CHAR_BUDGET)
})
// The domain index names PostHog resources, not callable tools, so without this
// line the payload never states what can actually be called. `render-ui` is only
// mounted on MCP Apps hosts — naming it elsewhere advertises a tool that isn't there.
it('names the callable tools, gating render-ui on the host mounting it', () => {
const formatter = new InstructionsFormatter()
const withUi = formatter.buildExecInstructions({ ...fullCtx, renderUiEnabled: true })
const withoutUi = formatter.buildExecInstructions({ ...fullCtx, renderUiEnabled: false })
expect(withUi).toContain('exec – run any PostHog command')
expect(withUi).toContain('render-ui – show a tool result as an interactive app.')
expect(withoutUi).toContain('exec – run any PostHog command')
expect(withoutUi).not.toContain('render-ui')
})
it('does not bleed the full command reference into the compact instructions', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecInstructions(fullCtx)
expect(result).not.toContain('SCHEMA DRILL-DOWN RULE')
expect(result).not.toContain('### Basic functionality')
expect(result).not.toContain('### Examples')
})
})
describe('buildExecToolDescription', () => {
it('renders project scope before the exec tool blurb', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecToolDescription()
expect(result).toContain('Using the `posthog` tool')
expect(result).toContain('use the explicit target or active project without reconfirming it')
expect(result.indexOf('If the target project is ambiguous')).toBe(0)
expect(result).toContain('Run `info <tool_name>` once if its schema is not in context.')
expect(result).not.toContain('### Basic functionality')
expect(result).not.toContain('### Examples')
expect(result).not.toContain('### PostHog knowledge sources')
})
it('includes docs guidance without business knowledge when only docs search is available', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecToolDescription({ docsSearchEnabled: true })
expect(result).toContain('### PostHog knowledge sources')
expect(result).not.toContain('`business-knowledge-documents-search`')
expect(result.indexOf('### PostHog knowledge sources')).toBeLessThan(
result.indexOf('Using the `posthog` tool')
)
// The mandate leads the description, ahead of the grammar it depends on.
expect(result).toContain('`call docs-search <json_input>`')
})
it('omits knowledge guidance when only business knowledge search is available', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecToolDescription({ businessKnowledgeSearchEnabled: true })
expect(result).not.toContain('### PostHog knowledge sources')
})
it('clarifies project scope before skills, business knowledge, repositories, and docs', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecToolDescription({
skillsEnabled: true,
docsSearchEnabled: true,
businessKnowledgeSearchEnabled: true,
businessKnowledgeRepoSearchEnabled: true,
})
expect(result.indexOf('If the target project is ambiguous')).toBe(0)
expect(result).toContain('After any needed project clarification')
expect(result).toContain('SKILLS FIRST')
expect(result.indexOf('SKILLS FIRST')).toBeLessThan(result.indexOf('### PostHog knowledge sources'))
expect(result.indexOf('business-knowledge-documents-search')).toBeLessThan(
result.indexOf('`call business-knowledge-repositories-search <json_input>`')
)
expect(result.indexOf('business-knowledge-repositories-search')).toBeLessThan(
result.indexOf('`call docs-search')
)
expect(result).toContain("The user's own code, or doc misses")
expect(result.length).toBeLessThanOrEqual(2048)
})
})
describe('business knowledge content gating', () => {
const bothTools = [
{ name: 'docs-search', category: 'Docs' },
{ name: 'business-knowledge-documents-search', category: 'Business knowledge' },
{ name: 'business-knowledge-repositories-search', category: 'Business knowledge' },
]
it.each([
{ name: 'unread content', content: undefined, documents: true, repos: 'fallback' },
{
name: 'sources and repositories',
content: { hasSources: true, hasRepositories: true },
documents: true,
repos: 'fallback',
},
{
name: 'sources only',
content: { hasSources: true, hasRepositories: false },
documents: true,
repos: 'none',
},
{
name: 'repositories only',
content: { hasSources: false, hasRepositories: true },
documents: false,
repos: 'primary',
},
{
name: 'no content',
content: { hasSources: false, hasRepositories: false },
documents: false,
repos: 'none',
},
{ name: 'only a failed source read', content: { hasRepositories: false }, documents: true, repos: 'none' },
])('$name', ({ content, documents, repos }) => {
const result = new InstructionsFormatter().buildToolsInstructions({
...fullCtx,
tools: bothTools,
businessKnowledgeContent: content,
})
expect(result).toContain('- PostHog features, SDKs, setup, or pricing: call the `docs-search` tool.')
expect(result.includes('call `business-knowledge-documents-search`')).toBe(documents)
expect(result.includes('call `business-knowledge-repositories-search`')).toBe(repos !== 'none')
expect(result.includes("- The user's own code, or a company question that document search")).toBe(
repos === 'fallback'
)
expect(result.includes("- The user's own company, product, code, or docs:")).toBe(repos === 'primary')
expect(result.includes('a company term that you cannot map to an event or property')).toBe(
documents || repos !== 'none'
)
})
it('gates the exec tool description the same way', () => {
const result = new InstructionsFormatter().buildExecToolDescription({
docsSearchEnabled: true,
businessKnowledgeSearchEnabled: true,
businessKnowledgeRepoSearchEnabled: true,
businessKnowledgeContent: { hasSources: false, hasRepositories: true },
})
expect(result).not.toContain('business-knowledge-documents-search')
expect(result).toContain(
"- The user's own company, code, or docs: `call business-knowledge-repositories-search <json_input>`."
)
})
})
describe('replay vision guidance', () => {
const visionTools = ['vision-scanners-inline-scan', 'vision-requests-create', 'vision-requests-get']
it.each([
{ name: 'all three scan tools', extra: visionTools, expected: true },
{ name: 'inline scan only', extra: ['vision-scanners-inline-scan'], expected: false },
{ name: 'no vision tools', extra: [], expected: false },
])('names Replay Vision only when the roster carries $name', ({ extra, expected }) => {
const tools = [...realisticTools, ...extra.map((name) => ({ name, category: 'Replay vision' }))]
const formatter = new InstructionsFormatter()
for (const rendered of [
formatter.buildToolsInstructions({ ...fullCtx, tools }),
formatter.buildExecCommandReference({ ...fullCtx, tools }),
]) {
expect(rendered.includes('use Replay Vision instead of SQL over the replay tables')).toBe(expected)
expect(rendered.includes('{replay_vision}')).toBe(false)
}
})
})
describe('inbox-first guidance', () => {
it.each([
{ name: 'tools mode', exec: false, claim: true, list: 'the `inbox-reports-list` tool' },
{ name: 'exec mode', exec: true, claim: true, list: '`call inbox-reports-list <json_input>`' },
{
name: 'exec mode without claim',
exec: true,
claim: false,
list: '`call inbox-reports-list <json_input>`',
},
])('renders the section in $name', ({ exec, claim, list }) => {
const ctx = { ...fullCtx, inboxFirstEnabled: true, inboxClaimEnabled: claim }
const formatter = new InstructionsFormatter()
const rendered = exec ? formatter.buildExecCommandReference(ctx) : formatter.buildToolsInstructions(ctx)
expect(rendered).toContain('### Existing Inbox reports')
expect(rendered).toContain(`Look for a report with ${list}`)
expect(rendered).toContain('`suppression_source` is `safety_judge`')
expect(rendered.includes('inbox-reports-claim')).toBe(claim)
expect(rendered).not.toMatch(/\{inbox_/)
})
it('omits the section when it is not enabled', () => {
const formatter = new InstructionsFormatter()
for (const rendered of [
formatter.buildToolsInstructions(fullCtx),
formatter.buildExecCommandReference(fullCtx),
formatter.buildExecToolDescription({ skillsEnabled: true, docsSearchEnabled: true }),
]) {
expect(rendered).not.toContain('Existing Inbox reports')
}
})
})
describe('buildExecCommandReference', () => {
it.each([
{ name: 'both lookup tools', tools: ['project-get', 'execute-sql'], project: true, groups: true },
{ name: 'only project-get', tools: ['project-get'], project: true, groups: false },
{ name: 'only execute-sql', tools: ['execute-sql'], project: false, groups: true },
{ name: 'neither lookup tool', tools: [], project: false, groups: false },
{
name: 'the switch tools too',
tools: ['project-get', 'switch-project', 'switch-organization', 'projects-get', 'organizations-get'],
project: true,
groups: false,
},
{
name: 'switch-project without its lookups',
tools: ['project-get', 'switch-project'],
project: true,
groups: false,
},
])('names only the env lookup tools the roster has: $name', ({ tools, project, groups }) => {
const ctx = { ...fullCtx, tools: tools.map((name) => ({ name, category: 'Core' })) }
const result = new InstructionsFormatter().buildExecCommandReference(ctx)
expect(result.includes(ENV_POINTER)).toBe(project)
expect(result.includes(GROUP_TYPES_POINTER)).toBe(groups)
expect(result.includes('call `switch-project` or `switch-organization`')).toBe(
tools.includes('organizations-get')
)
expect(result.includes('### Active environment')).toBe(project || groups)
})
it('carries the CLI mechanics', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecCommandReference(fullCtx)
expect(result).toContain('SCHEMA DRILL-DOWN RULE')
expect(result).not.toContain('### PostHog knowledge sources')
expect(result).toContain('### Basic functionality')
expect(result).toContain('### Examples')
})
it('does not include the exec-tool blurb (that lives on the tool description)', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecCommandReference(fullCtx)
expect(result).not.toContain('Using the `posthog` tool')
})
it('embeds the env-context pointer and query-tool catalog', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildExecCommandReference(fullCtx)
expect(result).toContain(ENV_POINTER)
// Tool domains are temporarily omitted from the command reference while
// probing claude.ai's per-tool size cap; discovery rides on `search`.
expect(result).not.toContain('dashboard|execute-sql')
expect(result).toContain('- `query-trends` — time series')
})
it('always includes the agent-feedback section', () => {
const formatter = new InstructionsFormatter()
const withFeedback = formatter.buildExecCommandReference(fullCtx)
expect(withFeedback).toContain('### Sharing feedback on PostHog')
})
it('includes the rendering section only when render-ui is available for the client', () => {
const formatter = new InstructionsFormatter()
const withRendering = formatter.buildExecCommandReference(fullCtx)
expect(withRendering).toContain('### Rendering visualizations')
// The raw flag being on isn't enough — a non-UI-host client (e.g. Claude Code)
// resolves `renderUiEnabled` to false and must not see the rendering section.
const withoutRendering = formatter.buildExecCommandReference({ ...fullCtx, renderUiEnabled: false })
expect(withoutRendering).not.toContain('### Rendering visualizations')
})
})
describe('Claude web/desktop exec guidance', () => {
it('keeps routine guidance inline and advertises optional topics', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildClaudeExecCommandReference(fullCtx)
// Skills off is the shipped default: the reference must carry no skill syntax.
expect(result).not.toContain('(posthog|project):<skill>')
expect(result).not.toContain('SKILLS FIRST')
expect(result).toContain('**LEARN FIRST: HARD REQUIREMENT**')
expect(result).toContain('learn <topic...> - load one or more learning topics')
expect(result).toContain('Topics are cumulative.')
expect(result).toContain('User: create pageviews visualization')
expect(result).toContain(
"Assistant: This needs analytics and visualization guidance, so I'll load both first."
)
expect(result).toContain('posthog:exec({"command":"learn analytics visualizations"})')
expect(result).toContain('User: How many weekly active users do we have?')
expect(result).toContain('render-ui({ "tool_name": "query-trends", "tool_input": {...} })')
expect(result.indexOf('render-ui({ "tool_name": "query-trends"')).toBeGreaterThan(
result.indexOf('posthog:exec({"command":"call query-trends {...}"})')
)
expect(result).toContain('- analytics:')
expect(result).toContain('- visualizations:')
expect(result).toContain('- urls:')
expect(result).toContain('- feedback:')
expect(result).toContain('SCHEMA DRILL-DOWN RULE')
expect(result).toContain('**Data discovery:**')
expect(result).toContain('**CORRECT usage pattern:**')
expect(result.indexOf('User: create pageviews visualization')).toBeGreaterThan(
result.indexOf('**CORRECT usage pattern:**')
)
expect(result).toContain('### Basic functionality')
expect(result).toContain('### Tool search')
expect(result).toContain(buildToolDomainsCompact(realisticTools))
expect(result).toContain(ENV_POINTER)
expect(result).not.toContain('### Retrieving data')
expect(result).not.toContain('### Examples')
expect(result).not.toContain('### Rendering visualizations')
expect(result).not.toContain('### URL patterns')
expect(result).not.toContain('### Sharing feedback on PostHog')
expect(result).not.toContain('- `query-trends` — time series')
expect(result).not.toMatch(/\{help_topics\}|\{query_tools\}|\{guidelines\}/)
})
it('advertises skill syntax inside the schema budget only when skills are enabled', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildClaudeExecCommandReference(fullCtx, { skillsEnabled: true })
expect(result).toContain('**SKILLS FIRST: HARD REQUIREMENT**')
expect(result).toContain('(posthog|project):<skill> [path...]')
// The compact variant, not the full routing paragraph, which belongs to the uncapped reference.
expect(result).not.toContain('never clone, grep, or web-search PostHog internals')
expect(result).toContain('- analytics:')
expect(result).toContain('learn <topic...> - load one or more learning topics')
})
it('keeps URL patterns inline when learn is unavailable', () => {
const formatter = new InstructionsFormatter()
const result = formatter.buildClaudeExecCommandReference(fullCtx, {
learnEnabled: false,
skillsEnabled: false,
})
expect(result).toContain('### URL patterns')
expect(result).not.toContain('- urls:')
expect(result).not.toContain('learn <topic...>')
})
it('adds the full skills-first routing to the uncapped reference only when enabled', () => {
const formatter = new InstructionsFormatter()
const off = formatter.buildExecCommandReference(fullCtx)
const on = formatter.buildExecCommandReference(fullCtx, { learnEnabled: true })
expect(off).not.toContain('SKILLS FIRST')
expect(on).toContain('**SKILLS FIRST: HARD REQUIREMENT**')
expect(on).toContain('never clone, grep, or web-search PostHog internals')
})
it('builds optional learning topics from their full guidance', () => {
const formatter = new InstructionsFormatter()
const entries = formatter.buildClaudeExecLearnGuides(fullCtx)
const analytics = entries.find((entry) => entry.id === 'analytics')
expect(entries.map(({ id }) => id)).toEqual(['analytics', 'visualizations', 'urls', 'feedback'])
expect(analytics?.content).toContain('### Retrieving data')
expect(analytics?.content).toContain('### Examples')
expect(analytics?.content).toContain('- `query-trends` — time series')
expect(entries.find((entry) => entry.id === 'visualizations')?.content).toContain(
'### Rendering visualizations'
)
expect(entries.find((entry) => entry.id === 'urls')?.content).toContain('### URL patterns')
expect(entries.find((entry) => entry.id === 'feedback')?.content).toContain(
'### Sharing feedback on PostHog'
)
})
it('only advertises visualizations when rendering is available', () => {
const formatter = new InstructionsFormatter()
const ctx = {
...fullCtx,
renderUiEnabled: false,
}
expect(formatter.buildClaudeExecLearnGuides(ctx).map((entry) => entry.id)).toEqual([
'analytics',
'urls',
'feedback',
])
const result = formatter.buildClaudeExecCommandReference(ctx)
expect(result).toContain('- analytics:')
expect(result).not.toContain('- visualizations:')
expect(result).toContain('- urls:')
expect(result).toContain('- feedback:')
})
})
// A blanket "search before every answer" mandate fired a docs or business-knowledge search
// on unrelated requests. The exec description reaches any client that lists tool
// descriptions, so both surfaces carrying the section must scope it by topic.
describe('knowledge-search scoping', () => {
it.each([
{
name: 'buildToolsInstructions',
render: (formatter: InstructionsFormatter) => formatter.buildToolsInstructions(fullCtx),
},
{
name: 'buildExecToolDescription',
render: (formatter: InstructionsFormatter) =>
formatter.buildExecToolDescription({
docsSearchEnabled: true,
businessKnowledgeSearchEnabled: true,
}),
},
])('$name only mandates a search for PostHog and company questions', ({ render }) => {
const result = render(new InstructionsFormatter())
expect(result).toContain('### PostHog knowledge sources')
expect(result).not.toMatch(/every user request|even when the request looks simple/)
expect(result).toContain('call neither search')
// Nearly every request to this server is about project data, so that topic must not trigger a search.
expect(result).not.toContain("the user's PostHog project and its data")
expect(result).toContain(
'A request to query, compute, or build something from project data: call neither search'
)
// The gate is the topic, never the location: a PostHog SDK question that names a
// file in the user's repository still has to reach docs-search.
expect(result).toContain('The topic decides, not the location')
})
})
describe('metric discovery gating (data catalog flag)', () => {
const surfaces: {
name: string
render: (formatter: InstructionsFormatter, ctx: InstructionsContext) => string
mustPrecede: string[]
}[] = [
{
name: 'buildToolsInstructions',
render: (formatter, ctx) => formatter.buildToolsInstructions(ctx),
mustPrecede: ['### Retrieving data', '#### Schema-first workflow'],
},
{
name: 'analytics learn topic content',
render: (formatter, ctx) =>
formatter.buildClaudeExecLearnGuides(ctx).find((entry) => entry.id === 'analytics')!.content,
mustPrecede: ['### Retrieving data', '#### Schema-first workflow'],
},
{
name: 'buildExecCommandReference',
render: (formatter, ctx) => formatter.buildExecCommandReference(ctx),
mustPrecede: ['SCHEMA DRILL-DOWN RULE', '### Retrieving data'],
},
]
it.each(surfaces)('$name puts metric routing before generic analytics guidance', ({ render, mustPrecede }) => {
const formatter = new InstructionsFormatter()
const rendered = render(formatter, fullCtx)
const metricRoutingPosition = rendered.indexOf('#### Metric discovery (semantic layer)')
expect(metricRoutingPosition).toBeGreaterThanOrEqual(0)
expect(rendered).toContain('metric-list')
expect(rendered).toContain('metric-describe')
expect(rendered).toContain('data-catalog-metric-run')
for (const genericGuidance of mustPrecede) {
expect(metricRoutingPosition).toBeLessThan(rendered.indexOf(genericGuidance))
}
})
it('advertises governed metrics in the analytics topic description', () => {
const formatter = new InstructionsFormatter()
const analyticsEntry = formatter
.buildClaudeExecLearnGuides(fullCtx)
.find((entry) => entry.id === 'analytics')!
expect(analyticsEntry.description).toContain('governed metrics')
})
})
describe('analysis artifact guidance', () => {
const surfaces: {
name: string
render: (formatter: InstructionsFormatter, ctx: InstructionsContext) => string
}[] = [
{
name: 'buildToolsInstructions',
render: (formatter, ctx) => formatter.buildToolsInstructions(ctx),
},
{
name: 'analytics learn topic content',
render: (formatter, ctx) =>
formatter.buildClaudeExecLearnGuides(ctx).find((entry) => entry.id === 'analytics')!.content,
},
{
name: 'buildExecCommandReference',
render: (formatter, ctx) => formatter.buildExecCommandReference(ctx),
},
]
it.each(surfaces)('$name routes deep dives to notebooks and tracking to dashboards', ({ render }) => {
const formatter = new InstructionsFormatter()
const result = render(formatter, fullCtx)
expect(result).toContain('### Where an analysis lands')
expect(result).toContain('**Notebook**')
expect(result).toContain('**Dashboard**')
})
it.each(surfaces)(
'$name sends chart explanations to explain-chart only when explain-chart exists',
({ render }) => {
const formatter = new InstructionsFormatter()
const withTool = render(formatter, {
...fullCtx,
tools: [...(fullCtx.tools ?? []), { name: 'explain-chart', category: 'Insights' }],
})
const withoutTool = render(formatter, {
...fullCtx,
tools: fullCtx.tools?.filter(({ name }) => name !== 'explain-chart'),
})
expect(withTool).toContain('### Explaining a chart')
expect(withTool.indexOf('### Explaining a chart')).toBeGreaterThan(withTool.indexOf('**Notebook**'))
expect(withoutTool).not.toContain('explain-chart')
}
)
// The SQL and Python guidance names `notebooks-add-cell`, so it must stay out of prompts
// for clients that aren't advertised the cell tools.
it.each(surfaces)('$name gates the SQL and Python sections on the notebook cell tools', ({ render }) => {
const formatter = new InstructionsFormatter()
const cellsOn = render(formatter, { ...fullCtx, notebookCellsEnabled: true })
expect(cellsOn).toContain('### SQL in an analysis')
expect(cellsOn).toContain("cell_type: 'sql'")
expect(cellsOn).toContain('### Python in an analysis')
expect(cellsOn).toContain("cell_type: 'python'")
// A connection can carry the cell tools without execute-sql, which needs another scope.
const sqlSection = (text: string): string =>
text.slice(text.indexOf('### SQL in an analysis'), text.indexOf('### Python in an analysis'))
expect(sqlSection(cellsOn)).toContain('`execute-sql`')
const withoutExecuteSql = render(formatter, {
...fullCtx,
notebookCellsEnabled: true,
tools: fullCtx.tools?.filter(({ name }) => name !== 'execute-sql'),
})
expect(sqlSection(withoutExecuteSql)).not.toContain('execute-sql')
const cellsOff = render(formatter, { ...fullCtx, notebookCellsEnabled: false })
expect(cellsOff).not.toContain('### SQL in an analysis')
expect(cellsOff).not.toContain('### Python in an analysis')
expect(cellsOff).toBe(render(formatter, fullCtx))
})
})
describe('activity history guidance', () => {
const surfaces: {
name: string
render: (formatter: InstructionsFormatter, ctx: InstructionsContext) => string
inline: boolean
}[] = [
{ name: 'tools', render: (f, ctx) => f.buildToolsInstructions(ctx), inline: true },
{ name: 'exec', render: (f, ctx) => f.buildExecCommandReference(ctx), inline: true },
{
name: 'Claude analytics guide',
render: (f, ctx) =>
f.buildClaudeExecLearnGuides(ctx).find((entry) => entry.id === 'analytics')!.content,
inline: true,
},
{
name: 'Claude without guides',
render: (f, ctx) => f.buildClaudeExecCommandReference(ctx, { learnEnabled: false }),
inline: true,
},
{
name: 'Claude with guides',
render: (f, ctx) => f.buildClaudeExecCommandReference(ctx),
inline: false,
},
]
it.each(
surfaces.flatMap((surface) =>
[
{ capability: 'neither reader', tools: [], history: false, sql: false },
{ capability: 'SQL only', tools: ['execute-sql'], history: false, sql: false },
{
capability: 'activity reader only',
tools: ['advanced-activity-logs-list'],
history: true,
sql: false,
},
{
capability: 'both readers',
tools: ['advanced-activity-logs-list', 'execute-sql'],
history: true,
sql: true,
},
].map((capability) => ({ ...surface, ...capability }))
)
)('$name exposes only supported history procedures: $capability', ({ render, inline, tools, history, sql }) => {
const result = render(new InstructionsFormatter(), {
...fullCtx,
tools: tools.map((name) => ({ name, category: 'Platform Features' })),
})
expect(result.includes('### Activity history')).toBe(inline && history)
expect(result.includes('`system.activity_logs`')).toBe(inline && sql)
if (inline && history) {
expect(result).toContain('`start_date`, `end_date`')
expect(result).toContain('`page_size: 10`')
expect(result).toContain('`clients: ["scout:<skill_name>"]`')
expect(result).toContain('If access is denied, stop using this reader')
expect(result).toContain('Other advertised, authorized history readers, including per-object readers')
}
})
})
// Mirrors the single-exec wiring in `src/mcp.ts`. When the client honors the MCP
// `instructions` field, that payload carries exactly one thing — the tool-domain index
// (including the `query` domain) — because clients hard-truncate it. Everything else,
// env-context and the query-tool catalog included, stays on the `command` description,
// which has no cap. Codex (no `instructions` support) keeps today's behavior: empty
// `instructions`, everything inlined in the `command` description.
describe('exec mode wiring', () => {
it.each([
{ name: 'supportsInstructions=true (Claude Code etc.)', supportsInstructions: true },
{ name: 'supportsInstructions=false (Codex)', supportsInstructions: false },
])('$name: splits product context between instructions and commandReference', ({ supportsInstructions }) => {
const formatter = new InstructionsFormatter()
const instructions = supportsInstructions ? formatter.buildExecInstructions(fullCtx) : ''
const commandReference = formatter.buildExecCommandReference(fullCtx)
expect(commandReference).toContain('SCHEMA DRILL-DOWN RULE')
expect(commandReference).toContain('### Basic functionality')
// the query catalog always lives on the command reference, both modes
expect(commandReference).toContain('- `query-trends` — time series')
if (supportsInstructions) {
// queries surface in instructions only as the `query` tool domain
expect(instructions).toContain(
'business-knowledge-documents|dashboard|docs-search|execute-sql|feature-flag|project|query'
)
expect(instructions).not.toContain('- `query-trends` — time series')
expect(instructions).not.toContain(ENV_POINTER)
expect(commandReference).toContain(ENV_POINTER)
expect(commandReference).not.toContain('dashboard|execute-sql')
} else {
expect(instructions).toBe('')
expect(commandReference).toContain('- `query-trends` — time series')
expect(commandReference).toContain(ENV_POINTER)
expect(commandReference).not.toContain('dashboard|execute-sql')
}
})
})
})