:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { strToU8, zipSync } from 'fflate'
import { describe, expect, it, vi } from 'vitest'
import type { ProjectSkillCatalog } from '@/skills/project-skill-catalog'
import {
formatLearnFile,
LEARN_OUTPUT_CHAR_LIMIT,
makeSkillFile,
MAX_GLOBAL_SKILLS,
readLearnLines,
searchLearnFile,
SkillCatalog,
} from '@/skills/skill-catalog'
import { ExecLearnCatalog } from '@/tools/exec-learn'
function makeArchive(files: Record<string, string>): Uint8Array {
return zipSync(Object.fromEntries(Object.entries(files).map(([path, content]) => [path, strToU8(content)])))
}
function makeSkill(name: string, description: string, body: string): string {
return `---\nname: ${name}\ndescription: ${description}\n---\n\n${body}`
}
function makeCatalog(): SkillCatalog {
return SkillCatalog.fromZip(
makeArchive({
'retention-analysis/SKILL.md': makeSkill(
'retention-analysis',
'Find where users stop returning. 日本語のガイド。',
'# Retention analysis\n\nQuery weekly retention cohorts.'
),
'retention-analysis/references/functions.md':
'# Available functions\n\nUse retention cohorts with dateDiff.',
'retention-analysis/scripts/run.ts': 'const privateSearchToken = "unindexed-token"',
'funnels/SKILL.md': makeSkill(
'funnels',
'Analyze conversion funnels.',
'# Funnels\n\nBuild a conversion funnel.'
),
})
)
}
function makeProjectSkills(overrides: Partial<Record<string, unknown>> = {}): ProjectSkillCatalog {
return {
listNames: vi.fn(async () => ({ count: 1, names: ['team-conventions'], truncated: false })),
describe: vi.fn(async () => new Map([['team-conventions', 'Team-specific conventions.']])),
searchResults: vi.fn(async () => []),
read: vi.fn(async () => 'project skill'),
searchFile: vi.fn(async () => 'project file match'),
readLines: vi.fn(async () => 'project lines'),
...overrides,
} as unknown as ProjectSkillCatalog
}
describe('SkillCatalog and exec learn', () => {
it('lists qualified names without descriptions and ranks full-text matches deterministically', async () => {
const catalog = makeCatalog()
const learn = new ExecLearnCatalog([], { posthog: catalog })
expect(JSON.parse(await learn.execute('skills'))).toEqual({
posthog: {
available: true,
count: 2,
listed: 2,
truncated: false,
skills: ['posthog:funnels', 'posthog:retention-analysis'],
},
project: {
available: false,
reason: 'Project skills are temporarily unavailable.',
},
})
const result = await learn.execute('-s weekly retention')
expect(result).toContain('## posthog:retention-analysis')
expect(result).toContain('SKILL.md:8: Query weekly retention cohorts.')
expect(result).toContain('[Project skills unavailable: Project skills are temporarily unavailable.]')
await expect(learn.execute('project:team-skill')).rejects.toThrow('Project skills are temporarily unavailable')
})
it('indexes every file path and Markdown content but not script contents', () => {
const catalog = makeCatalog()
expect(catalog.search('functions.md')).toContain('retention-analysis')
expect(catalog.search('dateDiff')).toContain('retention-analysis')
expect(catalog.search('run.ts')).toContain('retention-analysis')
expect(catalog.search('unindexed-token')).toBe('No skills matched "unindexed-token".')
expect(() => catalog.searchFile('retention-analysis', 'scripts/run.ts', 'const')).toThrow(
'Only Markdown contents are searchable'
)
})
it('uses the strongest companion file match for its score and snippets', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'team-guide/SKILL.md': makeSkill('team-guide', 'Internal workflow.', '# Team guide'),
'team-guide/references/a.md': '# Guide\n\nSupport only.',
'team-guide/references/z.md': '# Guide\n\nSupport queue.',
})
)
expect(catalog.searchResults('support queue')[0]).toMatchObject({
score: 120,
snippets: [{ path: 'references/z.md' }],
})
})
it('overlays a second archive, keeps the product skill on a shared name, and rejects duplicates within one archive', () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const catalog = SkillCatalog.fromZip(
makeArchive({
'funnels/SKILL.md': makeSkill('funnels', 'Product funnels.', '# Funnels'),
'retention-analysis/SKILL.md': makeSkill('retention-analysis', 'Retention.', '# Retention'),
}),
makeArchive({
'funnels/SKILL.md': makeSkill('funnels', 'Overlay funnels.', '# Overlay funnels'),
'instrument-logs/SKILL.md': makeSkill('instrument-logs', 'Add logs.', '# Add logs'),
})
)
expect(catalog.listNames()).toEqual(['funnels', 'instrument-logs', 'retention-analysis'])
expect(catalog.describe('funnels')).toBe('Product funnels.')
expect(warn).toHaveBeenCalledTimes(1)
expect(() =>
SkillCatalog.fromZip(
makeArchive({
'a/SKILL.md': makeSkill('twice', 'One.', '# One'),
'b/SKILL.md': makeSkill('twice', 'Two.', '# Two'),
})
)
).toThrow('Duplicate skill name')
} finally {
warn.mockRestore()
}
})
it('scores an omnibus instrument skill by its entry point only, never its references', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'instrument-logs/SKILL.md': makeSkill(
'instrument-logs',
'Add PostHog log capture to an app.',
'# Add logs\n\nInstall the SDK and ship logs.'
),
'instrument-logs/references/go.md': '# Go\n\nConfigure the zerolog adapter.',
'investigating-logs/SKILL.md': makeSkill(
'investigating-logs',
'Find failing requests in logs.',
'# Investigating logs\n\nFilter zerolog output by severity.'
),
})
)
expect(catalog.searchResults('zerolog').map((result) => result.identifier)).toEqual(['investigating-logs'])
expect(catalog.searchResults('capture').map((result) => result.identifier)).toEqual(['instrument-logs'])
expect(catalog.read('instrument-logs', 'references/go.md')).toContain('zerolog adapter')
})
it('ranks an exact name before substring matches at the result limit', () => {
const names = [
...Array.from({ length: MAX_GLOBAL_SKILLS }, (_, index) => `${String.fromCharCode(97 + index)}-support`),
'support',
]
const catalog = SkillCatalog.fromZip(
makeArchive(
Object.fromEntries(
names.map((name) => [`${name}/SKILL.md`, makeSkill(name, 'Internal workflow.', '# Guide')])
)
)
)
expect(catalog.searchResults('support')[0]?.identifier).toBe('support')
})
it('bounds token expansion while preserving informative matches', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'first-token-match/SKILL.md': makeSkill('first-token-match', 'Contains alpha.', '# First'),
'long-token-match/SKILL.md': makeSkill('long-token-match', 'Contains ninthtoken.', '# Long'),
'overflow-match/SKILL.md': makeSkill('overflow-match', 'Contains golf.', '# Overflow'),
})
)
expect(
catalog
.searchResults('a alpha bravo charlie delta echo foxtrot golf hotel ninthtoken')
.map((result) => result.identifier)
).toEqual(['first-token-match', 'long-token-match'])
})
it('returns the rendered skill with a manifest and supports scoped reads', async () => {
const catalog = makeCatalog()
const learn = new ExecLearnCatalog([], { posthog: catalog })
const result = await learn.execute('posthog:retention-analysis')
expect(result).toContain('Files:')
expect(result).toContain('- SKILL.md (8 lines,')
expect(result).toContain('- references/functions.md (3 lines,')
expect(result).toContain('Read a file with `learn posthog:retention-analysis <path>`.')
expect(await learn.execute('posthog:retention-analysis references/functions.md')).toContain(
'# Available functions'
)
expect(result).toContain('# Retention analysis')
expect(result).not.toContain('name: retention-analysis')
expect(catalog.searchFile('retention-analysis', 'references/functions.md', 'dateDiff')).toContain(
'3: Use retention cohorts with dateDiff.'
)
expect(catalog.readLines('retention-analysis', 'references/functions.md', 1, 2)).toContain(
'1: # Available functions\n2:'
)
expect(await learn.execute('posthog:retention-analysis references/functions.md -s dateDiff')).toContain(
'3: Use retention cohorts with dateDiff.'
)
expect(await learn.execute('posthog:retention-analysis references/functions.md --lines 1:2')).toContain(
'1: # Available functions\n2:'
)
})
it('merges PostHog and project search results by relevance and supports Unicode and quoted paths', async () => {
const projectSkills = makeProjectSkills({
searchResults: vi.fn(async (query: string) => [
{
identifier: 'project:conversion-playbook',
description: 'Team conversion playbook.',
snippets: [{ path: 'SKILL.md', line: 4, text: 'Our conversion funnel.' }],
// A name match on "conversion" must outrank a PostHog body-only match.
score: query.includes('conversion') ? 3300 : 900,
},
]),
})
const learn = new ExecLearnCatalog([], { posthog: makeCatalog(), project: projectSkills })
const listed = JSON.parse(await learn.execute('skills'))
const ranked = await learn.execute('-s conversion')
const unicodeResult = await learn.execute('-s 日本語')
expect(listed.project).toEqual({
available: true,
count: 1,
listed: 1,
truncated: false,
skills: ['project:team-conventions'],
})
// Regression guard: the name-matching project skill outranks PostHog's `funnels`,
// which only matches on description/body — the old code always ordered PostHog first.
expect(ranked.indexOf('## project:conversion-playbook')).toBeLessThan(ranked.indexOf('## posthog:funnels'))
expect(ranked.indexOf('## posthog:funnels')).toBeGreaterThan(-1)
expect(unicodeResult).toContain('## posthog:retention-analysis')
expect(unicodeResult).toContain('## project:conversion-playbook')
expect(projectSkills.searchResults).toHaveBeenLastCalledWith('日本語')
await expect(
learn.execute('project:team-conventions "references/team guide.md" -s weekly retention')
).resolves.toBe('project file match')
expect(projectSkills.searchFile).toHaveBeenCalledWith(
'team-conventions',
'references/team guide.md',
'weekly retention'
)
})
it('returns an outline for an oversized reference and rejects unsafe paths', () => {
const largeBody = `# Function catalog\n\n${'function signature details\n'.repeat(2_000)}\n## Examples\n`
const catalog = SkillCatalog.fromZip(
makeArchive({
'querying-data/SKILL.md': makeSkill('querying-data', 'Query product data.', '# Querying data'),
'querying-data/references/functions.md': largeBody,
})
)
const result = catalog.read('querying-data', 'references/functions.md')
expect(result.length).toBeLessThanOrEqual(LEARN_OUTPUT_CHAR_LIMIT)
expect(result).toContain('This file is too large to return in full.')
expect(result).toContain('1: # Function catalog')
expect(result).toContain('2004: ## Examples')
expect(() => catalog.read('querying-data', '../SKILL.md')).toThrow('Unsafe skill path')
})
it.each([
['needle\na\nb\nc', '1: needle\n2: a\n3: b'],
['a\nb\nc\nneedle', '2: b\n3: c\n4: needle'],
['a\nneedle\nneedle\nb\n', '1: a\n2: needle\n3: needle\n4: b\n\n1: a\n2: needle\n3: needle\n4: b\n5: '],
['\n\nneedle 日本語 🦔\n', '1: \n2: \n3: needle 日本語 🦔\n4: '],
])('preserves line numbers and overlapping search context for %j', (content, context) => {
const file = makeSkillFile('notes.md', content)
expect(searchLearnFile('example', file, 'needle')).toBe(`Matches in example/notes.md:\n\n${context}`)
})
it('caps file search at 50 matches while including the final match context', () => {
const file = makeSkillFile(
'notes.md',
Array.from({ length: 60 }, (_, index) => `needle ${index}\nbefore\nmiddle\nafter`).join('\n')
)
const output = searchLearnFile('example', file, 'needle')
expect(output.split('\n\n')).toHaveLength(51)
expect(output).toContain('197: needle 49\n198: before\n199: middle')
expect(output).not.toContain('needle 50')
})
it.each(['# ' + 'x'.repeat(100_000), '# Heading\n'.repeat(10_000)])(
'bounds oversized outlines and search output for long lines and many headings (%#)',
(content) => {
const file = makeSkillFile('notes.md', content)
const outline = formatLearnFile('example', file)
expect(outline).toHaveLength(LEARN_OUTPUT_CHAR_LIMIT)
expect(outline).toContain('[Output truncated.')
const search = searchLearnFile('example', file, file.lineCount === 1 ? 'x' : 'heading')
expect(search.length).toBeLessThanOrEqual(LEARN_OUTPUT_CHAR_LIMIT)
if (file.lineCount === 1) {
expect(search).toContain('[Output truncated.')
} else {
expect(search.split('\n\n')).toHaveLength(51)
}
}
)
it('reads and searches a newline-heavy file, including its last and empty lines', () => {
const file = makeSkillFile('notes.md', `x${'\n'.repeat(999_998)}x`)
expect(file.lineCount).toBe(999_999)
expect(formatLearnFile('example', file)).toContain('(No Markdown headings found.)')
expect(searchLearnFile('example', file, 'missing')).toContain('No matches for "missing"')
expect(readLearnLines('example', file, 999_998, 999_999)).toBe(
'File: example/notes.md lines 999998:999999\n\n999998: \n999999: x'
)
expect(() => readLearnLines('example', file, 1, file.lineCount)).toThrow('Requested line range exceeds')
expect(() => readLearnLines('example', file, 1, file.lineCount + 1)).toThrow('Invalid line range')
const empty = makeSkillFile('empty.md', '')
expect(empty.lineCount).toBe(1)
expect(readLearnLines('example', empty, 1, 1)).toBe('File: example/empty.md lines 1:1\n\n1: ')
})
it('rejects unsafe archive entries before exposing them', () => {
const archive = makeArchive({
'../escape/SKILL.md': makeSkill('escape', 'Unsafe archive.', '# Escape'),
})
expect(() => SkillCatalog.fromZip(archive)).toThrow('Unsafe skill archive path')
})
it.each([
['analyzing', 'trends', true], // analyzing → analyz ⊂ "analyze"
['funnels', 'conversion', true], // funnels → funnel
['queries', 'querying', true], // queries → query
['bayous', 'wetlands', true], // bayous → bayou
['types', 'classification', false], // four-character derived stems remain excluded
['sessions', 'assessment', false], // min-5 guard: "sessions" must not reach "assessing"
['states', 'statistics', false], // "states" must not lose the distinct final e in "state"
])('light stemming links query "%s" to %s content (match=%s)', (query, skill, shouldMatch) => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'trends/SKILL.md': makeSkill(
'trends',
'Chart product metrics over time.',
'# Trends\n\nAnalyze a cohort.'
),
'conversion/SKILL.md': makeSkill(
'conversion',
'Chart product metrics over time.',
'# Conversion\n\nBuild a conversion funnel.'
),
'querying/SKILL.md': makeSkill('querying', 'Run database lookups.', '# Querying\n\nRun a query.'),
'wetlands/SKILL.md': makeSkill('wetlands', 'Explore waterways.', '# Wetlands\n\nExplore a bayou.'),
'classification/SKILL.md': makeSkill(
'classification',
'Review classification labels.',
'# Classification\n\nReview a stereotype.'
),
'assessment/SKILL.md': makeSkill(
'assessment',
'Review internal controls.',
'# Assessment\n\nStart by assessing exposure.'
),
'statistics/SKILL.md': makeSkill(
'statistics',
'Calculate descriptive measures.',
'# Statistics\n\nSummarize the sample.'
),
})
)
if (shouldMatch) {
expect(catalog.search(query)).toContain(skill)
} else {
expect(catalog.search(query)).not.toContain(skill)
}
})
it('does not stem singular words ending in s', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'statue/SKILL.md': makeSkill('statue', 'Sculpture reference.', '# Statue'),
'support-workflow/SKILL.md': makeSkill(
'support-workflow',
'Track support status.',
'# Support workflow'
),
})
)
expect(catalog.searchResults('status').map((result) => result.identifier)).toEqual(['support-workflow'])
})
it('excludes SKILL.md frontmatter from snippets while keeping raw line numbers', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'cohort-guide/SKILL.md': makeSkill(
'cohort-guide',
'Analyze retention cohorts.',
'# Cohort guide\n\nAnalyze retention cohorts weekly.'
),
})
)
const output = catalog.search('retention cohorts')
// The `description:` frontmatter line (raw line 3) must not surface as a snippet — the
// description already prints on its own — and the body match keeps its real line number.
expect(output).not.toContain('description:')
expect(output).toContain('SKILL.md:8: Analyze retention cohorts weekly.')
})
it('exposes descending scores from searchResults', () => {
const catalog = SkillCatalog.fromZip(
makeArchive({
'funnels/SKILL.md': makeSkill(
'funnels',
'Analyze conversion funnels.',
'# Funnels\n\nBuild a conversion funnel.'
),
'trends/SKILL.md': makeSkill(
'trends',
'Chart metrics over time.',
'# Trends\n\nA funnel is not a trend.'
),
})
)
const results = catalog.searchResults('funnel')
expect(results.map((result) => result.identifier)).toEqual(['funnels', 'trends'])
expect(results[0]!.score!).toBeGreaterThan(results[1]!.score!)
})
it.each(['nonexistentxyz', '--file'])(
'offers a read and line-range recovery hint when a file search for %s finds nothing',
async (query) => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() })
const output = await learn.execute(`posthog:retention-analysis references/functions.md -s "${query}"`)
expect(output).toContain(`No matches for "${query}" in posthog:retention-analysis/references/functions.md.`)
expect(output).toContain(
'Read it with `learn posthog:retention-analysis references/functions.md` (3 lines,'
)
expect(output).toContain('--lines <start>:<end>')
}
)
it('describes a batch of qualified names, tolerating unknown names without failing the batch', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog(), project: makeProjectSkills() })
const output = await learn.execute('-d posthog:funnels posthog:missing project:team-conventions')
expect(output).toBe(
'posthog:funnels: Analyze conversion funnels.\n' +
'[unknown skill: posthog:missing]\n' +
'project:team-conventions: Team-specific conventions.'
)
})
it('caps the batch describe at 20 skills', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() })
const names = Array.from({ length: 21 }, (_, index) => `posthog:s${index}`).join(' ')
await expect(learn.execute(`-d ${names}`)).rejects.toThrow('at most 20 skills')
})
it('reads several skills in one command', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() })
const output = await learn.execute('posthog:funnels posthog:retention-analysis')
expect(output).toContain('Skill: posthog:funnels')
expect(output).toContain('Skill: posthog:retention-analysis')
})
it('reads several files of one skill in one command', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() })
const output = await learn.execute('posthog:retention-analysis SKILL.md references/functions.md')
expect(output).toContain('File: posthog:retention-analysis/SKILL.md')
expect(output).toContain('File: posthog:retention-analysis/references/functions.md')
})
it('rejects a scoped flag combined with multiple paths', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() })
await expect(
learn.execute('posthog:retention-analysis SKILL.md references/functions.md --lines 1:2')
).rejects.toThrow('Usage: learn <source>:<skill>')
})
it.each([
[1, true],
[40, false],
])('recovers from a zero-hit search (project count=%i, inline=%s)', async (count, inline) => {
const projectSkills = makeProjectSkills({
searchResults: vi.fn(async () => []),
listNames: vi.fn(async () => ({ count, names: ['team-conventions'], truncated: false })),
})
const learn = new ExecLearnCatalog([], { posthog: makeCatalog(), project: projectSkills })
const output = await learn.execute('-s zzzznomatch')
expect(output).toContain('No skills matched "zzzznomatch".')
expect(output).toContain('None of the query words appear in any skill.')
if (inline) {
expect(output).toContain('Available skills:')
expect(output).toContain('posthog: funnels, retention-analysis')
expect(output).toContain('project: team-conventions')
} else {
expect(output).toContain('2 posthog and 40 project skills exist')
expect(output).toContain('learn skills')
}
})
it('does not report zero project skills when the project source is unavailable', async () => {
const learn = new ExecLearnCatalog([], {
posthog: makeCatalog(),
projectUnavailableReason: 'This connection is missing the llm_skill:read scope.',
})
const output = await learn.execute('-s zzzznomatch')
expect(output).toContain('project skills were not searched')
expect(output).not.toContain('0 project skills')
expect(output).toContain('posthog: funnels, retention-analysis')
expect(output).toContain('[Project skills unavailable: This connection is missing the llm_skill:read scope.]')
})
describe('skill invocation reporting', () => {
it('reports successful loads with source, path, and read kind — never searches or describes', async () => {
const invocations: unknown[] = []
const learn = new ExecLearnCatalog([], { posthog: makeCatalog(), project: makeProjectSkills() }, (inv) =>
invocations.push(inv)
)
await learn.execute('posthog:retention-analysis')
await learn.execute('posthog:retention-analysis references/functions.md')
await learn.execute('project:team-conventions notes.md --lines 1:5')
await learn.execute('-s retention')
await learn.execute('-d posthog:funnels')
await learn.execute('skills')
expect(invocations).toEqual([
{ source: 'posthog', skill: 'retention-analysis', path: undefined, readKind: 'skill' },
{ source: 'posthog', skill: 'retention-analysis', path: 'references/functions.md', readKind: 'file' },
{ source: 'project', skill: 'team-conventions', path: 'notes.md', readKind: 'file_lines' },
])
})
it('reports one invocation per file in a successful multi-file batch', async () => {
const invocations: unknown[] = []
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() }, (inv) => invocations.push(inv))
await learn.execute('posthog:retention-analysis references/functions.md scripts/run.ts')
expect(invocations).toEqual([
{ source: 'posthog', skill: 'retention-analysis', path: 'references/functions.md', readKind: 'file' },
{ source: 'posthog', skill: 'retention-analysis', path: 'scripts/run.ts', readKind: 'file' },
])
})
it('fires no invocations when a file batch fails on one path', async () => {
const invocations: unknown[] = []
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() }, (inv) => invocations.push(inv))
// The command returns no content, so a successful sibling must not report.
await expect(
learn.execute('posthog:retention-analysis references/functions.md missing.md')
).rejects.toThrow('missing.md')
expect(invocations).toEqual([])
})
it('fires no invocations when a multi-skill batch fails on one skill', async () => {
const invocations: unknown[] = []
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() }, (inv) => invocations.push(inv))
await expect(learn.execute('posthog:retention-analysis posthog:missing')).rejects.toThrow(
'Unknown skill: "missing"'
)
expect(invocations).toEqual([])
})
it('keeps learn working when the listener throws', async () => {
const learn = new ExecLearnCatalog([], { posthog: makeCatalog() }, () => {
throw new Error('capture failed')
})
await expect(learn.execute('posthog:funnels')).resolves.toContain('Build a conversion funnel.')
})
})
})